<commit_before>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package tikv

import (
	"time"

	"github.com/juju/errors"
	"github.com/ngaut/log"
	"github.com/pingcap/kvproto/pkg/coprocessor"
	"github.com/pingcap/kvproto/pkg/errorpb"
	"github.com/pingcap/kvproto/pkg/kvrpcpb"
)

// RegionRequestSender sends KV/Cop requests to tikv server. It handles network
// errors and some region errors internally.
//
// Typically, a KV/Cop request is bind to a region, all keys that are involved
// in the request should be located in the region.
// The sending process begins with looking for the address of leader store's
// address of the target region from cache, and the request is then sent to the
// destination tikv server over TCP connection.
// If region is updated, can be caused by leader transfer, region split, region
// merge, or region balance, tikv server may not able to process request and
// send back a RegionError.
// RegionRequestSender takes care of errors that does not relevant to region
// range, such as 'I/O timeout', 'NotLeader', and 'ServerIsBusy'. For other
// errors, since region range have changed, the request may need to split, so we
// simply return the error to caller.
type RegionRequestSender struct {
	bo          *Backoffer
	regionCache *RegionCache
	client      Client
	storeAddr   string
}

// NewRegionRequestSender creates a new sender.
func NewRegionRequestSender(bo *Backoffer, regionCache *RegionCache, client Client) *RegionRequestSender {
	return &RegionRequestSender{
		bo:          bo,
		regionCache: regionCache,
		client:      client,
	}
}

// SendKVReq sends a KV request to tikv server.
func (s *RegionRequestSender) SendKVReq(req *kvrpcpb.Request, regionID RegionVerID, timeout time.Duration) (*kvrpcpb.Response, error) {
	for {
		select {
		case <-s.bo.ctx.Done():
			return nil, errors.Trace(s.bo.ctx.Err())
		default:
		}

		ctx, err := s.regionCache.GetRPCContext(s.bo, regionID)
		if err != nil {
			return nil, errors.Trace(err)
		}
		if ctx == nil {
			// If the region is not found in cache, it must be out
			// of date and already be cleaned up. We can skip the
			// RPC by returning RegionError directly.
			return &kvrpcpb.Response{
				Type:        req.GetType(),
				RegionError: &errorpb.Error{StaleEpoch: &errorpb.StaleEpoch{}},
			}, nil
		}

		resp, retry, err := s.sendKVReqToRegion(ctx, req, timeout)
		if err != nil {
			return nil, errors.Trace(err)
		}
		if retry {
			continue
		}

		if regionErr := resp.GetRegionError(); regionErr != nil {
			retry, err := s.onRegionError(ctx, regionErr)
			if err != nil {
				return nil, errors.Trace(err)
			}
			if retry {
				continue
			}
			return resp, nil
		}

		if resp.GetType() != req.GetType() {
			return nil, errors.Trace(errMismatch(resp, req))
		}
		return resp, nil
	}
}

// SendCopReq sends a coprocessor request to tikv server.
func (s *RegionRequestSender) SendCopReq(req *coprocessor.Request, regionID RegionVerID, timeout time.Duration) (*coprocessor.Response, error) {
	for {
		ctx, err := s.regionCache.GetRPCContext(s.bo, regionID)
		if err != nil {
			return nil, errors.Trace(err)
		}
		if ctx == nil {
			// If the region is not found in cache, it must be out
			// of date and already be cleaned up. We can skip the
			// RPC by returning RegionError directly.
			return &coprocessor.Response{
				RegionError: &errorpb.Error{StaleEpoch: &errorpb.StaleEpoch{}},
			}, nil
		}

		s.storeAddr = ctx.Addr
		resp, retry, err := s.sendCopReqToRegion(ctx, req, timeout)
		if err != nil {
			return nil, errors.Trace(err)
		}
		if retry {
			continue
		}

		if regionErr := resp.GetRegionError(); regionErr != nil {
			retry, err := s.onRegionError(ctx, regionErr)
			if err != nil {
				return nil, errors.Trace(err)
			}
			if retry {
				continue
			}
		}
		return resp, nil
	}
}

func (s *RegionRequestSender) sendKVReqToRegion(ctx *RPCContext, req *kvrpcpb.Request, timeout time.Duration) (resp *kvrpcpb.Response, retry bool, err error) {
	req.Context = ctx.KVCtx
	resp, err = s.client.SendKVReq(ctx.Context, ctx.Addr, req, timeout)
	if err != nil {
		if e := s.onSendFail(ctx, err); e != nil {
			return nil, false, errors.Trace(e)
		}
		return nil, true, nil
	}
	return
}

func (s *RegionRequestSender) sendCopReqToRegion(ctx *RPCContext, req *coprocessor.Request, timeout time.Duration) (resp *coprocessor.Response, retry bool, err error) {
	req.Context = ctx.KVCtx
	resp, err = s.client.SendCopReq(ctx.Context, ctx.Addr, req, timeout)
	if err != nil {
		if e := s.onSendFail(ctx, err); e != nil {
			return nil, false, errors.Trace(err)
		}
		return nil, true, nil
	}
	return
}

func (s *RegionRequestSender) onSendFail(ctx *RPCContext, err error) error {
	s.regionCache.OnRequestFail(ctx)
	err = s.bo.Backoff(boTiKVRPC, errors.Errorf("send tikv request error: %v, ctx: %s, try next peer later", err, ctx.KVCtx))
	return errors.Trace(err)
}

func (s *RegionRequestSender) onRegionError(ctx *RPCContext, regionErr *errorpb.Error) (retry bool, err error) {
	reportRegionError(regionErr)
	if notLeader := regionErr.GetNotLeader(); notLeader != nil {
		// Retry if error is `NotLeader`.
		log.Debugf("tikv reports `NotLeader`: %s, ctx: %s, retry later", notLeader, ctx.KVCtx)
		s.regionCache.UpdateLeader(ctx.Region, notLeader.GetLeader().GetStoreId())
		if notLeader.GetLeader() == nil {
			err = s.bo.Backoff(boRegionMiss, errors.Errorf("not leader: %v, ctx: %s", notLeader, ctx.KVCtx))
			if err != nil {
				return false, errors.Trace(err)
			}
		}
		return true, nil
	}

	if storeNotMatch := regionErr.GetStoreNotMatch(); storeNotMatch != nil {
		// store not match
		log.Warnf("tikv reports `StoreNotMatch`: %s, ctx: %s, retry later", storeNotMatch, ctx.KVCtx)
		s.regionCache.ClearStoreByID(ctx.GetStoreID())
		return true, nil
	}

	if staleEpoch := regionErr.GetStaleEpoch(); staleEpoch != nil {
		log.Debugf("tikv reports `StaleEpoch`, ctx: %s, retry later", ctx.KVCtx)
		err = s.regionCache.OnRegionStale(ctx, staleEpoch.NewRegions)
		return false, errors.Trace(err)
	}
	if regionErr.GetServerIsBusy() != nil {
		log.Debugf("tikv reports `ServerIsBusy`, ctx: %s, retry later", ctx.KVCtx)
		err = s.bo.Backoff(boServerBusy, errors.Errorf("server is busy, ctx: %s", ctx.KVCtx))
		if err != nil {
			return false, errors.Trace(err)
		}
		return true, nil
	}
	if regionErr.GetStaleCommand() != nil {
		log.Debugf("tikv reports `StaleCommand`, ctx: %s", ctx.KVCtx)
		return true, nil
	}
	if regionErr.GetRaftEntryTooLarge() != nil {
		return false, errors.New(regionErr.String())
	}
	// For other errors, we only drop cache here.
	// Because caller may need to re-split the request.
	log.Debugf("tikv reports region error: %s, ctx: %s", regionErr, ctx.KVCtx)
	s.regionCache.DropRegion(ctx.Region)
	return false, nil
}
<commit_msg>store/tikv: adjust log level for some region errors. (#2765)<commit_after>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package tikv

import (
	"time"

	"github.com/juju/errors"
	"github.com/ngaut/log"
	"github.com/pingcap/kvproto/pkg/coprocessor"
	"github.com/pingcap/kvproto/pkg/errorpb"
	"github.com/pingcap/kvproto/pkg/kvrpcpb"
)

// RegionRequestSender sends KV/Cop requests to tikv server. It handles network
// errors and some region errors internally.
//
// Typically, a KV/Cop request is bind to a region, all keys that are involved
// in the request should be located in the region.
// The sending process begins with looking for the address of leader store's
// address of the target region from cache, and the request is then sent to the
// destination tikv server over TCP connection.
// If region is updated, can be caused by leader transfer, region split, region
// merge, or region balance, tikv server may not able to process request and
// send back a RegionError.
// RegionRequestSender takes care of errors that does not relevant to region
// range, such as 'I/O timeout', 'NotLeader', and 'ServerIsBusy'. For other
// errors, since region range have changed, the request may need to split, so we
// simply return the error to caller.
type RegionRequestSender struct {
	bo          *Backoffer
	regionCache *RegionCache
	client      Client
	storeAddr   string
}

// NewRegionRequestSender creates a new sender.
func NewRegionRequestSender(bo *Backoffer, regionCache *RegionCache, client Client) *RegionRequestSender {
	return &RegionRequestSender{
		bo:          bo,
		regionCache: regionCache,
		client:      client,
	}
}

// SendKVReq sends a KV request to tikv server.
func (s *RegionRequestSender) SendKVReq(req *kvrpcpb.Request, regionID RegionVerID, timeout time.Duration) (*kvrpcpb.Response, error) {
	for {
		select {
		case <-s.bo.ctx.Done():
			return nil, errors.Trace(s.bo.ctx.Err())
		default:
		}

		ctx, err := s.regionCache.GetRPCContext(s.bo, regionID)
		if err != nil {
			return nil, errors.Trace(err)
		}
		if ctx == nil {
			// If the region is not found in cache, it must be out
			// of date and already be cleaned up. We can skip the
			// RPC by returning RegionError directly.
			return &kvrpcpb.Response{
				Type:        req.GetType(),
				RegionError: &errorpb.Error{StaleEpoch: &errorpb.StaleEpoch{}},
			}, nil
		}

		resp, retry, err := s.sendKVReqToRegion(ctx, req, timeout)
		if err != nil {
			return nil, errors.Trace(err)
		}
		if retry {
			continue
		}

		if regionErr := resp.GetRegionError(); regionErr != nil {
			retry, err := s.onRegionError(ctx, regionErr)
			if err != nil {
				return nil, errors.Trace(err)
			}
			if retry {
				continue
			}
			return resp, nil
		}

		if resp.GetType() != req.GetType() {
			return nil, errors.Trace(errMismatch(resp, req))
		}
		return resp, nil
	}
}

// SendCopReq sends a coprocessor request to tikv server.
func (s *RegionRequestSender) SendCopReq(req *coprocessor.Request, regionID RegionVerID, timeout time.Duration) (*coprocessor.Response, error) {
	for {
		ctx, err := s.regionCache.GetRPCContext(s.bo, regionID)
		if err != nil {
			return nil, errors.Trace(err)
		}
		if ctx == nil {
			// If the region is not found in cache, it must be out
			// of date and already be cleaned up. We can skip the
			// RPC by returning RegionError directly.
			return &coprocessor.Response{
				RegionError: &errorpb.Error{StaleEpoch: &errorpb.StaleEpoch{}},
			}, nil
		}

		s.storeAddr = ctx.Addr
		resp, retry, err := s.sendCopReqToRegion(ctx, req, timeout)
		if err != nil {
			return nil, errors.Trace(err)
		}
		if retry {
			continue
		}

		if regionErr := resp.GetRegionError(); regionErr != nil {
			retry, err := s.onRegionError(ctx, regionErr)
			if err != nil {
				return nil, errors.Trace(err)
			}
			if retry {
				continue
			}
		}
		return resp, nil
	}
}

func (s *RegionRequestSender) sendKVReqToRegion(ctx *RPCContext, req *kvrpcpb.Request, timeout time.Duration) (resp *kvrpcpb.Response, retry bool, err error) {
	req.Context = ctx.KVCtx
	resp, err = s.client.SendKVReq(ctx.Context, ctx.Addr, req, timeout)
	if err != nil {
		if e := s.onSendFail(ctx, err); e != nil {
			return nil, false, errors.Trace(e)
		}
		return nil, true, nil
	}
	return
}

func (s *RegionRequestSender) sendCopReqToRegion(ctx *RPCContext, req *coprocessor.Request, timeout time.Duration) (resp *coprocessor.Response, retry bool, err error) {
	req.Context = ctx.KVCtx
	resp, err = s.client.SendCopReq(ctx.Context, ctx.Addr, req, timeout)
	if err != nil {
		if e := s.onSendFail(ctx, err); e != nil {
			return nil, false, errors.Trace(err)
		}
		return nil, true, nil
	}
	return
}

func (s *RegionRequestSender) onSendFail(ctx *RPCContext, err error) error {
	s.regionCache.OnRequestFail(ctx)
	err = s.bo.Backoff(boTiKVRPC, errors.Errorf("send tikv request error: %v, ctx: %s, try next peer later", err, ctx.KVCtx))
	return errors.Trace(err)
}

func (s *RegionRequestSender) onRegionError(ctx *RPCContext, regionErr *errorpb.Error) (retry bool, err error) {
	reportRegionError(regionErr)
	if notLeader := regionErr.GetNotLeader(); notLeader != nil {
		// Retry if error is `NotLeader`.
		log.Debugf("tikv reports `NotLeader`: %s, ctx: %s, retry later", notLeader, ctx.KVCtx)
		s.regionCache.UpdateLeader(ctx.Region, notLeader.GetLeader().GetStoreId())
		if notLeader.GetLeader() == nil {
			err = s.bo.Backoff(boRegionMiss, errors.Errorf("not leader: %v, ctx: %s", notLeader, ctx.KVCtx))
			if err != nil {
				return false, errors.Trace(err)
			}
		}
		return true, nil
	}

	if storeNotMatch := regionErr.GetStoreNotMatch(); storeNotMatch != nil {
		// store not match
		log.Warnf("tikv reports `StoreNotMatch`: %s, ctx: %s, retry later", storeNotMatch, ctx.KVCtx)
		s.regionCache.ClearStoreByID(ctx.GetStoreID())
		return true, nil
	}

	if staleEpoch := regionErr.GetStaleEpoch(); staleEpoch != nil {
		log.Debugf("tikv reports `StaleEpoch`, ctx: %s, retry later", ctx.KVCtx)
		err = s.regionCache.OnRegionStale(ctx, staleEpoch.NewRegions)
		return false, errors.Trace(err)
	}
	if regionErr.GetServerIsBusy() != nil {
		log.Warnf("tikv reports `ServerIsBusy`, ctx: %s, retry later", ctx.KVCtx)
		err = s.bo.Backoff(boServerBusy, errors.Errorf("server is busy, ctx: %s", ctx.KVCtx))
		if err != nil {
			return false, errors.Trace(err)
		}
		return true, nil
	}
	if regionErr.GetStaleCommand() != nil {
		log.Debugf("tikv reports `StaleCommand`, ctx: %s", ctx.KVCtx)
		return true, nil
	}
	if regionErr.GetRaftEntryTooLarge() != nil {
		log.Warnf("tikv reports `RaftEntryTooLarge`, ctx: %s", ctx.KVCtx)
		return false, errors.New(regionErr.String())
	}
	// For other errors, we only drop cache here.
	// Because caller may need to re-split the request.
	log.Debugf("tikv reports region error: %s, ctx: %s", regionErr, ctx.KVCtx)
	s.regionCache.DropRegion(ctx.Region)
	return false, nil
}
<|endoftext|>


<commit_before>package device

import (
	"fmt"
	"github.com/Comcast/webpa-common/logging"
	"github.com/Comcast/webpa-common/wrp"
	"io/ioutil"
	"os"
)

func ExampleManagerSimple() {
	options := &Options{
		Logger: &logging.LoggerWriter{ioutil.Discard},
		MessageListener: func(device Interface, raw []byte, message *wrp.Message) {
			fmt.Printf("%s -> %s\n", message.Destination, message.Payload)
			err := device.Send(
				wrp.NewSimpleRequestResponse(message.Destination, message.Source, []byte("Homer Simpson, smiling politely")),
			)

			if err != nil {
				fmt.Fprintf(os.Stderr, "Unable to send response: %s", err)
			}
		},
	}

	_, server, websocketURL := startWebsocketServer(options)
	defer server.Close()

	dialer := NewDialer(options, nil)
	connection, _, err := dialer.Dial(
		websocketURL,
		"mac:111122223333",
		nil,
		nil,
	)

	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to connect to server: %s\n", err)
		return
	}

	defer connection.Close()
	requestMessage := wrp.NewSimpleRequestResponse("destination.com", "somewhere.com", []byte("Billy Corgan, Smashing Pumpkins"))
	if err := connection.Write(requestMessage); err != nil {
		fmt.Fprintf(os.Stderr, "Unable to send event: %s", err)
		return
	}

	_, responseMessage, err := connection.Read()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to read response: %s", err)
		return
	}

	fmt.Printf("%s\n", responseMessage.Payload)

	// Output:
	// destination.com -> Billy Corgan, Smashing Pumpkins
	// Homer Simpson, smiling politely
}
<commit_msg>Fixed the examples<commit_after>package device

import (
	"bytes"
	"fmt"
	"github.com/Comcast/webpa-common/logging"
	"github.com/Comcast/webpa-common/wrp"
	"io/ioutil"
	"os"
)

func ExampleManagerSimple() {
	options := &Options{
		Logger: &logging.LoggerWriter{ioutil.Discard},
		MessageListener: func(device Interface, raw []byte, message *wrp.Message) {
			fmt.Printf("%s -> %s\n", message.Destination, message.Payload)
			err := device.Send(
				wrp.NewSimpleRequestResponse(message.Destination, message.Source, []byte("Homer Simpson, smiling politely")),
			)

			if err != nil {
				fmt.Fprintf(os.Stderr, "Unable to send response: %s", err)
			}
		},
	}

	_, server, websocketURL := startWebsocketServer(options)
	defer server.Close()

	dialer := NewDialer(options, nil)
	connection, _, err := dialer.Dial(
		websocketURL,
		"mac:111122223333",
		nil,
		nil,
	)

	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to connect to server: %s\n", err)
		return
	}

	defer connection.Close()
	var (
		requestMessage = wrp.NewSimpleRequestResponse("destination.com", "somewhere.com", []byte("Billy Corgan, Smashing Pumpkins"))
		encoder        = wrp.NewEncoder(connection, wrp.Msgpack)
	)

	if err := encoder.Encode(requestMessage); err != nil {
		fmt.Printf("Unable to send event: %s\n", err)
		return
	}

	var (
		responseMessage wrp.Message
		responseBuffer  bytes.Buffer
		decoder         = wrp.NewDecoder(&responseBuffer, wrp.Msgpack)
	)

	if frameRead, err := connection.Read(&responseBuffer); err != nil {
		fmt.Printf("Unable to read response: %s\n", err)
		return
	} else if !frameRead {
		fmt.Println("Response frame skipped")
		return
	} else if err := decoder.Decode(&responseMessage); err != nil {
		fmt.Printf("Unable to decode response: %s\n", err)
		return
	}

	fmt.Printf("%s\n", responseMessage.Payload)

	// Output:
	// destination.com -> Billy Corgan, Smashing Pumpkins
	// Homer Simpson, smiling politely
}
<|endoftext|>


<commit_before>// Package analytics provides the methods to run an analytics reporting system
// for API requests which may be useful to users for measuring access and
// possibly identifying bad actors abusing requests.
package analytics

import (
	"fmt"
	"log"
	"net/http"
	"strings"
	"time"

	"runtime"

	"encoding/json"

	"github.com/boltdb/bolt"
)

type apiRequest struct {
	URL        string `json:"url"`
	Method     string `json:"http_method"`
	Origin     string `json:"origin"`
	Proto      string `json:"http_protocol"`
	RemoteAddr string `json:"ip_address"`
	Timestamp  int64  `json:"timestamp"`
	External   bool   `json:"external"`
}

var (
	store      *bolt.DB
	recordChan chan apiRequest
)

// Record queues an apiRequest for metrics
func Record(req *http.Request) {
	external := strings.Contains(req.URL.Path, "/external/")

	r := apiRequest{
		URL:        req.URL.String(),
		Method:     req.Method,
		Origin:     req.Header.Get("Origin"),
		Proto:      req.Proto,
		RemoteAddr: req.RemoteAddr,
		Timestamp:  time.Now().Unix() * 1000,
		External:   external,
	}

	// put r on buffered recordChan to take advantage of batch insertion in DB
	recordChan <- r
}

// Close exports the abillity to close our db file. Should be called with defer
// after call to Init() from the same place.
func Close() {
	err := store.Close()
	if err != nil {
		log.Println(err)
	}
}

// Init creates a db connection, should run an initial prune of old data, and
// sets up the queue/batching channel
func Init() {
	var err error
	store, err = bolt.Open("analytics.db", 0666, nil)
	if err != nil {
		log.Fatalln(err)
	}

	recordChan = make(chan apiRequest, 1024*64*runtime.NumCPU())

	go serve()

	if err != nil {
		log.Fatalln(err)
	}
}

func serve() {
	// make timer to notify select to batch request insert from recordChan
	// interval: 30 seconds
	apiRequestTimer := time.NewTicker(time.Second * 30)

	// make timer to notify select to remove old analytics
	// interval: 2 weeks
	// TODO: enable analytics backup service to cloud
	pruneDBTimer := time.NewTicker(time.Hour * 24 * 14)

	for {
		select {
		case <-apiRequestTimer.C:
			var reqs []apiRequest
			batchSize := len(recordChan)

			for i := 0; i < batchSize; i++ {
				reqs = append(reqs, <-recordChan)
			}

			err := batchInsert(reqs)
			if err != nil {
				log.Println(err)
			}

		case <-pruneDBTimer.C:

		default:
		}
	}
}

// Week returns the map containing decoded javascript needed to chart a week of data by day
func Week() (map[string]string, error) {
	// set thresholds for today and the 6 days preceeding
	times := [7]time.Time{}
	now := time.Now()
	today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)

	for i := range times {
		// subtract 24 * i hours to make days prior
		dur := time.Duration(24 * i * -1)
		day := today.Add(time.Hour * dur)

		// day threshold is [...n-1-i, n-1, n]
		times[len(times)-1-i] = day
	}

	// get api request analytics from db
	var requests = []apiRequest{}
	err := store.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("requests"))

		err := b.ForEach(func(k, v []byte) error {
			var r apiRequest
			err := json.Unmarshal(v, &r)
			if err != nil {
				log.Println("Error decoding json from analytics db:", err)
				return nil
			}

			requests = append(requests, r)

			return nil
		})
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return nil, err
	}

	dates := [7]string{}
	ips := [7]map[string]struct{}{}
	total := [7]int{}
	unique := [7]int{}

CHECK_REQUEST:
	for i := range requests {
		ts := time.Unix(requests[i].Timestamp/1000, 0)

		for j := range times {
			// format times[j] (time.Time) into a MM/DD format for dates
			dates[j] = times[j].Format("01/02")

			// if on today, there will be no next iteration to set values for
			// day prior so all valid requests belong to today
			if j == len(times) {
				if ts.After(times[j]) || ts.Equal(times[j]) {
					// do all record keeping
					total[j]++

					if _, ok := ips[j][requests[i].RemoteAddr]; !ok {
						unique[j-1]++
						ips[j][requests[i].RemoteAddr] = struct{}{}
					}
				}
			}

			if ts.Equal(times[j]) {
				// increment total count for current time threshold (day)
				total[j]++

				// if no IP found for current threshold, increment unique and record IP
				if _, ok := ips[j][requests[i].RemoteAddr]; !ok {
					unique[j]++
					ips[j][requests[i].RemoteAddr] = struct{}{}
				}
			}

			if ts.Before(times[j]) {
				// check if older than earliest threshold
				if j == 0 {
					continue CHECK_REQUEST
				}

				// increment total count for previous time threshold (day)
				total[j-1]++

				// if no IP found for day prior, increment unique and record IP
				if _, ok := ips[j-1][requests[i].RemoteAddr]; !ok {
					unique[j-1]++
					ips[j-1][requests[i].RemoteAddr] = struct{}{}
				}
			}
		}
	}

	jsUnique, err := json.Marshal(unique)
	if err != nil {
		return nil, err
	}

	jsTotal, err := json.Marshal(total)
	if err != nil {
		return nil, err
	}

	jsDates, err := json.Marshal(dates)
	if err != nil {
		return nil, err
	}

	fmt.Println(dates)

	return map[string]string{
		"dates":  string(jsDates),
		"unique": string(jsUnique),
		"total":  string(jsTotal),
	}, nil
}
<commit_msg>test for unescaping string in js map<commit_after>// Package analytics provides the methods to run an analytics reporting system
// for API requests which may be useful to users for measuring access and
// possibly identifying bad actors abusing requests.
package analytics

import (
	"html"
	"log"
	"net/http"
	"strings"
	"time"

	"runtime"

	"encoding/json"

	"github.com/boltdb/bolt"
)

type apiRequest struct {
	URL        string `json:"url"`
	Method     string `json:"http_method"`
	Origin     string `json:"origin"`
	Proto      string `json:"http_protocol"`
	RemoteAddr string `json:"ip_address"`
	Timestamp  int64  `json:"timestamp"`
	External   bool   `json:"external"`
}

var (
	store      *bolt.DB
	recordChan chan apiRequest
)

// Record queues an apiRequest for metrics
func Record(req *http.Request) {
	external := strings.Contains(req.URL.Path, "/external/")

	r := apiRequest{
		URL:        req.URL.String(),
		Method:     req.Method,
		Origin:     req.Header.Get("Origin"),
		Proto:      req.Proto,
		RemoteAddr: req.RemoteAddr,
		Timestamp:  time.Now().Unix() * 1000,
		External:   external,
	}

	// put r on buffered recordChan to take advantage of batch insertion in DB
	recordChan <- r
}

// Close exports the abillity to close our db file. Should be called with defer
// after call to Init() from the same place.
func Close() {
	err := store.Close()
	if err != nil {
		log.Println(err)
	}
}

// Init creates a db connection, should run an initial prune of old data, and
// sets up the queue/batching channel
func Init() {
	var err error
	store, err = bolt.Open("analytics.db", 0666, nil)
	if err != nil {
		log.Fatalln(err)
	}

	recordChan = make(chan apiRequest, 1024*64*runtime.NumCPU())

	go serve()

	if err != nil {
		log.Fatalln(err)
	}
}

func serve() {
	// make timer to notify select to batch request insert from recordChan
	// interval: 30 seconds
	apiRequestTimer := time.NewTicker(time.Second * 30)

	// make timer to notify select to remove old analytics
	// interval: 2 weeks
	// TODO: enable analytics backup service to cloud
	pruneDBTimer := time.NewTicker(time.Hour * 24 * 14)

	for {
		select {
		case <-apiRequestTimer.C:
			var reqs []apiRequest
			batchSize := len(recordChan)

			for i := 0; i < batchSize; i++ {
				reqs = append(reqs, <-recordChan)
			}

			err := batchInsert(reqs)
			if err != nil {
				log.Println(err)
			}

		case <-pruneDBTimer.C:

		default:
		}
	}
}

// Week returns the map containing decoded javascript needed to chart a week of data by day
func Week() (map[string]string, error) {
	// set thresholds for today and the 6 days preceeding
	times := [7]time.Time{}
	now := time.Now()
	today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)

	for i := range times {
		// subtract 24 * i hours to make days prior
		dur := time.Duration(24 * i * -1)
		day := today.Add(time.Hour * dur)

		// day threshold is [...n-1-i, n-1, n]
		times[len(times)-1-i] = day
	}

	// get api request analytics from db
	var requests = []apiRequest{}
	err := store.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("requests"))

		err := b.ForEach(func(k, v []byte) error {
			var r apiRequest
			err := json.Unmarshal(v, &r)
			if err != nil {
				log.Println("Error decoding json from analytics db:", err)
				return nil
			}

			requests = append(requests, r)

			return nil
		})
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return nil, err
	}

	dates := [7]string{}
	ips := [7]map[string]struct{}{}
	total := [7]int{}
	unique := [7]int{}

CHECK_REQUEST:
	for i := range requests {
		ts := time.Unix(requests[i].Timestamp/1000, 0)

		for j := range times {
			// format times[j] (time.Time) into a MM/DD format for dates
			dates[j] = times[j].Format("01/02")

			// if on today, there will be no next iteration to set values for
			// day prior so all valid requests belong to today
			if j == len(times) {
				if ts.After(times[j]) || ts.Equal(times[j]) {
					// do all record keeping
					total[j]++

					if _, ok := ips[j][requests[i].RemoteAddr]; !ok {
						unique[j-1]++
						ips[j][requests[i].RemoteAddr] = struct{}{}
					}
				}
			}

			if ts.Equal(times[j]) {
				// increment total count for current time threshold (day)
				total[j]++

				// if no IP found for current threshold, increment unique and record IP
				if _, ok := ips[j][requests[i].RemoteAddr]; !ok {
					unique[j]++
					ips[j][requests[i].RemoteAddr] = struct{}{}
				}
			}

			if ts.Before(times[j]) {
				// check if older than earliest threshold
				if j == 0 {
					continue CHECK_REQUEST
				}

				// increment total count for previous time threshold (day)
				total[j-1]++

				// if no IP found for day prior, increment unique and record IP
				if _, ok := ips[j-1][requests[i].RemoteAddr]; !ok {
					unique[j-1]++
					ips[j-1][requests[i].RemoteAddr] = struct{}{}
				}
			}
		}
	}

	jsUnique, err := json.Marshal(unique)
	if err != nil {
		return nil, err
	}

	jsTotal, err := json.Marshal(total)
	if err != nil {
		return nil, err
	}

	jsDates, err := json.Marshal(dates)
	if err != nil {
		return nil, err
	}

	return map[string]string{
		"dates":  html.UnescapeString(string(jsDates)),
		"unique": string(jsUnique),
		"total":  string(jsTotal),
	}, nil
}
<|endoftext|>


<commit_before>package websockets

// Message is the data to send back
type Message struct {
	Author string `json:"author"`
	Body   string `json:"body"`
}

// Ping is a simple ping message
type Ping struct {
	P int `json:"p"`
}

func (m *Message) String() string {
	return m.Author + " says " + m.Body
}
<commit_msg>Make websocket messages type agnostic<commit_after>package websockets

// Message is the data to send back
type Message struct {
	Data interface{} `json:"data"`
}

// Ping is a simple ping message
type Ping struct {
	P int `json:"p"`
}
<|endoftext|>


<commit_before>// replicator.go copies S3 files from one bucket to another.
// This is used to replicate files in one S3 region (Virginia)
// to another region (Oregon).

package workers

import (
	"encoding/json"
	"fmt"
	"github.com/APTrust/bagman/bagman"
	"github.com/bitly/go-nsq"
	"github.com/crowdmob/goamz/aws"
	"github.com/crowdmob/goamz/s3"
	"os"
	"path/filepath"
	"time"
)

type ReplicationObject struct {
	File       *bagman.File
	NsqMessage *nsq.Message
}

type Replicator struct {
	ReplicationChannel  chan *ReplicationObject
	S3ReplicationClient *bagman.S3Client
	ProcUtil            *bagman.ProcessUtil
}

func NewReplicator(procUtil *bagman.ProcessUtil) (*Replicator) {
	replicationClient, _ := bagman.NewS3Client(aws.USWest2)
	replicator := &Replicator{
		ProcUtil: procUtil,
		S3ReplicationClient: replicationClient,
	}
	workerBufferSize := procUtil.Config.StoreWorker.Workers * 10
	replicator.ReplicationChannel = make(chan *ReplicationObject, workerBufferSize)
	for i := 0; i < procUtil.Config.StoreWorker.Workers; i++ {
		go replicator.replicate()
	}
	return replicator
}

// MessageHandler handles messages from the queue, putting each
// item into the replication channel.
func (replicator *Replicator) HandleMessage(message *nsq.Message) error {
	message.DisableAutoResponse()
	var file bagman.File
	err := json.Unmarshal(message.Body, &file)
	if err != nil {
		replicator.ProcUtil.MessageLog.Error("Could not unmarshal JSON data from nsq:",
			string(message.Body))
		message.Finish()
		return fmt.Errorf("Could not unmarshal JSON data from nsq")
	}
	if replicator.ReplicatedFileExists(&file) {
		replicator.ProcUtil.MessageLog.Info("File %s already exists in replication bucket",
			file.Identifier)
		message.Finish()
		return nil
	}
	replicationObject := &ReplicationObject{
		NsqMessage: message,
		File: &file,
	}

	// Unfortunately, we're probably running on a machine that's
	// also running either ingest or restore. The Volume monitor
	// can know how much disk space those processes are using, but
	// not how much they have reserved. So until we have a volume
	// monitoring service, all we can do is pad the estimate of
	// how much space we might need.
	err = replicator.ProcUtil.Volume.Reserve(uint64(file.Size * 2))
	if err != nil {
		// Not enough room on disk
		replicator.ProcUtil.MessageLog.Warning("Requeueing %s (%d bytes) - not enough disk space",
			file.Identifier, file.Size)
		message.Requeue(10 * time.Minute)
		return nil
	}

	replicator.ReplicationChannel <- replicationObject
	replicator.ProcUtil.MessageLog.Debug("Put %s (%d bytes) into replication queue",
		file.Identifier, file.Size)
	return nil
}

func (replicator *Replicator) ReplicatedFileExists(file *bagman.File) (bool) {
	exists, _ := replicator.S3ReplicationClient.Exists(
		replicator.ProcUtil.Config.PreservationBucket,
		file.Uuid)
	return exists
}

func (replicator *Replicator) replicate() {
	for replicationObject := range replicator.ReplicationChannel {
		replicator.ProcUtil.MessageLog.Info("Starting %s",
			replicationObject.File.Identifier)
		url, err := replicator.CopyFile(replicationObject)
		if err != nil {
			replicator.ProcUtil.MessageLog.Error(
				"Requeuing %s (%s) because copy failed. Error: %v",
				replicationObject.File.Identifier,
				replicationObject.File.StorageURL,
				err)
			replicationObject.NsqMessage.Requeue(5 * time.Minute)
		} else {
			replicator.ProcUtil.MessageLog.Info("Finished %s. Replication " +
				"copy is at %s.",
				replicationObject.File.Identifier,
				url)
			replicationObject.NsqMessage.Finish()
		}
	}
}

// Copies a file from one bucket to another, across regions,
// including all of APTrust's custom metadata. Returns the URL
// of the destination file (that should be in the replication
// bucket in Oregon), or an error.
//
// This does NOT use PUT COPY internally because PUT COPY is
// limited to files of 5GB or less, and we'll have many files
// over 5GB. The copy operation downloads data from the S3
// preservation bucket and uploads it to the replication bucket.
//
// As long as we're running in the same region as our S3
// preservation bucket (USEast), the download should be fast
// and free. Running this code outside of USEast will be
// slow and expensive, since we'll have to pay for the bandwidth
// of BOTH download and upload.
func (replicator *Replicator) CopyFile(replicationObject *ReplicationObject) (string, error) {
	replicator.ProcUtil.MessageLog.Info("Starting copy of %s (%s)",
		replicationObject.File.Identifier, replicationObject.File.Uuid)
	// Copy options include the md5 sum of the file we're copying
	// and all of our custom meta data.
	copyOptions, err := replicator.GetCopyOptions(replicationObject.File)
	if err != nil {
		return "", err
	}

	// Touch before dowload, because large files can take a long time!
	replicationObject.NsqMessage.Touch()

	localPath, err := replicator.DownloadFromPreservation(replicationObject.File)
	if err != nil {
		return "", err
	}

	reader, err := os.Open(localPath)
	if err != nil {
		return "", err
	}

	// Touch again before upload, because large files are slow
	replicationObject.NsqMessage.Touch()

	// Replication client is configured to us USWest-2 (Oregon),
	// but the bucket name should be enough.
	url := ""
	if replicationObject.File.Size <= bagman.S3_LARGE_FILE {
		url, err = replicator.S3ReplicationClient.SaveToS3(
			replicator.ProcUtil.Config.ReplicationBucket,
			replicationObject.File.Uuid,
			replicationObject.File.MimeType,
			reader,
			replicationObject.File.Size,
			copyOptions)
	} else {
		url, err = replicator.S3ReplicationClient.SaveLargeFileToS3(
			replicator.ProcUtil.Config.ReplicationBucket,
			replicationObject.File.Uuid,
			replicationObject.File.MimeType,
			reader,
			replicationObject.File.Size,
			copyOptions,
			bagman.S3_CHUNK_SIZE)
	}

	// Touch so NSQ knows we're not dead yet!
	replicationObject.NsqMessage.Touch()

	// Delete the local file.
	delErr := os.Remove(localPath)
	if delErr != nil {
		replicator.ProcUtil.MessageLog.Warning("Could not delete local file %s: %v",
			localPath, delErr)
	} else {
		replicator.ProcUtil.Volume.Release(uint64(replicationObject.File.Size * 2))
	}

	if err == nil {
		replicator.ProcUtil.MessageLog.Info("Finished copy of %s (%s)",
			replicationObject.File.Identifier,
			replicationObject.File.Uuid)
	}

	return url, err
}

// Returns S3 options, including the md5 checksum and APTrust's custom
// metadata. These options must accompany the file copy.
func (replicator *Replicator) GetCopyOptions(file *bagman.File) (s3.Options, error) {
	// Copy all of the meta data
	resp, err := replicator.ProcUtil.S3Client.Head(
		replicator.ProcUtil.Config.PreservationBucket,
		file.Uuid)
	if err != nil {
		detailedErr :=  fmt.Errorf("Head request for %s at %s returned error: %v",
			file.Uuid,
			replicator.ProcUtil.Config.PreservationBucket,
			err)
		return s3.Options{}, detailedErr
	}

	s3Metadata := make(map[string][]string)
	s3Metadata["md5"]         = []string{ file.Md5 }
	s3Metadata["institution"] = []string{ resp.Header["X-Amz-Meta-Institution"][0] }
	s3Metadata["bag"]         = []string{ resp.Header["X-Amz-Meta-Bag"][0] }
	s3Metadata["bagpath"]     = []string{ file.Path }

	base64md5, err := bagman.Base64EncodeMd5(file.Md5)
	if err != nil {
		return s3.Options{}, err
	}

	options := replicator.S3ReplicationClient.MakeOptions(base64md5, s3Metadata)
	return options, nil
}

// Copies a file from the preservation bucket to a local file
// and returns the path to the local file.
func (replicator *Replicator) DownloadFromPreservation(file *bagman.File) (string, error) {
	// Make sure we have a folder to put the file in.
	replicationDir := filepath.Join(
		replicator.ProcUtil.Config.PreservationBucket,
		"replication")
	if _, err := os.Stat(replicationDir); os.IsNotExist(err) {
		err = os.MkdirAll(replicationDir, 0755)
		if err != nil {
			return "", err
		}
	}

	// Now copy the file from S3 to local path.
	localPath := filepath.Join(replicationDir, file.Uuid)
	err := replicator.ProcUtil.S3Client.FetchToFileWithoutChecksum(
		replicator.ProcUtil.Config.PreservationBucket,
		file.Uuid,
		localPath)
	if err != nil {
		detailedErr := fmt.Errorf("Cannot read file %s from %s: %v",
			file.Uuid,
			replicator.ProcUtil.Config.PreservationBucket,
			err)
		return "", detailedErr
	}

	replicator.ProcUtil.MessageLog.Info("Downloaded %s", localPath)
	return localPath, nil
}
<commit_msg>Log stats<commit_after>// replicator.go copies S3 files from one bucket to another.
// This is used to replicate files in one S3 region (Virginia)
// to another region (Oregon).

package workers

import (
	"encoding/json"
	"fmt"
	"github.com/APTrust/bagman/bagman"
	"github.com/bitly/go-nsq"
	"github.com/crowdmob/goamz/aws"
	"github.com/crowdmob/goamz/s3"
	"os"
	"path/filepath"
	"time"
)

type ReplicationObject struct {
	File       *bagman.File
	NsqMessage *nsq.Message
}

type Replicator struct {
	ReplicationChannel  chan *ReplicationObject
	S3ReplicationClient *bagman.S3Client
	ProcUtil            *bagman.ProcessUtil
}

func NewReplicator(procUtil *bagman.ProcessUtil) (*Replicator) {
	replicationClient, _ := bagman.NewS3Client(aws.USWest2)
	replicator := &Replicator{
		ProcUtil: procUtil,
		S3ReplicationClient: replicationClient,
	}
	workerBufferSize := procUtil.Config.StoreWorker.Workers * 10
	replicator.ReplicationChannel = make(chan *ReplicationObject, workerBufferSize)
	for i := 0; i < procUtil.Config.StoreWorker.Workers; i++ {
		go replicator.replicate()
	}
	return replicator
}

// MessageHandler handles messages from the queue, putting each
// item into the replication channel.
func (replicator *Replicator) HandleMessage(message *nsq.Message) error {
	message.DisableAutoResponse()
	var file bagman.File
	err := json.Unmarshal(message.Body, &file)
	if err != nil {
		replicator.ProcUtil.MessageLog.Error("Could not unmarshal JSON data from nsq:",
			string(message.Body))
		message.Finish()
		return fmt.Errorf("Could not unmarshal JSON data from nsq")
	}
	if replicator.ReplicatedFileExists(&file) {
		replicator.ProcUtil.MessageLog.Info("File %s already exists in replication bucket",
			file.Identifier)
		message.Finish()
		return nil
	}
	replicationObject := &ReplicationObject{
		NsqMessage: message,
		File: &file,
	}

	// Unfortunately, we're probably running on a machine that's
	// also running either ingest or restore. The Volume monitor
	// can know how much disk space those processes are using, but
	// not how much they have reserved. So until we have a volume
	// monitoring service, all we can do is pad the estimate of
	// how much space we might need.
	err = replicator.ProcUtil.Volume.Reserve(uint64(file.Size * 2))
	if err != nil {
		// Not enough room on disk
		replicator.ProcUtil.MessageLog.Warning("Requeueing %s (%d bytes) - not enough disk space",
			file.Identifier, file.Size)
		message.Requeue(10 * time.Minute)
		return nil
	}

	replicator.ReplicationChannel <- replicationObject
	replicator.ProcUtil.MessageLog.Debug("Put %s (%d bytes) into replication queue",
		file.Identifier, file.Size)
	return nil
}

func (replicator *Replicator) ReplicatedFileExists(file *bagman.File) (bool) {
	exists, _ := replicator.S3ReplicationClient.Exists(
		replicator.ProcUtil.Config.PreservationBucket,
		file.Uuid)
	return exists
}

func (replicator *Replicator) replicate() {
	for replicationObject := range replicator.ReplicationChannel {
		replicator.ProcUtil.MessageLog.Info("Starting %s",
			replicationObject.File.Identifier)
		url, err := replicator.CopyFile(replicationObject)
		if err != nil {
			replicator.ProcUtil.MessageLog.Error(
				"Requeuing %s (%s) because copy failed. Error: %v",
				replicationObject.File.Identifier,
				replicationObject.File.StorageURL,
				err)
			replicationObject.NsqMessage.Requeue(5 * time.Minute)
			replicator.ProcUtil.IncrementFailed()
		} else {
			replicator.ProcUtil.MessageLog.Info("Finished %s. Replication " +
				"copy is at %s.",
				replicationObject.File.Identifier,
				url)
			replicationObject.NsqMessage.Finish()
			replicator.ProcUtil.IncrementSucceeded()
		}
		replicator.ProcUtil.MessageLog.Info(
			"**STATS** Succeeded: %d, Failed: %d",
			replicator.ProcUtil.Succeeded(),
			replicator.ProcUtil.Failed())
	}
}

// Copies a file from one bucket to another, across regions,
// including all of APTrust's custom metadata. Returns the URL
// of the destination file (that should be in the replication
// bucket in Oregon), or an error.
//
// This does NOT use PUT COPY internally because PUT COPY is
// limited to files of 5GB or less, and we'll have many files
// over 5GB. The copy operation downloads data from the S3
// preservation bucket and uploads it to the replication bucket.
//
// As long as we're running in the same region as our S3
// preservation bucket (USEast), the download should be fast
// and free. Running this code outside of USEast will be
// slow and expensive, since we'll have to pay for the bandwidth
// of BOTH download and upload.
func (replicator *Replicator) CopyFile(replicationObject *ReplicationObject) (string, error) {
	replicator.ProcUtil.MessageLog.Info("Starting copy of %s (%s)",
		replicationObject.File.Identifier, replicationObject.File.Uuid)
	// Copy options include the md5 sum of the file we're copying
	// and all of our custom meta data.
	copyOptions, err := replicator.GetCopyOptions(replicationObject.File)
	if err != nil {
		return "", err
	}

	// Touch before dowload, because large files can take a long time!
	replicationObject.NsqMessage.Touch()

	localPath, err := replicator.DownloadFromPreservation(replicationObject.File)
	if err != nil {
		return "", err
	}

	reader, err := os.Open(localPath)
	if err != nil {
		return "", err
	}

	// Touch again before upload, because large files are slow
	replicationObject.NsqMessage.Touch()

	// Replication client is configured to us USWest-2 (Oregon),
	// but the bucket name should be enough.
	url := ""
	if replicationObject.File.Size <= bagman.S3_LARGE_FILE {
		url, err = replicator.S3ReplicationClient.SaveToS3(
			replicator.ProcUtil.Config.ReplicationBucket,
			replicationObject.File.Uuid,
			replicationObject.File.MimeType,
			reader,
			replicationObject.File.Size,
			copyOptions)
	} else {
		url, err = replicator.S3ReplicationClient.SaveLargeFileToS3(
			replicator.ProcUtil.Config.ReplicationBucket,
			replicationObject.File.Uuid,
			replicationObject.File.MimeType,
			reader,
			replicationObject.File.Size,
			copyOptions,
			bagman.S3_CHUNK_SIZE)
	}

	// Touch so NSQ knows we're not dead yet!
	replicationObject.NsqMessage.Touch()

	// Delete the local file.
	delErr := os.Remove(localPath)
	if delErr != nil {
		replicator.ProcUtil.MessageLog.Warning("Could not delete local file %s: %v",
			localPath, delErr)
	} else {
		replicator.ProcUtil.Volume.Release(uint64(replicationObject.File.Size * 2))
	}

	if err == nil {
		replicator.ProcUtil.MessageLog.Info("Finished copy of %s (%s)",
			replicationObject.File.Identifier,
			replicationObject.File.Uuid)
	}

	return url, err
}

// Returns S3 options, including the md5 checksum and APTrust's custom
// metadata. These options must accompany the file copy.
func (replicator *Replicator) GetCopyOptions(file *bagman.File) (s3.Options, error) {
	// Copy all of the meta data
	resp, err := replicator.ProcUtil.S3Client.Head(
		replicator.ProcUtil.Config.PreservationBucket,
		file.Uuid)
	if err != nil {
		detailedErr :=  fmt.Errorf("Head request for %s at %s returned error: %v",
			file.Uuid,
			replicator.ProcUtil.Config.PreservationBucket,
			err)
		return s3.Options{}, detailedErr
	}

	s3Metadata := make(map[string][]string)
	s3Metadata["md5"]         = []string{ file.Md5 }
	s3Metadata["institution"] = []string{ resp.Header["X-Amz-Meta-Institution"][0] }
	s3Metadata["bag"]         = []string{ resp.Header["X-Amz-Meta-Bag"][0] }
	s3Metadata["bagpath"]     = []string{ file.Path }

	base64md5, err := bagman.Base64EncodeMd5(file.Md5)
	if err != nil {
		return s3.Options{}, err
	}

	options := replicator.S3ReplicationClient.MakeOptions(base64md5, s3Metadata)
	return options, nil
}

// Copies a file from the preservation bucket to a local file
// and returns the path to the local file.
func (replicator *Replicator) DownloadFromPreservation(file *bagman.File) (string, error) {
	// Make sure we have a folder to put the file in.
	replicationDir := filepath.Join(
		replicator.ProcUtil.Config.PreservationBucket,
		"replication")
	if _, err := os.Stat(replicationDir); os.IsNotExist(err) {
		err = os.MkdirAll(replicationDir, 0755)
		if err != nil {
			return "", err
		}
	}

	// Now copy the file from S3 to local path.
	localPath := filepath.Join(replicationDir, file.Uuid)
	err := replicator.ProcUtil.S3Client.FetchToFileWithoutChecksum(
		replicator.ProcUtil.Config.PreservationBucket,
		file.Uuid,
		localPath)
	if err != nil {
		detailedErr := fmt.Errorf("Cannot read file %s from %s: %v",
			file.Uuid,
			replicator.ProcUtil.Config.PreservationBucket,
			err)
		return "", detailedErr
	}

	replicator.ProcUtil.MessageLog.Info("Downloaded %s", localPath)
	return localPath, nil
}
<|endoftext|>


<commit_before>package collectors

import (
	"fmt"
	"regexp"
	"strings"

	"github.com/StackExchange/wmi"
	"github.com/bosun-monitor/scollector/metadata"
	"github.com/bosun-monitor/scollector/opentsdb"
)

func init() {
	collectors = append(collectors, &IntervalCollector{F: c_windows_processes})
}

func WatchProcesses(procs []*WatchedProc) error {
	return fmt.Errorf("process watching not implemented on Darwin")
}

// These are silly processes but exist on my machine, will need to update KMB
var processInclusions = regexp.MustCompile("chrome|powershell|scollector|SocketServer")
var serviceInclusions = regexp.MustCompile("WinRM")

func c_windows_processes() (opentsdb.MultiDataPoint, error) {
	var dst []Win32_PerfRawData_PerfProc_Process
	var q = wmi.CreateQuery(&dst, `WHERE Name <> '_Total'`)
	err := queryWmi(q, &dst)
	if err != nil {
		return nil, err
	}

	var svc_dst []Win32_Service
	var svc_q = wmi.CreateQuery(&svc_dst, `WHERE Name <> '_Total'`)
	err = queryWmi(svc_q, &svc_dst)
	if err != nil {
		return nil, err
	}

	var iis_dst []WorkerProcess
	iis_q := wmi.CreateQuery(&iis_dst, "")
	err = queryWmiNamespace(iis_q, &iis_dst, "root\\WebAdministration")
	if err != nil {
		// Don't return from this error since the name space might exist.
		iis_dst = nil
	}

	var md opentsdb.MultiDataPoint
	for _, v := range dst {
		var name string
		service_match := false
		iis_match := false
		process_match := processInclusions.MatchString(v.Name)

		id := "0"

		if process_match {
			raw_name := strings.Split(v.Name, "#")
			name = raw_name[0]
			if len(raw_name) == 2 {
				id = raw_name[1]
			}
			// If you have a hash sign in your process name you don't deserve monitoring ;-)
			if len(raw_name) > 2 {
				continue
			}
		}

		// A Service match could "overwrite" a process match, but that is probably what we would want
		for _, svc := range svc_dst {
			if serviceInclusions.MatchString(svc.Name) {
				// It is possible the pid has gone and been reused, but I think this unlikely
				// And I'm not aware of an atomic join we could do anyways
				if svc.ProcessId == v.IDProcess {
					id = "0"
					service_match = true
					name = svc.Name
					break
				}
			}
		}

		for _, a_pool := range iis_dst {
			if a_pool.ProcessId == v.IDProcess {
				id = "0"
				iis_match = true
				name = strings.Join([]string{"iis", a_pool.AppPoolName}, "_")
				break
			}
		}

		if !(service_match || process_match || iis_match) {
			continue
		}

		//Use timestamp from WMI to fix issues with CPU metrics
		ts := TSys100NStoEpoch(v.Timestamp_Sys100NS)
		AddTS(&md, "win.proc.cpu", ts, v.PercentPrivilegedTime/NS100_Seconds, opentsdb.TagSet{"name": name, "id": id, "type": "privileged"}, metadata.Counter, metadata.Pct, descWinProcCpu_priv)
		AddTS(&md, "win.proc.cpu", ts, v.PercentUserTime/NS100_Seconds, opentsdb.TagSet{"name": name, "id": id, "type": "user"}, metadata.Counter, metadata.Pct, descWinProcCpu_user)
		AddTS(&md, "win.proc.cpu_total", ts, v.PercentProcessorTime/NS100_Seconds, opentsdb.TagSet{"name": name, "id": id}, metadata.Counter, metadata.Pct, descWinProcCpu_total)
		Add(&md, "win.proc.elapsed_time", (v.Timestamp_Object-v.ElapsedTime)/v.Frequency_Object, opentsdb.TagSet{"name": name, "id": id}, metadata.Gauge, metadata.Second, descWinProcElapsed_time)
		Add(&md, "win.proc.handle_count", v.HandleCount, opentsdb.TagSet{"name": name, "id": id}, metadata.Gauge, metadata.Count, descWinProcHandle_count)
		Add(&md, "win.proc.io_bytes", v.IOOtherBytesPersec, opentsdb.TagSet{"name": name, "id": id, "type": "other"}, metadata.Counter, metadata.BytesPerSecond, descWinProcIo_bytes_other)
		Add(&md, "win.proc.io_bytes", v.IOReadBytesPersec, opentsdb.TagSet{"name": name, "id": id, "type": "read"}, metadata.Counter, metadata.BytesPerSecond, descWinProcIo_bytes_read)
		Add(&md, "win.proc.io_bytes", v.IOWriteBytesPersec, opentsdb.TagSet{"name": name, "id": id, "type": "write"}, metadata.Counter, metadata.BytesPerSecond, descWinProcIo_bytes_write)
		Add(&md, "win.proc.io_operations", v.IOOtherOperationsPersec, opentsdb.TagSet{"name": name, "id": id, "type": "other"}, metadata.Counter, metadata.Operation, descWinProcIo_operations)
		Add(&md, "win.proc.io_operations", v.IOReadOperationsPersec, opentsdb.TagSet{"name": name, "id": id, "type": "read"}, metadata.Counter, metadata.Operation, descWinProcIo_operations)
		Add(&md, "win.proc.io_operations", v.IOWriteOperationsPersec, opentsdb.TagSet{"name": name, "id": id, "type": "write"}, metadata.Counter, metadata.Operation, descWinProcIo_operations)
		Add(&md, "win.proc.mem.page_faults", v.PageFaultsPersec, opentsdb.TagSet{"name": name, "id": id}, metadata.Counter, metadata.PerSecond, descWinProcMemPage_faults)
		Add(&md, "win.proc.mem.pagefile_bytes", v.PageFileBytes, opentsdb.TagSet{"name": name, "id": id}, metadata.Gauge, metadata.Bytes, descWinProcMemPagefile_bytes)
		Add(&md, "win.proc.mem.pagefile_bytes_peak", v.PageFileBytesPeak, opentsdb.TagSet{"name": name, "id": id}, metadata.Gauge, metadata.Bytes, descWinProcMemPagefile_bytes_peak)
		Add(&md, "win.proc.mem.pool_nonpaged_bytes", v.PoolNonpagedBytes, opentsdb.TagSet{"name": name, "id": id}, metadata.Gauge, metadata.Bytes, descWinProcMemPool_nonpaged_bytes)
		Add(&md, "win.proc.mem.pool_paged_bytes", v.PoolPagedBytes, opentsdb.TagSet{"name": name, "id": id}, metadata.Gauge, metadata.Bytes, descWinProcMemPool_paged_bytes)
		Add(&md, "win.proc.mem.vm.bytes", v.VirtualBytes, opentsdb.TagSet{"name": name, "id": id}, metadata.Gauge, metadata.Bytes, descWinProcMemVmBytes)
		Add(&md, "win.proc.mem.vm.bytes_peak", v.VirtualBytesPeak, opentsdb.TagSet{"name": name, "id": id}, metadata.Gauge, metadata.Bytes, descWinProcMemVmBytes_peak)
		Add(&md, "win.proc.mem.working_set", v.WorkingSet, opentsdb.TagSet{"name": name, "id": id}, metadata.Gauge, metadata.Bytes, descWinProcMemWorking_set)
		Add(&md, "win.proc.mem.working_set_peak", v.WorkingSetPeak, opentsdb.TagSet{"name": name, "id": id}, metadata.Gauge, metadata.Bytes, descWinProcMemWorking_set_peak)
		Add(&md, "win.proc.mem.working_set_private", v.WorkingSetPrivate, opentsdb.TagSet{"name": name, "id": id}, metadata.Gauge, metadata.Bytes, descWinProcMemWorking_set_private)
		Add(&md, "win.proc.priority_base", v.PriorityBase, opentsdb.TagSet{"name": name, "id": id}, metadata.Gauge, metadata.None, descWinProcPriority_base)
		Add(&md, "win.proc.private_bytes", v.PrivateBytes, opentsdb.TagSet{"name": name, "id": id}, metadata.Gauge, metadata.Bytes, descWinProcPrivate_bytes)
		Add(&md, "win.proc.thread_count", v.ThreadCount, opentsdb.TagSet{"name": name, "id": id}, metadata.Gauge, metadata.Count, descWinProcthread_count)
	}
	return md, nil
}

// Divide CPU by 1e5 because: 1 seconds / 100 Nanoseconds = 1e7. This is the
// percent time as a decimal, so divide by two less zeros to make it the same as
// the result * 100.
const NS100_Seconds = 1e5

const (
	descWinProcCpu_priv               = "Percentage of elapsed time that this thread has spent executing code in privileged mode."
	descWinProcCpu_total              = "Percentage of elapsed time that this process's threads have spent executing code in user or privileged mode."
	descWinProcCpu_user               = "Percentage of elapsed time that this process's threads have spent executing code in user mode."
	descWinProcElapsed_time           = "Elapsed time in seconds this process has been running."
	descWinProcHandle_count           = "Total number of handles the process has open across all threads."
	descWinProcIo_bytes_other         = "Rate at which the process is issuing bytes to I/O operations that do not involve data such as control operations."
	descWinProcIo_bytes_read          = "Rate at which the process is reading bytes from I/O operations."
	descWinProcIo_bytes_write         = "Rate at which the process is writing bytes to I/O operations."
	descWinProcIo_operations          = "Rate at which the process is issuing I/O operations that are neither a read or a write request."
	descWinProcIo_operations_read     = "Rate at which the process is issuing read I/O operations."
	descWinProcIo_operations_write    = "Rate at which the process is issuing write I/O operations."
	descWinProcMemPage_faults         = "Rate of page faults by the threads executing in this process."
	descWinProcMemPagefile_bytes      = "Current number of bytes this process has used in the paging file(s)."
	descWinProcMemPagefile_bytes_peak = "Maximum number of bytes this process has used in the paging file(s)."
	descWinProcMemPool_nonpaged_bytes = "Total number of bytes for objects that cannot be written to disk when they are not being used."
	descWinProcMemPool_paged_bytes    = "Total number of bytes for objects that can be written to disk when they are not being used."
	descWinProcMemVmBytes             = "Current size, in bytes, of the virtual address space that the process is using."
	descWinProcMemVmBytes_peak        = "Maximum number of bytes of virtual address space that the process has used at any one time."
	descWinProcMemWorking_set         = "Current number of bytes in the working set of this process at any point in time."
	descWinProcMemWorking_set_peak    = "Maximum number of bytes in the working set of this process at any point in time."
	descWinProcMemWorking_set_private = "Current number of bytes in the working set that are not shared with other processes."
	descWinProcPriority_base          = "Current base priority of this process. Threads within a process can raise and lower their own base priority relative to the process base priority of the process."
	descWinProcPrivate_bytes          = "Current number of bytes this process has allocated that cannot be shared with other processes."
	descWinProcthread_count           = "Number of threads currently active in this process."
)

// Actually a CIM_StatisticalInformation.
type Win32_PerfRawData_PerfProc_Process struct {
	ElapsedTime             uint64
	Frequency_Object        uint64
	HandleCount             uint32
	IDProcess               uint32
	IOOtherBytesPersec      uint64
	IOOtherOperationsPersec uint64
	IOReadBytesPersec       uint64
	IOReadOperationsPersec  uint64
	IOWriteBytesPersec      uint64
	IOWriteOperationsPersec uint64
	Name                    string
	PageFaultsPersec        uint32
	PageFileBytes           uint64
	PageFileBytesPeak       uint64
	PercentPrivilegedTime   uint64
	PercentProcessorTime    uint64
	PercentUserTime         uint64
	PoolNonpagedBytes       uint32
	PoolPagedBytes          uint32
	PriorityBase            uint32
	PrivateBytes            uint64
	ThreadCount             uint32
	Timestamp_Object        uint64
	Timestamp_Sys100NS      uint64
	VirtualBytes            uint64
	VirtualBytesPeak        uint64
	WorkingSet              uint64
	WorkingSetPeak          uint64
	WorkingSetPrivate       uint64
}

// Actually a Win32_BaseServce.
type Win32_Service struct {
	Name      string
	ProcessId uint32
}

type WorkerProcess struct {
	AppPoolName string
	ProcessId   uint32
}
<commit_msg>cmd/scollector: Convert win.proc.cpu* to 0-100% range instead of being per core.<commit_after>package collectors

import (
	"fmt"
	"regexp"
	"strings"

	"github.com/StackExchange/wmi"
	"github.com/bosun-monitor/scollector/metadata"
	"github.com/bosun-monitor/scollector/opentsdb"
)

func init() {
	collectors = append(collectors, &IntervalCollector{F: c_windows_processes})
}

func WatchProcesses(procs []*WatchedProc) error {
	return fmt.Errorf("process watching not implemented on Darwin")
}

// These are silly processes but exist on my machine, will need to update KMB
var processInclusions = regexp.MustCompile("chrome|powershell|scollector|SocketServer")
var serviceInclusions = regexp.MustCompile("WinRM")

func c_windows_processes() (opentsdb.MultiDataPoint, error) {
	var dst []Win32_PerfRawData_PerfProc_Process
	var q = wmi.CreateQuery(&dst, `WHERE Name <> '_Total'`)
	err := queryWmi(q, &dst)
	if err != nil {
		return nil, err
	}

	var svc_dst []Win32_Service
	var svc_q = wmi.CreateQuery(&svc_dst, `WHERE Name <> '_Total'`)
	err = queryWmi(svc_q, &svc_dst)
	if err != nil {
		return nil, err
	}

	var iis_dst []WorkerProcess
	iis_q := wmi.CreateQuery(&iis_dst, "")
	err = queryWmiNamespace(iis_q, &iis_dst, "root\\WebAdministration")
	if err != nil {
		// Don't return from this error since the name space might exist.
		iis_dst = nil
	}

	var numberOfLogicalProcessors uint64
	var core_dst []Win32_Processor
	var core_q = wmi.CreateQuery(&core_dst, `WHERE Name <> '_Total'`)
	err = queryWmi(core_q, &core_dst)
	if err != nil {
		return nil, err
	}
	for _, y := range core_dst {
		numberOfLogicalProcessors = uint64(y.NumberOfLogicalProcessors)
	}

	var md opentsdb.MultiDataPoint
	for _, v := range dst {
		var name string
		service_match := false
		iis_match := false
		process_match := processInclusions.MatchString(v.Name)

		id := "0"

		if process_match {
			raw_name := strings.Split(v.Name, "#")
			name = raw_name[0]
			if len(raw_name) == 2 {
				id = raw_name[1]
			}
			// If you have a hash sign in your process name you don't deserve monitoring ;-)
			if len(raw_name) > 2 {
				continue
			}
		}

		// A Service match could "overwrite" a process match, but that is probably what we would want
		for _, svc := range svc_dst {
			if serviceInclusions.MatchString(svc.Name) {
				// It is possible the pid has gone and been reused, but I think this unlikely
				// And I'm not aware of an atomic join we could do anyways
				if svc.ProcessId == v.IDProcess {
					id = "0"
					service_match = true
					name = svc.Name
					break
				}
			}
		}

		for _, a_pool := range iis_dst {
			if a_pool.ProcessId == v.IDProcess {
				id = "0"
				iis_match = true
				name = strings.Join([]string{"iis", a_pool.AppPoolName}, "_")
				break
			}
		}

		if !(service_match || process_match || iis_match) {
			continue
		}

		//Use timestamp from WMI to fix issues with CPU metrics
		ts := TSys100NStoEpoch(v.Timestamp_Sys100NS)
		AddTS(&md, "win.proc.cpu", ts, v.PercentPrivilegedTime/NS100_Seconds/numberOfLogicalProcessors, opentsdb.TagSet{"name": name, "id": id, "type": "privileged"}, metadata.Counter, metadata.Pct, descWinProcCpu_priv)
		AddTS(&md, "win.proc.cpu", ts, v.PercentUserTime/NS100_Seconds/numberOfLogicalProcessors, opentsdb.TagSet{"name": name, "id": id, "type": "user"}, metadata.Counter, metadata.Pct, descWinProcCpu_user)
		AddTS(&md, "win.proc.cpu_total", ts, v.PercentProcessorTime/NS100_Seconds/numberOfLogicalProcessors, opentsdb.TagSet{"name": name, "id": id}, metadata.Counter, metadata.Pct, descWinProcCpu_total)
		Add(&md, "win.proc.elapsed_time", (v.Timestamp_Object-v.ElapsedTime)/v.Frequency_Object, opentsdb.TagSet{"name": name, "id": id}, metadata.Gauge, metadata.Second, descWinProcElapsed_time)
		Add(&md, "win.proc.handle_count", v.HandleCount, opentsdb.TagSet{"name": name, "id": id}, metadata.Gauge, metadata.Count, descWinProcHandle_count)
		Add(&md, "win.proc.io_bytes", v.IOOtherBytesPersec, opentsdb.TagSet{"name": name, "id": id, "type": "other"}, metadata.Counter, metadata.BytesPerSecond, descWinProcIo_bytes_other)
		Add(&md, "win.proc.io_bytes", v.IOReadBytesPersec, opentsdb.TagSet{"name": name, "id": id, "type": "read"}, metadata.Counter, metadata.BytesPerSecond, descWinProcIo_bytes_read)
		Add(&md, "win.proc.io_bytes", v.IOWriteBytesPersec, opentsdb.TagSet{"name": name, "id": id, "type": "write"}, metadata.Counter, metadata.BytesPerSecond, descWinProcIo_bytes_write)
		Add(&md, "win.proc.io_operations", v.IOOtherOperationsPersec, opentsdb.TagSet{"name": name, "id": id, "type": "other"}, metadata.Counter, metadata.Operation, descWinProcIo_operations)
		Add(&md, "win.proc.io_operations", v.IOReadOperationsPersec, opentsdb.TagSet{"name": name, "id": id, "type": "read"}, metadata.Counter, metadata.Operation, descWinProcIo_operations)
		Add(&md, "win.proc.io_operations", v.IOWriteOperationsPersec, opentsdb.TagSet{"name": name, "id": id, "type": "write"}, metadata.Counter, metadata.Operation, descWinProcIo_operations)
		Add(&md, "win.proc.mem.page_faults", v.PageFaultsPersec, opentsdb.TagSet{"name": name, "id": id}, metadata.Counter, metadata.PerSecond, descWinProcMemPage_faults)
		Add(&md, "win.proc.mem.pagefile_bytes", v.PageFileBytes, opentsdb.TagSet{"name": name, "id": id}, metadata.Gauge, metadata.Bytes, descWinProcMemPagefile_bytes)
		Add(&md, "win.proc.mem.pagefile_bytes_peak", v.PageFileBytesPeak, opentsdb.TagSet{"name": name, "id": id}, metadata.Gauge, metadata.Bytes, descWinProcMemPagefile_bytes_peak)
		Add(&md, "win.proc.mem.pool_nonpaged_bytes", v.PoolNonpagedBytes, opentsdb.TagSet{"name": name, "id": id}, metadata.Gauge, metadata.Bytes, descWinProcMemPool_nonpaged_bytes)
		Add(&md, "win.proc.mem.pool_paged_bytes", v.PoolPagedBytes, opentsdb.TagSet{"name": name, "id": id}, metadata.Gauge, metadata.Bytes, descWinProcMemPool_paged_bytes)
		Add(&md, "win.proc.mem.vm.bytes", v.VirtualBytes, opentsdb.TagSet{"name": name, "id": id}, metadata.Gauge, metadata.Bytes, descWinProcMemVmBytes)
		Add(&md, "win.proc.mem.vm.bytes_peak", v.VirtualBytesPeak, opentsdb.TagSet{"name": name, "id": id}, metadata.Gauge, metadata.Bytes, descWinProcMemVmBytes_peak)
		Add(&md, "win.proc.mem.working_set", v.WorkingSet, opentsdb.TagSet{"name": name, "id": id}, metadata.Gauge, metadata.Bytes, descWinProcMemWorking_set)
		Add(&md, "win.proc.mem.working_set_peak", v.WorkingSetPeak, opentsdb.TagSet{"name": name, "id": id}, metadata.Gauge, metadata.Bytes, descWinProcMemWorking_set_peak)
		Add(&md, "win.proc.mem.working_set_private", v.WorkingSetPrivate, opentsdb.TagSet{"name": name, "id": id}, metadata.Gauge, metadata.Bytes, descWinProcMemWorking_set_private)
		Add(&md, "win.proc.priority_base", v.PriorityBase, opentsdb.TagSet{"name": name, "id": id}, metadata.Gauge, metadata.None, descWinProcPriority_base)
		Add(&md, "win.proc.private_bytes", v.PrivateBytes, opentsdb.TagSet{"name": name, "id": id}, metadata.Gauge, metadata.Bytes, descWinProcPrivate_bytes)
		Add(&md, "win.proc.thread_count", v.ThreadCount, opentsdb.TagSet{"name": name, "id": id}, metadata.Gauge, metadata.Count, descWinProcthread_count)
	}
	return md, nil
}

// Divide CPU by 1e5 because: 1 seconds / 100 Nanoseconds = 1e7. This is the
// percent time as a decimal, so divide by two less zeros to make it the same as
// the result * 100.
const NS100_Seconds = 1e5

const (
	descWinProcCpu_priv               = "Percentage of elapsed time that this thread has spent executing code in privileged mode."
	descWinProcCpu_total              = "Percentage of elapsed time that this process's threads have spent executing code in user or privileged mode."
	descWinProcCpu_user               = "Percentage of elapsed time that this process's threads have spent executing code in user mode."
	descWinProcElapsed_time           = "Elapsed time in seconds this process has been running."
	descWinProcHandle_count           = "Total number of handles the process has open across all threads."
	descWinProcIo_bytes_other         = "Rate at which the process is issuing bytes to I/O operations that do not involve data such as control operations."
	descWinProcIo_bytes_read          = "Rate at which the process is reading bytes from I/O operations."
	descWinProcIo_bytes_write         = "Rate at which the process is writing bytes to I/O operations."
	descWinProcIo_operations          = "Rate at which the process is issuing I/O operations that are neither a read or a write request."
	descWinProcIo_operations_read     = "Rate at which the process is issuing read I/O operations."
	descWinProcIo_operations_write    = "Rate at which the process is issuing write I/O operations."
	descWinProcMemPage_faults         = "Rate of page faults by the threads executing in this process."
	descWinProcMemPagefile_bytes      = "Current number of bytes this process has used in the paging file(s)."
	descWinProcMemPagefile_bytes_peak = "Maximum number of bytes this process has used in the paging file(s)."
	descWinProcMemPool_nonpaged_bytes = "Total number of bytes for objects that cannot be written to disk when they are not being used."
	descWinProcMemPool_paged_bytes    = "Total number of bytes for objects that can be written to disk when they are not being used."
	descWinProcMemVmBytes             = "Current size, in bytes, of the virtual address space that the process is using."
	descWinProcMemVmBytes_peak        = "Maximum number of bytes of virtual address space that the process has used at any one time."
	descWinProcMemWorking_set         = "Current number of bytes in the working set of this process at any point in time."
	descWinProcMemWorking_set_peak    = "Maximum number of bytes in the working set of this process at any point in time."
	descWinProcMemWorking_set_private = "Current number of bytes in the working set that are not shared with other processes."
	descWinProcPriority_base          = "Current base priority of this process. Threads within a process can raise and lower their own base priority relative to the process base priority of the process."
	descWinProcPrivate_bytes          = "Current number of bytes this process has allocated that cannot be shared with other processes."
	descWinProcthread_count           = "Number of threads currently active in this process."
)

// Actually a CIM_StatisticalInformation.
type Win32_PerfRawData_PerfProc_Process struct {
	ElapsedTime             uint64
	Frequency_Object        uint64
	HandleCount             uint32
	IDProcess               uint32
	IOOtherBytesPersec      uint64
	IOOtherOperationsPersec uint64
	IOReadBytesPersec       uint64
	IOReadOperationsPersec  uint64
	IOWriteBytesPersec      uint64
	IOWriteOperationsPersec uint64
	Name                    string
	PageFaultsPersec        uint32
	PageFileBytes           uint64
	PageFileBytesPeak       uint64
	PercentPrivilegedTime   uint64
	PercentProcessorTime    uint64
	PercentUserTime         uint64
	PoolNonpagedBytes       uint32
	PoolPagedBytes          uint32
	PriorityBase            uint32
	PrivateBytes            uint64
	ThreadCount             uint32
	Timestamp_Object        uint64
	Timestamp_Sys100NS      uint64
	VirtualBytes            uint64
	VirtualBytesPeak        uint64
	WorkingSet              uint64
	WorkingSetPeak          uint64
	WorkingSetPrivate       uint64
}

// Actually a Win32_BaseServce.
type Win32_Service struct {
	Name      string
	ProcessId uint32
}

type WorkerProcess struct {
	AppPoolName string
	ProcessId   uint32
}
<|endoftext|>


<commit_before>package termsize

import "fmt"

func GetTerminalColumns() int {
	return 80
}

func Println(s ...interface{}) {
	fmt.Println(s...)
}
<commit_msg>add windows withd<commit_after>package termsize

import "fmt"

func GetTerminalColumns() int {
	return 80
}

func Println(s ...interface{}) {
	fmt.Println(s...)
}

func Width() int {
	return 80
}

func Height() int {
	return 40
}
<|endoftext|>


<commit_before>package agent_cli

import (
	"testing"
	"regexp"
	"net/rpc"
	"github.com/ChaosXu/nerv/test/util"
)

func TestNervCmd(t *testing.T) {

	//create
	cmd := &util.Cmd{
		Dir: "../../release/nerv/nerv-cli/bin",
		Cli:"./nerv-cli",
		Items:[]string{"nerv", "create", "-t", "../../resources/templates/nerv/server_core.json", "-o", "nerv-test", "-n", "../../../../test/agent-cli/nerv_standalone_inputs.json"},
	}

	var id string
	if out, err := cmd.Run(t); err != nil {
		t.Log(string(out))
		t.Errorf("%s", err.Error())
	} else {
		res := string(out)
		t.Log(res)
		regex := regexp.MustCompile(`.*id=(.+)`)
		id = regex.FindStringSubmatch(res)[1]
	}

	//install
	cmd = &util.Cmd{
		Dir: "../../release/nerv/nerv-cli/bin",
		Cli:"./nerv-cli",
		Items:[]string{"nerv", "install", "-i", id},
	}

	if out, err := cmd.Run(t); err != nil {
		t.Log(string(out))
		t.Errorf("%s", err.Error())
	} else {
		res := string(out)
		t.Log(res)
	}

	//setup
	cmd = &util.Cmd{
		Dir: "../../release/nerv/nerv-cli/bin",
		Cli:"./nerv-cli",
		Items:[]string{"nerv", "setup", "-i", id},
	}

	if out, err := cmd.Run(t); err != nil {
		t.Log(string(out))
		t.Errorf("%s", err.Error())
	} else {
		t.Log(string(out))
	}

	//start
	cmd = &util.Cmd{
		Dir: "../../release/nerv/nerv-cli/bin",
		Cli:"./nerv-cli",
		Items:[]string{"nerv", "start", "-i", id},
	}

	if out, err := cmd.Run(t); err != nil {
		t.Log(string(out))
		t.Errorf("%s", err.Error())
	} else {
		t.Log(string(out))
	}

	testAgent(t);

	//stop
	cmd = &util.Cmd{
		Dir: "../../release/nerv/nerv-cli/bin",
		Cli:"./nerv-cli",
		Items:[]string{"nerv", "stop", "-i", id},
	}

	if out, err := cmd.Run(t); err != nil {
		t.Log(string(out))
		t.Errorf("%s", err.Error())
	} else {
		t.Log(string(out))
	}

	//uninstall
	//cmd = &util.Cmd{
	//	Dir: "../../release/nerv/nerv-cli/bin",
	//	Cli:"./nerv-cli",
	//	Items:[]string{"nerv", "uninstall", "-i", id},
	//}
	//
	//if out, err := cmd.Run(t); err != nil {
	//	t.Log(string(out))
	//	t.Errorf("%s", err.Error())
	//} else {
	//	t.Log(string(out))
	//}

	//delete
	//cmd = &util.Cmd{
	//	Dir: "../../release/nerv/nerv-cli/bin",
	//	Cli:"./nerv-cli",
	//	Items:[]string{"nerv", "delete", "-i", id},
	//}
	//
	//if out, err := cmd.Run(t); err != nil {
	//	t.Log(string(out))
	//	t.Errorf("%s", err.Error())
	//} else {
	//	t.Log(string(out))
	//}
}

func testAgent(t *testing.T) {
	testHttp(t)
	testAppCmd(t)
}

func testHttp(t *testing.T) {
	client, err := rpc.DialHTTP("tcp", "localhost:3334")
	if err != nil {
		t.Log("DialHTTP:", err.Error())
	}

	script := &util.RemoteScript{Content:"echo agnet ok", Args:map[string]string{}}
	var reply string
	err = client.Call("Agent.Execute", script, &reply)
	if err != nil {
		t.Log("Agent.Execute failed.", err)
	} else {
		t.Log(reply)
	}
}

func testAppCmd(t *testing.T) {
	runCmd(t, "../../release/nerv/agent/bin", "./agent-cli", []string{"app", "create", "-D", "../../../../test/agent-cli/app.json"})
	runCmd(t, "../../release/nerv/agent/bin", "./agent-cli", []string{"app", "list"})
}

func runCmd(t *testing.T, dir string, cli string, args []string) {
	cmd := &util.Cmd{
		Dir: dir,
		Cli:cli,
		Items:args,
	}

	if out, err := cmd.Run(t); err != nil {
		t.Log(string(out))
		t.Errorf("%s", err.Error())
	} else {
		t.Log(string(out))
	}
}

<commit_msg>agent-cli test<commit_after>package agent_cli

import (
	"testing"
	"regexp"
	"net/rpc"
	"github.com/ChaosXu/nerv/test/util"
)

func TestNervCmd(t *testing.T) {

	//create
	cmd := &util.Cmd{
		Dir: "../../release/nerv/nerv-cli/bin",
		Cli:"./nerv-cli",
		Items:[]string{"nerv", "create", "-t", "../../resources/templates/nerv/server_core.json", "-o", "nerv-test", "-n", "../../../../test/agent-cli/nerv_standalone_inputs.json"},
	}

	var id string
	if out, err := cmd.Run(t); err != nil {
		t.Log(string(out))
		t.Errorf("%s", err.Error())
	} else {
		res := string(out)
		t.Log(res)
		regex := regexp.MustCompile(`.*id=(.+)`)
		id = regex.FindStringSubmatch(res)[1]
	}

	//install
	cmd = &util.Cmd{
		Dir: "../../release/nerv/nerv-cli/bin",
		Cli:"./nerv-cli",
		Items:[]string{"nerv", "install", "-i", id},
	}

	if out, err := cmd.Run(t); err != nil {
		t.Log(string(out))
		t.Errorf("%s", err.Error())
	} else {
		res := string(out)
		t.Log(res)
	}

	//setup
	cmd = &util.Cmd{
		Dir: "../../release/nerv/nerv-cli/bin",
		Cli:"./nerv-cli",
		Items:[]string{"nerv", "setup", "-i", id},
	}

	if out, err := cmd.Run(t); err != nil {
		t.Log(string(out))
		t.Errorf("%s", err.Error())
	} else {
		t.Log(string(out))
	}

	//start
	cmd = &util.Cmd{
		Dir: "../../release/nerv/nerv-cli/bin",
		Cli:"./nerv-cli",
		Items:[]string{"nerv", "start", "-i", id},
	}

	if out, err := cmd.Run(t); err != nil {
		t.Log(string(out))
		t.Errorf("%s", err.Error())
	} else {
		t.Log(string(out))
	}

	testAgent(t);

	//stop
	cmd = &util.Cmd{
		Dir: "../../release/nerv/nerv-cli/bin",
		Cli:"./nerv-cli",
		Items:[]string{"nerv", "stop", "-i", id},
	}

	if out, err := cmd.Run(t); err != nil {
		t.Log(string(out))
		t.Errorf("%s", err.Error())
	} else {
		t.Log(string(out))
	}

	//uninstall
	//cmd = &util.Cmd{
	//	Dir: "../../release/nerv/nerv-cli/bin",
	//	Cli:"./nerv-cli",
	//	Items:[]string{"nerv", "uninstall", "-i", id},
	//}
	//
	//if out, err := cmd.Run(t); err != nil {
	//	t.Log(string(out))
	//	t.Errorf("%s", err.Error())
	//} else {
	//	t.Log(string(out))
	//}

	//delete
	//cmd = &util.Cmd{
	//	Dir: "../../release/nerv/nerv-cli/bin",
	//	Cli:"./nerv-cli",
	//	Items:[]string{"nerv", "delete", "-i", id},
	//}
	//
	//if out, err := cmd.Run(t); err != nil {
	//	t.Log(string(out))
	//	t.Errorf("%s", err.Error())
	//} else {
	//	t.Log(string(out))
	//}
}

func testAgent(t *testing.T) {
	testHttp(t)
	testAppCmd(t)
}

func testHttp(t *testing.T) {
	client, err := rpc.DialHTTP("tcp", "localhost:3334")
	if err != nil {
		t.Log("DialHTTP:", err.Error())
	}

	script := &util.RemoteScript{Content:"echo agnet ok", Args:map[string]string{}}
	var reply string
	err = client.Call("Agent.Execute", script, &reply)
	if err != nil {
		t.Log("Agent.Execute failed.", err)
	} else {
		t.Log(reply)
	}
}

func testAppCmd(t *testing.T) {
	id := runCmd(t, "../../release/nerv/agent/bin", "./agent-cli", []string{"app", "create", "-D", "../../../../test/agent-cli/app.json"})
	t.Log(id)
	runCmd(t, "../../release/nerv/agent/bin", "./agent-cli", []string{"app", "list"})
	runCmd(t, "../../release/nerv/agent/bin", "./agent-cli", []string{"app", "get", "-i", id})
	runCmd(t, "../../release/nerv/agent/bin", "./agent-cli", []string{"app", "delete", "-i", id})
	runCmd(t, "../../release/nerv/agent/bin", "./agent-cli", []string{"app", "list"})
	runCmd(t, "../../release/nerv/agent/bin", "./agent-cli", []string{"app", "get", "-i", id})
}

func runCmd(t *testing.T, dir string, cli string, args []string) string {
	r := "0"
	cmd := &util.Cmd{
		Dir: dir,
		Cli:cli,
		Items:args,
	}

	if out, err := cmd.Run(t); err != nil {
		t.Log(string(out))
		t.Errorf("%s", err.Error())
	} else {

		res := string(out)
		regex := regexp.MustCompile(`.*([0-9]+),.*`)
		match := regex.FindStringSubmatch(res)
		if len(match) > 0 {
			r = match[1]
			t.Log(r)
		}
		t.Log(res)
	}
	return r
}

<|endoftext|>


<commit_before>// Copyright 2016 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

// These tests all do one conflict-free operation while a user is unstaged.

package test

import "testing"

// bob writes a multi-block file while unmerged, no conflicts
func TestCrUnmergedWriteMultiblockFile(t *testing.T) {
	test(t,
		blockSize(20), users("alice", "bob"),
		as(alice,
			mkdir("a"),
		),
		as(bob,
			disableUpdates(),
		),
		as(alice,
			write("a/foo", "hello"),
		),
		as(bob,
			write("a/b", ntimesString(15, "0123456789")),
			reenableUpdates(),
			lsdir("a/", m{"b": "FILE", "foo": "FILE"}),
			read("a/b", ntimesString(15, "0123456789")),
			read("a/foo", "hello"),
		),
		as(alice,
			lsdir("a/", m{"b": "FILE", "foo": "FILE"}),
			read("a/b", ntimesString(15, "0123456789")),
			read("a/foo", "hello"),
		),
	)
}

// bob writes a multi-block file that conflicts with a file created by alice
func TestCrConflictUnmergedWriteMultiblockFile(t *testing.T) {
	test(t,
		blockSize(20), users("alice", "bob"),
		as(alice,
			mkdir("a"),
		),
		as(bob,
			disableUpdates(),
		),
		as(alice,
			write("a/b", "hello"),
		),
		as(bob, noSync(),
			write("a/b", ntimesString(15, "0123456789")),
			reenableUpdates(),
			lsdir("a/", m{"b$": "FILE", crnameEsc("b", bob): "FILE"}),
			read("a/b", "hello"),
			read(crname("a/b", bob), ntimesString(15, "0123456789")),
		),
		as(alice,
			lsdir("a/", m{"b$": "FILE", crnameEsc("b", bob): "FILE"}),
			read("a/b", "hello"),
			read(crname("a/b", bob), ntimesString(15, "0123456789")),
		),
	)
}

// alice writes a multi-block file that conflicts with a directory
// created by alice
func TestCrConflictMergedWriteMultiblockFile(t *testing.T) {
	test(t,
		blockSize(20), users("alice", "bob"),
		as(alice,
			mkdir("a"),
		),
		as(bob,
			disableUpdates(),
		),
		as(alice,
			write("a/b", ntimesString(15, "0123456789")),
		),
		as(bob, noSync(),
			write("a/b/c", "hello"),
			reenableUpdates(),
			lsdir("a/", m{"b$": "DIR", crnameEsc("b", alice): "FILE"}),
			read("a/b/c", "hello"),
			read(crname("a/b", alice), ntimesString(15, "0123456789")),
		),
		as(alice,
			lsdir("a/", m{"b$": "DIR", crnameEsc("b", alice): "FILE"}),
			read("a/b/c", "hello"),
			read(crname("a/b", alice), ntimesString(15, "0123456789")),
		),
	)
}

// bob resurrects a file that was removed by alice
func TestCrConflictWriteToRemovedMultiblockFile(t *testing.T) {
	test(t,
		blockSize(20), users("alice", "bob"),
		as(alice,
			mkdir("a"),
			write("a/b", ntimesString(15, "0123456789")),
		),
		as(bob,
			disableUpdates(),
		),
		as(alice,
			rm("a/b"),
		),
		as(bob, noSync(),
			write("a/b", ntimesString(15, "9876543210")),
			reenableUpdates(),
			lsdir("a/", m{"b$": "FILE"}),
			read("a/b", ntimesString(15, "9876543210")),
		),
		as(alice,
			lsdir("a/", m{"b$": "FILE"}),
			read("a/b", ntimesString(15, "9876543210")),
		),
	)
}

// bob makes a file that was removed by alice executable
func TestCrConflictSetexToRemovedMultiblockFile(t *testing.T) {
	test(t,
		blockSize(20), users("alice", "bob"),
		as(alice,
			mkdir("a"),
			write("a/b", ntimesString(15, "0123456789")),
		),
		as(bob,
			disableUpdates(),
		),
		as(alice,
			rm("a/b"),
		),
		as(bob, noSync(),
			setex("a/b", true),
			reenableUpdates(),
			lsdir("a/", m{"b$": "EXEC"}),
			read("a/b", ntimesString(15, "0123456789")),
		),
		as(alice,
			lsdir("a/", m{"b$": "EXEC"}),
			read("a/b", ntimesString(15, "0123456789")),
		),
	)
}

// bob moves a file that was removed by alice
func TestCrConflictMoveRemovedMultiblockFile(t *testing.T) {
	test(t,
		blockSize(20), users("alice", "bob"),
		as(alice,
			mkdir("a"),
			write("a/b", ntimesString(15, "0123456789")),
		),
		as(bob,
			disableUpdates(),
		),
		as(alice,
			rm("a/b"),
		),
		as(bob, noSync(),
			rename("a/b", "a/c"),
			reenableUpdates(),
			lsdir("a/", m{"c$": "FILE"}),
			read("a/c", ntimesString(15, "0123456789")),
		),
		as(alice,
			lsdir("a/", m{"c$": "FILE"}),
			read("a/c", ntimesString(15, "0123456789")),
		),
	)
}

// TODO: test md.RefBytes, md.UnrefBytes, and md.DiskUsage as well!
<commit_msg>test: new CR test with a small block change size<commit_after>// Copyright 2016 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

// These tests all do one conflict-free operation while a user is unstaged.

package test

import "testing"

// bob writes a multi-block file while unmerged, no conflicts
func TestCrUnmergedWriteMultiblockFile(t *testing.T) {
	test(t,
		blockSize(20), users("alice", "bob"),
		as(alice,
			mkdir("a"),
		),
		as(bob,
			disableUpdates(),
		),
		as(alice,
			write("a/foo", "hello"),
		),
		as(bob, noSync(),
			write("a/b", ntimesString(15, "0123456789")),
			reenableUpdates(),
			lsdir("a/", m{"b": "FILE", "foo": "FILE"}),
			read("a/b", ntimesString(15, "0123456789")),
			read("a/foo", "hello"),
		),
		as(alice,
			lsdir("a/", m{"b": "FILE", "foo": "FILE"}),
			read("a/b", ntimesString(15, "0123456789")),
			read("a/foo", "hello"),
		),
	)
}

// bob writes a multi-block file that conflicts with a file created by alice
func TestCrConflictUnmergedWriteMultiblockFile(t *testing.T) {
	test(t,
		blockSize(20), users("alice", "bob"),
		as(alice,
			mkdir("a"),
		),
		as(bob,
			disableUpdates(),
		),
		as(alice,
			write("a/b", "hello"),
		),
		as(bob, noSync(),
			write("a/b", ntimesString(15, "0123456789")),
			reenableUpdates(),
			lsdir("a/", m{"b$": "FILE", crnameEsc("b", bob): "FILE"}),
			read("a/b", "hello"),
			read(crname("a/b", bob), ntimesString(15, "0123456789")),
		),
		as(alice,
			lsdir("a/", m{"b$": "FILE", crnameEsc("b", bob): "FILE"}),
			read("a/b", "hello"),
			read(crname("a/b", bob), ntimesString(15, "0123456789")),
		),
	)
}

// alice writes a multi-block file that conflicts with a directory
// created by alice
func TestCrConflictMergedWriteMultiblockFile(t *testing.T) {
	test(t,
		blockSize(20), users("alice", "bob"),
		as(alice,
			mkdir("a"),
		),
		as(bob,
			disableUpdates(),
		),
		as(alice,
			write("a/b", ntimesString(15, "0123456789")),
		),
		as(bob, noSync(),
			write("a/b/c", "hello"),
			reenableUpdates(),
			lsdir("a/", m{"b$": "DIR", crnameEsc("b", alice): "FILE"}),
			read("a/b/c", "hello"),
			read(crname("a/b", alice), ntimesString(15, "0123456789")),
		),
		as(alice,
			lsdir("a/", m{"b$": "DIR", crnameEsc("b", alice): "FILE"}),
			read("a/b/c", "hello"),
			read(crname("a/b", alice), ntimesString(15, "0123456789")),
		),
	)
}

// bob resurrects a file that was removed by alice
func TestCrConflictWriteToRemovedMultiblockFile(t *testing.T) {
	test(t,
		blockSize(20), users("alice", "bob"),
		as(alice,
			mkdir("a"),
			write("a/b", ntimesString(15, "0123456789")),
		),
		as(bob,
			disableUpdates(),
		),
		as(alice,
			rm("a/b"),
		),
		as(bob, noSync(),
			write("a/b", ntimesString(15, "9876543210")),
			reenableUpdates(),
			lsdir("a/", m{"b$": "FILE"}),
			read("a/b", ntimesString(15, "9876543210")),
		),
		as(alice,
			lsdir("a/", m{"b$": "FILE"}),
			read("a/b", ntimesString(15, "9876543210")),
		),
	)
}

// bob makes a file that was removed by alice executable
func TestCrConflictSetexToRemovedMultiblockFile(t *testing.T) {
	test(t,
		blockSize(20), users("alice", "bob"),
		as(alice,
			mkdir("a"),
			write("a/b", ntimesString(15, "0123456789")),
		),
		as(bob,
			disableUpdates(),
		),
		as(alice,
			rm("a/b"),
		),
		as(bob, noSync(),
			setex("a/b", true),
			reenableUpdates(),
			lsdir("a/", m{"b$": "EXEC"}),
			read("a/b", ntimesString(15, "0123456789")),
		),
		as(alice,
			lsdir("a/", m{"b$": "EXEC"}),
			read("a/b", ntimesString(15, "0123456789")),
		),
	)
}

// bob moves a file that was removed by alice
func TestCrConflictMoveRemovedMultiblockFile(t *testing.T) {
	test(t,
		blockSize(20), users("alice", "bob"),
		as(alice,
			mkdir("a"),
			write("a/b", ntimesString(15, "0123456789")),
		),
		as(bob,
			disableUpdates(),
		),
		as(alice,
			rm("a/b"),
		),
		as(bob, noSync(),
			rename("a/b", "a/c"),
			reenableUpdates(),
			lsdir("a/", m{"c$": "FILE"}),
			read("a/c", ntimesString(15, "0123456789")),
		),
		as(alice,
			lsdir("a/", m{"c$": "FILE"}),
			read("a/c", ntimesString(15, "0123456789")),
		),
	)
}

// bob writes a multi-block file while unmerged and the block change
// size is small, no conflicts
func TestCrUnmergedWriteMultiblockFileWithSmallBlockChangeSize(t *testing.T) {
	test(t,
		blockSize(20), blockChangeSize(5), users("alice", "bob"),
		as(alice,
			mkdir("a"),
		),
		as(bob,
			disableUpdates(),
		),
		as(alice,
			write("a/foo", "hello"),
		),
		as(bob,
			write("a/b", ntimesString(15, "0123456789")),
			reenableUpdates(),
			lsdir("a/", m{"b": "FILE", "foo": "FILE"}),
			read("a/b", ntimesString(15, "0123456789")),
			read("a/foo", "hello"),
		),
		as(alice,
			lsdir("a/", m{"b": "FILE", "foo": "FILE"}),
			read("a/b", ntimesString(15, "0123456789")),
			read("a/foo", "hello"),
		),
	)
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package common

import (
	"fmt"

	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/kubernetes/test/e2e/framework"
	e2elog "k8s.io/kubernetes/test/e2e/framework/log"
	imageutils "k8s.io/kubernetes/test/utils/image"

	"github.com/onsi/ginkgo"
	"github.com/onsi/gomega"
)

var _ = ginkgo.Describe("[sig-node] ConfigMap", func() {
	f := framework.NewDefaultFramework("configmap")

	/*
		Release : v1.9
		Testname: ConfigMap, from environment field
		Description: Create a Pod with an environment variable value set using a value from ConfigMap. A ConfigMap value MUST be accessible in the container environment.
	*/
	framework.ConformanceIt("should be consumable via environment variable [NodeConformance]", func() {
		name := "configmap-test-" + string(uuid.NewUUID())
		configMap := newConfigMap(f, name)
		ginkgo.By(fmt.Sprintf("Creating configMap %v/%v", f.Namespace.Name, configMap.Name))
		var err error
		if configMap, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(configMap); err != nil {
			e2elog.Failf("unable to create test configMap %s: %v", configMap.Name, err)
		}

		pod := &v1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name: "pod-configmaps-" + string(uuid.NewUUID()),
			},
			Spec: v1.PodSpec{
				Containers: []v1.Container{
					{
						Name:    "env-test",
						Image:   imageutils.GetE2EImage(imageutils.BusyBox),
						Command: []string{"sh", "-c", "env"},
						Env: []v1.EnvVar{
							{
								Name: "CONFIG_DATA_1",
								ValueFrom: &v1.EnvVarSource{
									ConfigMapKeyRef: &v1.ConfigMapKeySelector{
										LocalObjectReference: v1.LocalObjectReference{
											Name: name,
										},
										Key: "data-1",
									},
								},
							},
						},
					},
				},
				RestartPolicy: v1.RestartPolicyNever,
			},
		}

		f.TestContainerOutput("consume configMaps", pod, 0, []string{
			"CONFIG_DATA_1=value-1",
		})
	})

	/*
		Release: v1.9
		Testname: ConfigMap, from environment variables
		Description: Create a Pod with a environment source from ConfigMap. All ConfigMap values MUST be available as environment variables in the container.
	*/
	framework.ConformanceIt("should be consumable via the environment [NodeConformance]", func() {
		name := "configmap-test-" + string(uuid.NewUUID())
		configMap := newEnvFromConfigMap(f, name)
		ginkgo.By(fmt.Sprintf("Creating configMap %v/%v", f.Namespace.Name, configMap.Name))
		var err error
		if configMap, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(configMap); err != nil {
			e2elog.Failf("unable to create test configMap %s: %v", configMap.Name, err)
		}

		pod := &v1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name: "pod-configmaps-" + string(uuid.NewUUID()),
			},
			Spec: v1.PodSpec{
				Containers: []v1.Container{
					{
						Name:    "env-test",
						Image:   imageutils.GetE2EImage(imageutils.BusyBox),
						Command: []string{"sh", "-c", "env"},
						EnvFrom: []v1.EnvFromSource{
							{
								ConfigMapRef: &v1.ConfigMapEnvSource{LocalObjectReference: v1.LocalObjectReference{Name: name}},
							},
							{
								Prefix:       "p_",
								ConfigMapRef: &v1.ConfigMapEnvSource{LocalObjectReference: v1.LocalObjectReference{Name: name}},
							},
						},
					},
				},
				RestartPolicy: v1.RestartPolicyNever,
			},
		}

		f.TestContainerOutput("consume configMaps", pod, 0, []string{
			"data_1=value-1", "data_2=value-2", "data_3=value-3",
			"p_data_1=value-1", "p_data_2=value-2", "p_data_3=value-3",
		})
	})

	/*
	   Release : v1.14
	   Testname: ConfigMap, with empty-key
	   Description: Attempt to create a ConfigMap with an empty key. The creation MUST fail.
	*/
	framework.ConformanceIt("should fail to create ConfigMap with empty key", func() {
		configMap, err := newConfigMapWithEmptyKey(f)
		framework.ExpectError(err, "created configMap %q with empty key in namespace %q", configMap.Name, f.Namespace.Name)
	})

	ginkgo.It("should patch ConfigMap successfully", func() {
		name := "configmap-test-" + string(uuid.NewUUID())
		configMap := newConfigMap(f, name)
		configMapOriginalState := *configMap
		ginkgo.By(fmt.Sprintf("Creating configMap %v/%v", f.Namespace.Name, configMap.Name))
		_, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(configMap)
		framework.ExpectNoError(err)

		configMap.Data = map[string]string{
			"data": "value",
		}
		ginkgo.By(fmt.Sprintf("Updating configMap %v/%v", f.Namespace.Name, configMap.Name))
		_, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Update(configMap)
		framework.ExpectNoError(err)

		configMapFromUpdate, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Get(name, metav1.GetOptions{})
		framework.ExpectNoError(err)
		ginkgo.By(fmt.Sprintf("Verifying update of configMap %v/%v", f.Namespace.Name, configMap.Name))
		gomega.Expect(configMapFromUpdate.Data).NotTo(gomega.Equal(configMapOriginalState.Data))
	})
})

func newEnvFromConfigMap(f *framework.Framework, name string) *v1.ConfigMap {
	return &v1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: f.Namespace.Name,
			Name:      name,
		},
		Data: map[string]string{
			"data_1": "value-1",
			"data_2": "value-2",
			"data_3": "value-3",
		},
	}
}

func newConfigMapWithEmptyKey(f *framework.Framework) (*v1.ConfigMap, error) {
	name := "configmap-test-emptyKey-" + string(uuid.NewUUID())
	configMap := &v1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: f.Namespace.Name,
			Name:      name,
		},
		Data: map[string]string{
			"": "value-1",
		},
	}

	ginkgo.By(fmt.Sprintf("Creating configMap that has name %s", configMap.Name))
	return f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(configMap)
}
<commit_msg>Update function for data inequality verification<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package common

import (
	"fmt"

	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/kubernetes/test/e2e/framework"
	e2elog "k8s.io/kubernetes/test/e2e/framework/log"
	imageutils "k8s.io/kubernetes/test/utils/image"

	"github.com/onsi/ginkgo"
)

var _ = ginkgo.Describe("[sig-node] ConfigMap", func() {
	f := framework.NewDefaultFramework("configmap")

	/*
		Release : v1.9
		Testname: ConfigMap, from environment field
		Description: Create a Pod with an environment variable value set using a value from ConfigMap. A ConfigMap value MUST be accessible in the container environment.
	*/
	framework.ConformanceIt("should be consumable via environment variable [NodeConformance]", func() {
		name := "configmap-test-" + string(uuid.NewUUID())
		configMap := newConfigMap(f, name)
		ginkgo.By(fmt.Sprintf("Creating configMap %v/%v", f.Namespace.Name, configMap.Name))
		var err error
		if configMap, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(configMap); err != nil {
			e2elog.Failf("unable to create test configMap %s: %v", configMap.Name, err)
		}

		pod := &v1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name: "pod-configmaps-" + string(uuid.NewUUID()),
			},
			Spec: v1.PodSpec{
				Containers: []v1.Container{
					{
						Name:    "env-test",
						Image:   imageutils.GetE2EImage(imageutils.BusyBox),
						Command: []string{"sh", "-c", "env"},
						Env: []v1.EnvVar{
							{
								Name: "CONFIG_DATA_1",
								ValueFrom: &v1.EnvVarSource{
									ConfigMapKeyRef: &v1.ConfigMapKeySelector{
										LocalObjectReference: v1.LocalObjectReference{
											Name: name,
										},
										Key: "data-1",
									},
								},
							},
						},
					},
				},
				RestartPolicy: v1.RestartPolicyNever,
			},
		}

		f.TestContainerOutput("consume configMaps", pod, 0, []string{
			"CONFIG_DATA_1=value-1",
		})
	})

	/*
		Release: v1.9
		Testname: ConfigMap, from environment variables
		Description: Create a Pod with a environment source from ConfigMap. All ConfigMap values MUST be available as environment variables in the container.
	*/
	framework.ConformanceIt("should be consumable via the environment [NodeConformance]", func() {
		name := "configmap-test-" + string(uuid.NewUUID())
		configMap := newEnvFromConfigMap(f, name)
		ginkgo.By(fmt.Sprintf("Creating configMap %v/%v", f.Namespace.Name, configMap.Name))
		var err error
		if configMap, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(configMap); err != nil {
			e2elog.Failf("unable to create test configMap %s: %v", configMap.Name, err)
		}

		pod := &v1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name: "pod-configmaps-" + string(uuid.NewUUID()),
			},
			Spec: v1.PodSpec{
				Containers: []v1.Container{
					{
						Name:    "env-test",
						Image:   imageutils.GetE2EImage(imageutils.BusyBox),
						Command: []string{"sh", "-c", "env"},
						EnvFrom: []v1.EnvFromSource{
							{
								ConfigMapRef: &v1.ConfigMapEnvSource{LocalObjectReference: v1.LocalObjectReference{Name: name}},
							},
							{
								Prefix:       "p_",
								ConfigMapRef: &v1.ConfigMapEnvSource{LocalObjectReference: v1.LocalObjectReference{Name: name}},
							},
						},
					},
				},
				RestartPolicy: v1.RestartPolicyNever,
			},
		}

		f.TestContainerOutput("consume configMaps", pod, 0, []string{
			"data_1=value-1", "data_2=value-2", "data_3=value-3",
			"p_data_1=value-1", "p_data_2=value-2", "p_data_3=value-3",
		})
	})

	/*
	   Release : v1.14
	   Testname: ConfigMap, with empty-key
	   Description: Attempt to create a ConfigMap with an empty key. The creation MUST fail.
	*/
	framework.ConformanceIt("should fail to create ConfigMap with empty key", func() {
		configMap, err := newConfigMapWithEmptyKey(f)
		framework.ExpectError(err, "created configMap %q with empty key in namespace %q", configMap.Name, f.Namespace.Name)
	})

	ginkgo.It("should patch ConfigMap successfully", func() {
		name := "configmap-test-" + string(uuid.NewUUID())
		configMap := newConfigMap(f, name)
		configMapOriginalState := *configMap
		ginkgo.By(fmt.Sprintf("Creating configMap %v/%v", f.Namespace.Name, configMap.Name))
		_, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(configMap)
		framework.ExpectNoError(err)

		configMap.Data = map[string]string{
			"data": "value",
		}
		ginkgo.By(fmt.Sprintf("Updating configMap %v/%v", f.Namespace.Name, configMap.Name))
		_, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Update(configMap)
		framework.ExpectNoError(err)

		configMapFromUpdate, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Get(name, metav1.GetOptions{})
		framework.ExpectNoError(err)
		ginkgo.By(fmt.Sprintf("Verifying update of configMap %v/%v", f.Namespace.Name, configMap.Name))
		framework.ExpectNotEqual(configMapFromUpdate.Data, configMapOriginalState.Data)
	})
})

func newEnvFromConfigMap(f *framework.Framework, name string) *v1.ConfigMap {
	return &v1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: f.Namespace.Name,
			Name:      name,
		},
		Data: map[string]string{
			"data_1": "value-1",
			"data_2": "value-2",
			"data_3": "value-3",
		},
	}
}

func newConfigMapWithEmptyKey(f *framework.Framework) (*v1.ConfigMap, error) {
	name := "configmap-test-emptyKey-" + string(uuid.NewUUID())
	configMap := &v1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: f.Namespace.Name,
			Name:      name,
		},
		Data: map[string]string{
			"": "value-1",
		},
	}

	ginkgo.By(fmt.Sprintf("Creating configMap that has name %s", configMap.Name))
	return f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(configMap)
}
<|endoftext|>