<commit_before>package esi

import (
	"errors"
	"fmt"
	"log"
	"sync"
	"time"

	"github.com/evepraisal/go-evepraisal"
	"github.com/sethgrid/pester"
)

type MarketOrder struct {
	ID            int64   `json:"order_id"`
	Type          int64   `json:"type_id"`
	StationID     int64   `json:"location_id"`
	Volume        int64   `json:"volume_remain"`
	MinVolume     int64   `json:"min_volume"`
	Price         float64 `json:"price"`
	Buy           bool    `json:"is_buy_order"`
	Duration      int64   `json:"duration"`
	Issued        string  `json:"issued"`
	VolumeEntered int64   `json:"volumeEntered"`
	Range         string  `json:"range"`
}

var SpecialRegions = []struct {
	name     string
	stations []int64
}{
	{
		// 10000002
		name:     "jita",
		stations: []int64{60003466, 60003760, 60003757, 60000361, 60000451, 60004423, 60002959, 60003460, 60003055, 60003469, 60000364, 60002953, 60000463, 60003463},
	}, {
		// 10000043
		name:     "amarr",
		stations: []int64{60008950, 60002569, 60008494},
	}, {
		// 10000032
		name:     "dodixie",
		stations: []int64{60011866, 60001867},
	}, {
		// 10000042
		name:     "hek",
		stations: []int64{60005236, 60004516, 60015140, 60005686, 60011287, 60005236},
	},
}

type PriceFetcher struct {
	db      evepraisal.PriceDB
	client  *pester.Client
	baseURL string

	stop chan bool
	wg   *sync.WaitGroup
}

func NewPriceFetcher(priceDB evepraisal.PriceDB, baseURL string, client *pester.Client) (*PriceFetcher, error) {

	p := &PriceFetcher{
		db:      priceDB,
		client:  client,
		baseURL: baseURL,

		stop: make(chan bool),
		wg:   &sync.WaitGroup{},
	}

	p.wg.Add(1)
	go func() {
		defer p.wg.Done()
		for {
			start := time.Now()
			p.runOnce()
			select {
			case <-time.After((5 * time.Minute) - time.Since(start)):
			case <-p.stop:
				return
			}
		}
	}()

	return p, nil
}

func (p *PriceFetcher) Close() error {
	close(p.stop)
	p.wg.Wait()
	return nil
}

func regionNames() []string {
	regions := make([]string, len(SpecialRegions)+1)
	regions[0] = "universe"
	for i, region := range SpecialRegions {
		regions[i+1] = region.name
	}
	return regions
}

func (p *PriceFetcher) runOnce() {
	log.Println("Fetch market data")
	priceMap, err := p.FetchOrderData(p.client, p.baseURL, []int{10000002, 10000042, 10000027, 10000032, 10000043})
	if err != nil {
		log.Println("ERROR: fetching market data: ", err)
		return
	}

	pricesFromCCP, err := p.FetchPriceData(p.client, p.baseURL)
	if err != nil {
		log.Println("ERROR: fetching CCP price data: ", err)
		return
	}

	for _, regionName := range regionNames() {
		// Use CCP's price if our regional price is too low
		for typeID, prices := range pricesFromCCP {
			p, ok := priceMap[regionName][typeID]
			if !ok || p.All.Volume < 50 {
				priceMap[regionName][typeID] = prices
			}
		}

		// Use the universe price if our regional price is too low (override CCP's price)
		for typeID, p := range priceMap[regionName] {
			universePrice, ok := priceMap["universe"][typeID]
			if ok && p.All.Volume < 50 {
				universePrice.Strategy = "orders_universe"
				priceMap[regionName][typeID] = universePrice
			}
		}
	}

	for market, pmap := range priceMap {
		// this takes awhile, so let's check to see if we should stop between markets
		select {
		case <-p.stop:
			return
		default:
		}

		for itemName, price := range pmap {
			err = p.db.UpdatePrice(market, itemName, price)
			if err != nil {
				log.Printf("Error when updating price: %s", err)
			}
		}
	}
	log.Println("Done fetching market data")
}

func (p *PriceFetcher) freshPriceMap() map[string]map[int64]evepraisal.Prices {
	priceMap := make(map[string]map[int64]evepraisal.Prices)
	for _, region := range SpecialRegions {
		priceMap[region.name] = make(map[int64]evepraisal.Prices)
	}
	priceMap["universe"] = make(map[int64]evepraisal.Prices)
	return priceMap
}

func (p *PriceFetcher) FetchPriceData(client *pester.Client, baseURL string) (map[int64]evepraisal.Prices, error) {
	start := time.Now()
	url := fmt.Sprintf("%s/markets/prices/?datasource=tranquility", baseURL)
	esiPrices := make([]struct {
		TypeID        int64   `json:"type_id"`
		AveragePrice  float64 `json:"average_price"`
		AdjustedPrice float64 `json:"adjusted_price"`
	}, 0)
	err := fetchURL(client, url, &esiPrices)
	if err != nil {
		return nil, err
	}

	allPrices := make(map[int64]evepraisal.Prices, len(esiPrices))
	for _, p := range esiPrices {
		stats := evepraisal.PriceStats{
			Average:    p.AveragePrice,
			Max:        p.AdjustedPrice,
			Median:     p.AdjustedPrice,
			Min:        p.AdjustedPrice,
			Percentile: p.AdjustedPrice,
		}
		allPrices[p.TypeID] = evepraisal.Prices{
			All:      stats,
			Buy:      stats,
			Sell:     stats,
			Updated:  start,
			Strategy: "ccp",
		}
	}
	return allPrices, nil
}

func (p *PriceFetcher) FetchOrderData(client *pester.Client, baseURL string, regionIDs []int) (map[string]map[int64]evepraisal.Prices, error) {
	allOrdersByType := make(map[int64][]MarketOrder)
	finished := make(chan bool, 1)
	workerStop := make(chan bool, 1)
	errChannel := make(chan error, 1)
	fetchStart := time.Now()

	l := &sync.Mutex{}
	requestAndProcess := func(url string) (bool, error) {
		var orders []MarketOrder
		err := fetchURL(client, url, &orders)
		if err != nil {
			return false, err
		}

		l.Lock()
		for _, order := range orders {
			allOrdersByType[order.Type] = append(allOrdersByType[order.Type], order)
		}
		l.Unlock()
		if len(orders) == 0 {
			return false, nil
		}
		return true, nil
	}

	wg := &sync.WaitGroup{}
	for _, regionID := range regionIDs {
		wg.Add(1)
		go func(regionID int) {
			defer wg.Done()
			page := 1
			for {
				select {
				case <-workerStop:
					return
				default:
				}

				url := fmt.Sprintf("%s/markets/%d/orders/?datasource=tranquility&order_type=all&page=%d", baseURL, regionID, page)
				hasMore, err := requestAndProcess(url)
				if err != nil {
					errChannel <- fmt.Errorf("Failed to fetch market orders: %s", err)
					return
				}

				if !hasMore {
					break
				}
				page++
			}
		}(regionID)
	}

	go func() {
		wg.Wait()
		close(finished)
	}()

	select {
	case <-finished:
	case <-p.stop:
		close(workerStop)
		return nil, errors.New("Stopping during price fetch")
	case err := <-errChannel:
		if err != nil {
			close(workerStop)
			return nil, err
		}
	}

	log.Println("Performing aggregates on order data")
	// Calculate aggregates that we care about:
	newPriceMap := p.freshPriceMap()
	for k, orders := range allOrdersByType {
		for _, region := range SpecialRegions {
			filteredOrders := make([]MarketOrder, 0)
			ordercount := 0
			for _, order := range orders {
				matched := false
				for _, station := range region.stations {
					if station == order.StationID {
						matched = true
						ordercount++
						break
					}
				}
				if matched {
					filteredOrders = append(filteredOrders, order)
				}
			}
			agg := getPriceAggregatesForOrders(filteredOrders)
			agg.Updated = fetchStart
			agg.Strategy = "orders"
			newPriceMap[region.name][k] = agg
		}
		agg := getPriceAggregatesForOrders(orders)
		agg.Updated = fetchStart
		newPriceMap["universe"][k] = agg
	}

	log.Println("Finished performing aggregates on order data")

	return newPriceMap, nil
}
<commit_msg>Use sell volume (instead of buy/sell) to determine pricing fallback<commit_after>package esi

import (
	"errors"
	"fmt"
	"log"
	"sync"
	"time"

	"github.com/evepraisal/go-evepraisal"
	"github.com/sethgrid/pester"
)

type MarketOrder struct {
	ID            int64   `json:"order_id"`
	Type          int64   `json:"type_id"`
	StationID     int64   `json:"location_id"`
	Volume        int64   `json:"volume_remain"`
	MinVolume     int64   `json:"min_volume"`
	Price         float64 `json:"price"`
	Buy           bool    `json:"is_buy_order"`
	Duration      int64   `json:"duration"`
	Issued        string  `json:"issued"`
	VolumeEntered int64   `json:"volumeEntered"`
	Range         string  `json:"range"`
}

var SpecialRegions = []struct {
	name     string
	stations []int64
}{
	{
		// 10000002
		name:     "jita",
		stations: []int64{60003466, 60003760, 60003757, 60000361, 60000451, 60004423, 60002959, 60003460, 60003055, 60003469, 60000364, 60002953, 60000463, 60003463},
	}, {
		// 10000043
		name:     "amarr",
		stations: []int64{60008950, 60002569, 60008494},
	}, {
		// 10000032
		name:     "dodixie",
		stations: []int64{60011866, 60001867},
	}, {
		// 10000042
		name:     "hek",
		stations: []int64{60005236, 60004516, 60015140, 60005686, 60011287, 60005236},
	},
}

type PriceFetcher struct {
	db      evepraisal.PriceDB
	client  *pester.Client
	baseURL string

	stop chan bool
	wg   *sync.WaitGroup
}

func NewPriceFetcher(priceDB evepraisal.PriceDB, baseURL string, client *pester.Client) (*PriceFetcher, error) {

	p := &PriceFetcher{
		db:      priceDB,
		client:  client,
		baseURL: baseURL,

		stop: make(chan bool),
		wg:   &sync.WaitGroup{},
	}

	p.wg.Add(1)
	go func() {
		defer p.wg.Done()
		for {
			start := time.Now()
			p.runOnce()
			select {
			case <-time.After((5 * time.Minute) - time.Since(start)):
			case <-p.stop:
				return
			}
		}
	}()

	return p, nil
}

func (p *PriceFetcher) Close() error {
	close(p.stop)
	p.wg.Wait()
	return nil
}

func regionNames() []string {
	regions := make([]string, len(SpecialRegions)+1)
	regions[0] = "universe"
	for i, region := range SpecialRegions {
		regions[i+1] = region.name
	}
	return regions
}

func (p *PriceFetcher) runOnce() {
	log.Println("Fetch market data")
	priceMap, err := p.FetchOrderData(p.client, p.baseURL, []int{10000002, 10000042, 10000027, 10000032, 10000043})
	if err != nil {
		log.Println("ERROR: fetching market data: ", err)
		return
	}

	pricesFromCCP, err := p.FetchPriceData(p.client, p.baseURL)
	if err != nil {
		log.Println("ERROR: fetching CCP price data: ", err)
		return
	}

	for _, regionName := range regionNames() {
		// Use CCP's price if our regional price is too low
		for typeID, prices := range pricesFromCCP {
			p, ok := priceMap[regionName][typeID]
			if !ok || p.Sell.Volume < 20 {
				priceMap[regionName][typeID] = prices
			}
		}

		// Use the universe price if our regional price is too low (override CCP's price)
		for typeID, p := range priceMap[regionName] {
			universePrice, ok := priceMap["universe"][typeID]
			if ok && p.Sell.Volume < 20 {
				universePrice.Strategy = "orders_universe"
				priceMap[regionName][typeID] = universePrice
			}
		}
	}

	for market, pmap := range priceMap {
		// this takes awhile, so let's check to see if we should stop between markets
		select {
		case <-p.stop:
			return
		default:
		}

		for itemName, price := range pmap {
			err = p.db.UpdatePrice(market, itemName, price)
			if err != nil {
				log.Printf("Error when updating price: %s", err)
			}
		}
	}
	log.Println("Done fetching market data")
}

func (p *PriceFetcher) freshPriceMap() map[string]map[int64]evepraisal.Prices {
	priceMap := make(map[string]map[int64]evepraisal.Prices)
	for _, region := range SpecialRegions {
		priceMap[region.name] = make(map[int64]evepraisal.Prices)
	}
	priceMap["universe"] = make(map[int64]evepraisal.Prices)
	return priceMap
}

func (p *PriceFetcher) FetchPriceData(client *pester.Client, baseURL string) (map[int64]evepraisal.Prices, error) {
	start := time.Now()
	url := fmt.Sprintf("%s/markets/prices/?datasource=tranquility", baseURL)
	esiPrices := make([]struct {
		TypeID        int64   `json:"type_id"`
		AveragePrice  float64 `json:"average_price"`
		AdjustedPrice float64 `json:"adjusted_price"`
	}, 0)
	err := fetchURL(client, url, &esiPrices)
	if err != nil {
		return nil, err
	}

	allPrices := make(map[int64]evepraisal.Prices, len(esiPrices))
	for _, p := range esiPrices {
		stats := evepraisal.PriceStats{
			Average:    p.AveragePrice,
			Max:        p.AdjustedPrice,
			Median:     p.AdjustedPrice,
			Min:        p.AdjustedPrice,
			Percentile: p.AdjustedPrice,
		}
		allPrices[p.TypeID] = evepraisal.Prices{
			All:      stats,
			Buy:      stats,
			Sell:     stats,
			Updated:  start,
			Strategy: "ccp",
		}
	}
	return allPrices, nil
}

func (p *PriceFetcher) FetchOrderData(client *pester.Client, baseURL string, regionIDs []int) (map[string]map[int64]evepraisal.Prices, error) {
	allOrdersByType := make(map[int64][]MarketOrder)
	finished := make(chan bool, 1)
	workerStop := make(chan bool, 1)
	errChannel := make(chan error, 1)
	fetchStart := time.Now()

	l := &sync.Mutex{}
	requestAndProcess := func(url string) (bool, error) {
		var orders []MarketOrder
		err := fetchURL(client, url, &orders)
		if err != nil {
			return false, err
		}

		l.Lock()
		for _, order := range orders {
			allOrdersByType[order.Type] = append(allOrdersByType[order.Type], order)
		}
		l.Unlock()
		if len(orders) == 0 {
			return false, nil
		}
		return true, nil
	}

	wg := &sync.WaitGroup{}
	for _, regionID := range regionIDs {
		wg.Add(1)
		go func(regionID int) {
			defer wg.Done()
			page := 1
			for {
				select {
				case <-workerStop:
					return
				default:
				}

				url := fmt.Sprintf("%s/markets/%d/orders/?datasource=tranquility&order_type=all&page=%d", baseURL, regionID, page)
				hasMore, err := requestAndProcess(url)
				if err != nil {
					errChannel <- fmt.Errorf("Failed to fetch market orders: %s", err)
					return
				}

				if !hasMore {
					break
				}
				page++
			}
		}(regionID)
	}

	go func() {
		wg.Wait()
		close(finished)
	}()

	select {
	case <-finished:
	case <-p.stop:
		close(workerStop)
		return nil, errors.New("Stopping during price fetch")
	case err := <-errChannel:
		if err != nil {
			close(workerStop)
			return nil, err
		}
	}

	log.Println("Performing aggregates on order data")
	// Calculate aggregates that we care about:
	newPriceMap := p.freshPriceMap()
	for k, orders := range allOrdersByType {
		for _, region := range SpecialRegions {
			filteredOrders := make([]MarketOrder, 0)
			ordercount := 0
			for _, order := range orders {
				matched := false
				for _, station := range region.stations {
					if station == order.StationID {
						matched = true
						ordercount++
						break
					}
				}
				if matched {
					filteredOrders = append(filteredOrders, order)
				}
			}
			agg := getPriceAggregatesForOrders(filteredOrders)
			agg.Updated = fetchStart
			agg.Strategy = "orders"
			newPriceMap[region.name][k] = agg
		}
		agg := getPriceAggregatesForOrders(orders)
		agg.Updated = fetchStart
		newPriceMap["universe"][k] = agg
	}

	log.Println("Finished performing aggregates on order data")

	return newPriceMap, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/garyburd/redigo/redis"
	"encoding/json"
	"compress/gzip"
	"bytes"
	"io/ioutil"
	"strings"
	//"os"
)

/*

type FileStat struct {
	Path     string
	Hash     string
	Size     int64
	Hostname string
}
*/

// Make the Hostnames an array,
// Store every hostname that matches the hash
// Fake the file for each to the stdout buffer
type Hash struct {
	Hostnames  []string
	Contents   []byte
}

func StringInArray(a string, list map[string]Hash) bool {
	for q, _ := range list {
		if q == a {
			return true
		}
	}
	return false
}

func cater(path string) bytes.Buffer {
	var response bytes.Buffer
	knownHashes := grab(path)

	for _, hash := range knownHashes {

		var b bytes.Buffer
		b.Write([]byte(hash.Contents))

		gr, _ := gzip.NewReader(&b)
		defer gr.Close()
		plaintext, _ := ioutil.ReadAll(gr)

		for _, a := range strings.Split(string(plaintext), "\n") {
			if len(a) == 0 {
				continue
			}

			for _, b := range hash.Hostnames {
				line := fmt.Sprintf("%s:%s\n", b, a)
				response.Write([]byte(line))
			}
		}
	}
	return response

}

func grab(path string) map[string]Hash {
	var file_contents_gzipped []byte
	knownHashes := make(map[string]Hash)

	fmt.Printf("search path: %s\n", path)
	conn, _ := redis.Dial("tcp", ":6379")
	defer conn.Close()
	x, _ := redis.Values(conn.Do("KEYS", path))
	var fs FileStat
	for _, z := range x {
		z = fmt.Sprintf("%s", z)
		reply, _ := redis.String(conn.Do("GET", z))
		y := []byte(reply)

		json.Unmarshal(y, &fs)

		// Check if we already have this hash, before wasting redis time
		if ! StringInArray(fs.Hash, knownHashes) {
			file_contents_gzipped_string, _ := redis.String(conn.Do("GET", fs.Hash))
			file_contents_gzipped = []byte(file_contents_gzipped_string)

			fmt.Println("NEW HASH")
			knownHashes[fs.Hash] = Hash{
				Hostnames: []string{fs.Hostname},
				Contents: []byte(file_contents_gzipped),
			}
		} else {
			file_contents_gzipped = knownHashes[fs.Hash].Contents
			old_hostnames := knownHashes[fs.Hash].Hostnames
			new_hostnames := append(old_hostnames, fs.Hostname)
			knownHashes[fs.Hash] = Hash{
				Hostnames: new_hostnames,
				Contents: file_contents_gzipped,
			}
		}
	}
	return  knownHashes
}

/*
func main() {
	if len(os.Args) != 2 {
		fmt.Printf("usage: %s filepattern\n", os.Args[0])
		return
	}
	cater_response := cater(os.Args[1])
	fmt.Printf(cater_response.String())
}
*/
<commit_msg>Added grep & grepcount functions<commit_after>package main

import (
	"fmt"
	"github.com/garyburd/redigo/redis"
	"encoding/json"
	"compress/gzip"
	"bytes"
	"io/ioutil"
	"strings"
	"os"
	"github.com/codegangsta/cli"
)

type FileStat struct {
	Path     string
	Hash     string
	Size     int64
	Hostname string
}

// Make the Hostnames an array,
// Store every hostname that matches the hash
// Fake the file for each to the stdout buffer
type Hash struct {
	Hostnames  []string
	Contents   []byte
}

func StringInArray(a string, list map[string]Hash) bool {
	for q, _ := range list {
		if q == a {
			return true
		}
	}
	return false
}

func greper(path string, pattern string, countonly bool) {
	var response bytes.Buffer
	var count int
	count = 0
	knownHashes := grab(path)
	fmt.Printf("searching for '%s' in %s\n", pattern, path)

	for _, hash := range knownHashes {

		var b bytes.Buffer
		b.Write([]byte(hash.Contents))

		gr, _ := gzip.NewReader(&b)
		defer gr.Close()
		plaintext, _ := ioutil.ReadAll(gr)

		if countonly {
			if strings.Contains(string(plaintext), pattern) {
				count += len(hash.Hostnames)
				continue
			}
		}

		for _, a := range strings.Split(string(plaintext), "\n") {
			if len(a) == 0 {
				continue
			}

			if ! strings.Contains(a, pattern) {
				continue
			}

			for _, b := range hash.Hostnames {

				line := fmt.Sprintf("%s:%s\n", b, a)
				response.Write([]byte(line))
			}
		}
	}
	if countonly {
		fmt.Printf("Count: %d\n", count)
	}
	fmt.Printf(response.String())

}

func cater(path string) {
	var response bytes.Buffer
	knownHashes := grab(path)

	for _, hash := range knownHashes {

		var b bytes.Buffer
		b.Write([]byte(hash.Contents))

		gr, _ := gzip.NewReader(&b)
		defer gr.Close()
		plaintext, _ := ioutil.ReadAll(gr)

		for _, a := range strings.Split(string(plaintext), "\n") {
			if len(a) == 0 {
				continue
			}

			for _, b := range hash.Hostnames {
				line := fmt.Sprintf("%s:%s\n", b, a)
				response.Write([]byte(line))
			}
		}
	}
	// return response
	fmt.Printf(response.String())

}

func grab(path string) map[string]Hash {
	var file_contents_gzipped []byte
	knownHashes := make(map[string]Hash)

	conn, _ := redis.Dial("tcp", ":6379")
	defer conn.Close()
	x, _ := redis.Values(conn.Do("KEYS", path))
	var fs FileStat
	for _, z := range x {
		z = fmt.Sprintf("%s", z)
		reply, _ := redis.String(conn.Do("GET", z))
		y := []byte(reply)

		json.Unmarshal(y, &fs)

		// Check if we already have this hash, before wasting redis time
		if ! StringInArray(fs.Hash, knownHashes) {
			file_contents_gzipped_string, _ := redis.String(conn.Do("GET", fs.Hash))
			file_contents_gzipped = []byte(file_contents_gzipped_string)

			knownHashes[fs.Hash] = Hash{
				Hostnames: []string{fs.Hostname},
				Contents: []byte(file_contents_gzipped),
			}
		} else {
			file_contents_gzipped = knownHashes[fs.Hash].Contents
			old_hostnames := knownHashes[fs.Hash].Hostnames
			new_hostnames := append(old_hostnames, fs.Hostname)
			knownHashes[fs.Hash] = Hash{
				Hostnames: new_hostnames,
				Contents: file_contents_gzipped,
			}
		}
	}
	return  knownHashes
}

func main() {

	app := cli.NewApp()
	app.Name = "grab"
	app.Usage = "Grab file data from redis server"

	app.Commands = []cli.Command{
		{
			Name:      "cat",
			Usage:     "add a task to the list",
			Action:    func(c *cli.Context) {
				if len(c.Args()) != 1 {
					return
				}
				path := c.Args().First()
				cater(path)
			},
		},
		{
			Name:      "grep",
			Usage:     "grep for pattern",
			Action:    func(c *cli.Context) {
				if len(c.Args()) != 2 {
					fmt.Printf("missing file and pattern\n")
					return
				}
				path := c.Args().First()
				pattern := c.Args()[1]
				greper(path, pattern, false)
			},
		},
		{
			Name:      "grepcount",
			Usage:     "count pattern",
			Action:    func(c *cli.Context) {
				if len(c.Args()) != 2 {
					fmt.Printf("missing file and pattern\n")
					return
				}
				path := c.Args().First()
				pattern := c.Args()[1]
				greper(path, pattern, true)
			},
		},
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e_node

import (
	"fmt"
	"os"
	"path/filepath"
	"time"

	"regexp"

	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/kubernetes/pkg/features"
	"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
	"k8s.io/kubernetes/test/e2e/framework"

	pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1"
	dm "k8s.io/kubernetes/pkg/kubelet/cm/devicemanager"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	// fake resource name
	resourceName                 = "fake.com/resource"
	resourceNameWithProbeSupport = "fake.com/resource2"
)

// Serial because the test restarts Kubelet
var _ = framework.KubeDescribe("Device Plugin [Feature:DevicePlugin][NodeFeature:DevicePlugin][Serial]", func() {
	f := framework.NewDefaultFramework("device-plugin-errors")
	testDevicePlugin(f, false, pluginapi.DevicePluginPath)
})

var _ = framework.KubeDescribe("Device Plugin [Feature:DevicePluginProbe][NodeFeature:DevicePluginProbe][Serial]", func() {
	f := framework.NewDefaultFramework("device-plugin-errors")
	testDevicePlugin(f, true, "/var/lib/kubelet/plugins/")
})

func testDevicePlugin(f *framework.Framework, enablePluginWatcher bool, pluginSockDir string) {
	Context("DevicePlugin", func() {
		By("Enabling support for Kubelet Plugins Watcher")
		tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
			initialConfig.FeatureGates[string(features.KubeletPluginsWatcher)] = enablePluginWatcher
		})
		//devicePluginSockPaths := []string{pluginapi.DevicePluginPath}
		It("Verifies the Kubelet device plugin functionality.", func() {
			By("Start stub device plugin")
			// fake devices for e2e test
			devs := []*pluginapi.Device{
				{ID: "Dev-1", Health: pluginapi.Healthy},
				{ID: "Dev-2", Health: pluginapi.Healthy},
			}

			socketPath := pluginSockDir + "dp." + fmt.Sprintf("%d", time.Now().Unix())
			framework.Logf("socketPath %v", socketPath)

			dp1 := dm.NewDevicePluginStub(devs, socketPath, resourceName, false)
			dp1.SetAllocFunc(stubAllocFunc)
			err := dp1.Start()
			framework.ExpectNoError(err)

			By("Register resources")
			err = dp1.Register(pluginapi.KubeletSocket, resourceName, pluginapi.DevicePluginPath)
			framework.ExpectNoError(err)

			By("Waiting for the resource exported by the stub device plugin to become available on the local node")
			devsLen := int64(len(devs))
			Eventually(func() bool {
				node, err := f.ClientSet.CoreV1().Nodes().Get(framework.TestContext.NodeName, metav1.GetOptions{})
				framework.ExpectNoError(err)
				return numberOfDevicesCapacity(node, resourceName) == devsLen &&
					numberOfDevicesAllocatable(node, resourceName) == devsLen
			}, 30*time.Second, framework.Poll).Should(BeTrue())

			By("Creating one pod on node with at least one fake-device")
			podRECMD := "devs=$(ls /tmp/ | egrep '^Dev-[0-9]+$') && echo stub devices: $devs"
			pod1 := f.PodClient().CreateSync(makeBusyboxPod(resourceName, podRECMD))
			deviceIDRE := "stub devices: (Dev-[0-9]+)"
			devId1 := parseLog(f, pod1.Name, pod1.Name, deviceIDRE)
			Expect(devId1).To(Not(Equal("")))

			pod1, err = f.PodClient().Get(pod1.Name, metav1.GetOptions{})
			framework.ExpectNoError(err)

			ensurePodContainerRestart(f, pod1.Name, pod1.Name)

			By("Confirming that device assignment persists even after container restart")
			devIdAfterRestart := parseLog(f, pod1.Name, pod1.Name, deviceIDRE)
			Expect(devIdAfterRestart).To(Equal(devId1))

			By("Restarting Kubelet")
			restartKubelet()

			ensurePodContainerRestart(f, pod1.Name, pod1.Name)
			By("Confirming that after a kubelet restart, fake-device assignement is kept")
			devIdRestart1 := parseLog(f, pod1.Name, pod1.Name, deviceIDRE)
			Expect(devIdRestart1).To(Equal(devId1))

			By("Wait for node is ready")
			framework.WaitForAllNodesSchedulable(f.ClientSet, framework.TestContext.NodeSchedulableTimeout)

			By("Re-Register resources")
			dp1 = dm.NewDevicePluginStub(devs, socketPath, resourceName, false)
			dp1.SetAllocFunc(stubAllocFunc)
			err = dp1.Start()
			framework.ExpectNoError(err)

			err = dp1.Register(pluginapi.KubeletSocket, resourceName, pluginSockDir)
			framework.ExpectNoError(err)

			By("Waiting for resource to become available on the local node after re-registration")
			Eventually(func() bool {
				node, err := f.ClientSet.CoreV1().Nodes().Get(framework.TestContext.NodeName, metav1.GetOptions{})
				framework.ExpectNoError(err)
				return numberOfDevicesCapacity(node, resourceName) == devsLen &&
					numberOfDevicesAllocatable(node, resourceName) == devsLen
			}, 30*time.Second, framework.Poll).Should(BeTrue())

			By("Creating another pod")
			pod2 := f.PodClient().CreateSync(makeBusyboxPod(resourceName, podRECMD))

			By("Checking that pod got a different fake device")
			devId2 := parseLog(f, pod2.Name, pod2.Name, deviceIDRE)

			Expect(devId1).To(Not(Equal(devId2)))

			By("Deleting device plugin.")
			err = dp1.Stop()
			framework.ExpectNoError(err)

			By("Waiting for stub device plugin to become unhealthy on the local node")
			Eventually(func() int64 {
				node, err := f.ClientSet.CoreV1().Nodes().Get(framework.TestContext.NodeName, metav1.GetOptions{})
				framework.ExpectNoError(err)
				return numberOfDevicesAllocatable(node, resourceName)
			}, 30*time.Second, framework.Poll).Should(Equal(int64(0)))

			By("Checking that scheduled pods can continue to run even after we delete device plugin.")
			ensurePodContainerRestart(f, pod1.Name, pod1.Name)
			devIdRestart1 = parseLog(f, pod1.Name, pod1.Name, deviceIDRE)
			Expect(devIdRestart1).To(Equal(devId1))

			ensurePodContainerRestart(f, pod2.Name, pod2.Name)
			devIdRestart2 := parseLog(f, pod2.Name, pod2.Name, deviceIDRE)
			Expect(devIdRestart2).To(Equal(devId2))

			By("Re-register resources")
			dp1 = dm.NewDevicePluginStub(devs, socketPath, resourceName, false)
			dp1.SetAllocFunc(stubAllocFunc)
			err = dp1.Start()
			framework.ExpectNoError(err)

			err = dp1.Register(pluginapi.KubeletSocket, resourceName, pluginSockDir)
			framework.ExpectNoError(err)

			By("Waiting for the resource exported by the stub device plugin to become healthy on the local node")
			Eventually(func() int64 {
				node, err := f.ClientSet.CoreV1().Nodes().Get(framework.TestContext.NodeName, metav1.GetOptions{})
				framework.ExpectNoError(err)
				return numberOfDevicesAllocatable(node, resourceName)
			}, 30*time.Second, framework.Poll).Should(Equal(devsLen))

			By("Deleting device plugin again.")
			err = dp1.Stop()
			framework.ExpectNoError(err)

			By("Waiting for stub device plugin to become unavailable on the local node")
			Eventually(func() bool {
				node, err := f.ClientSet.CoreV1().Nodes().Get(framework.TestContext.NodeName, metav1.GetOptions{})
				framework.ExpectNoError(err)
				return numberOfDevicesCapacity(node, resourceName) <= 0
			}, 10*time.Minute, framework.Poll).Should(BeTrue())

			By("Restarting Kubelet second time.")
			restartKubelet()

			By("Checking that scheduled pods can continue to run even after we delete device plugin and restart Kubelet Eventually.")
			ensurePodContainerRestart(f, pod1.Name, pod1.Name)
			devIdRestart1 = parseLog(f, pod1.Name, pod1.Name, deviceIDRE)
			Expect(devIdRestart1).To(Equal(devId1))

			ensurePodContainerRestart(f, pod2.Name, pod2.Name)
			devIdRestart2 = parseLog(f, pod2.Name, pod2.Name, deviceIDRE)
			Expect(devIdRestart2).To(Equal(devId2))

			// Cleanup
			f.PodClient().DeleteSync(pod1.Name, &metav1.DeleteOptions{}, framework.DefaultPodDeletionTimeout)
			f.PodClient().DeleteSync(pod2.Name, &metav1.DeleteOptions{}, framework.DefaultPodDeletionTimeout)
		})
	})
}

// makeBusyboxPod returns a simple Pod spec with a busybox container
// that requests resourceName and runs the specified command.
func makeBusyboxPod(resourceName, cmd string) *v1.Pod {
	podName := "device-plugin-test-" + string(uuid.NewUUID())
	rl := v1.ResourceList{v1.ResourceName(resourceName): *resource.NewQuantity(1, resource.DecimalSI)}

	return &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{Name: podName},
		Spec: v1.PodSpec{
			RestartPolicy: v1.RestartPolicyAlways,
			Containers: []v1.Container{{
				Image: busyboxImage,
				Name:  podName,
				// Runs the specified command in the test pod.
				Command: []string{"sh", "-c", cmd},
				Resources: v1.ResourceRequirements{
					Limits:   rl,
					Requests: rl,
				},
			}},
		},
	}
}

// ensurePodContainerRestart confirms that pod container has restarted at least once
func ensurePodContainerRestart(f *framework.Framework, podName string, contName string) {
	var initialCount int32
	var currentCount int32
	p, err := f.PodClient().Get(podName, metav1.GetOptions{})
	if err != nil || len(p.Status.ContainerStatuses) < 1 {
		framework.Failf("ensurePodContainerRestart failed for pod %q: %v", podName, err)
	}
	initialCount = p.Status.ContainerStatuses[0].RestartCount
	Eventually(func() bool {
		p, err = f.PodClient().Get(podName, metav1.GetOptions{})
		if err != nil || len(p.Status.ContainerStatuses) < 1 {
			return false
		}
		currentCount = p.Status.ContainerStatuses[0].RestartCount
		framework.Logf("initial %v, current %v", initialCount, currentCount)
		return currentCount > initialCount
	}, 5*time.Minute, framework.Poll).Should(BeTrue())
}

// parseLog returns the matching string for the specified regular expression parsed from the container logs.
func parseLog(f *framework.Framework, podName string, contName string, re string) string {
	logs, err := framework.GetPodLogs(f.ClientSet, f.Namespace.Name, podName, contName)
	if err != nil {
		framework.Failf("GetPodLogs for pod %q failed: %v", podName, err)
	}

	framework.Logf("got pod logs: %v", logs)
	regex := regexp.MustCompile(re)
	matches := regex.FindStringSubmatch(logs)
	if len(matches) < 2 {
		return ""
	}

	return matches[1]
}

// numberOfDevicesCapacity returns the number of devices of resourceName advertised by a node capacity
func numberOfDevicesCapacity(node *v1.Node, resourceName string) int64 {
	val, ok := node.Status.Capacity[v1.ResourceName(resourceName)]
	if !ok {
		return 0
	}

	return val.Value()
}

// numberOfDevicesAllocatable returns the number of devices of resourceName advertised by a node allocatable
func numberOfDevicesAllocatable(node *v1.Node, resourceName string) int64 {
	val, ok := node.Status.Allocatable[v1.ResourceName(resourceName)]
	if !ok {
		return 0
	}

	return val.Value()
}

// stubAllocFunc will pass to stub device plugin
func stubAllocFunc(r *pluginapi.AllocateRequest, devs map[string]pluginapi.Device) (*pluginapi.AllocateResponse, error) {
	var responses pluginapi.AllocateResponse
	for _, req := range r.ContainerRequests {
		response := &pluginapi.ContainerAllocateResponse{}
		for _, requestID := range req.DevicesIDs {
			dev, ok := devs[requestID]
			if !ok {
				return nil, fmt.Errorf("invalid allocation request with non-existing device %s", requestID)
			}

			if dev.Health != pluginapi.Healthy {
				return nil, fmt.Errorf("invalid allocation request with unhealthy device: %s", requestID)
			}

			// create fake device file
			fpath := filepath.Join("/tmp", dev.ID)

			// clean first
			os.RemoveAll(fpath)
			f, err := os.Create(fpath)
			if err != nil && !os.IsExist(err) {
				return nil, fmt.Errorf("failed to create fake device file: %s", err)
			}

			f.Close()

			response.Mounts = append(response.Mounts, &pluginapi.Mount{
				ContainerPath: fpath,
				HostPath:      fpath,
			})
		}
		responses.ContainerResponses = append(responses.ContainerResponses, response)
	}

	return &responses, nil
}
<commit_msg>Updates test/e2e_node/device_plugin.go to cope with recent device manager change in commit 7b1ae66. Also changes the test to make sure node is indeed ready after Kubelet restart. The previous readiness check may use old API state but didn't run into the issue due to the delay of waiting for pod restart.<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e_node

import (
	"fmt"
	"os"
	"path/filepath"
	"time"

	"regexp"

	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/kubernetes/pkg/features"
	"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
	"k8s.io/kubernetes/test/e2e/framework"

	pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1beta1"
	dm "k8s.io/kubernetes/pkg/kubelet/cm/devicemanager"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	// fake resource name
	resourceName                 = "fake.com/resource"
	resourceNameWithProbeSupport = "fake.com/resource2"
)

// Serial because the test restarts Kubelet
var _ = framework.KubeDescribe("Device Plugin [Feature:DevicePlugin][NodeFeature:DevicePlugin][Serial]", func() {
	f := framework.NewDefaultFramework("device-plugin-errors")
	testDevicePlugin(f, false, pluginapi.DevicePluginPath)
})

var _ = framework.KubeDescribe("Device Plugin [Feature:DevicePluginProbe][NodeFeature:DevicePluginProbe][Serial]", func() {
	f := framework.NewDefaultFramework("device-plugin-errors")
	testDevicePlugin(f, true, "/var/lib/kubelet/plugins/")
})

func testDevicePlugin(f *framework.Framework, enablePluginWatcher bool, pluginSockDir string) {
	Context("DevicePlugin", func() {
		By("Enabling support for Kubelet Plugins Watcher")
		tempSetCurrentKubeletConfig(f, func(initialConfig *kubeletconfig.KubeletConfiguration) {
			initialConfig.FeatureGates[string(features.KubeletPluginsWatcher)] = enablePluginWatcher
		})
		It("Verifies the Kubelet device plugin functionality.", func() {
			By("Start stub device plugin")
			// fake devices for e2e test
			devs := []*pluginapi.Device{
				{ID: "Dev-1", Health: pluginapi.Healthy},
				{ID: "Dev-2", Health: pluginapi.Healthy},
			}

			socketPath := pluginSockDir + "dp." + fmt.Sprintf("%d", time.Now().Unix())
			framework.Logf("socketPath %v", socketPath)

			dp1 := dm.NewDevicePluginStub(devs, socketPath, resourceName, false)
			dp1.SetAllocFunc(stubAllocFunc)
			err := dp1.Start()
			framework.ExpectNoError(err)

			By("Register resources")
			err = dp1.Register(pluginapi.KubeletSocket, resourceName, pluginSockDir)
			framework.ExpectNoError(err)

			By("Waiting for the resource exported by the stub device plugin to become available on the local node")
			devsLen := int64(len(devs))
			Eventually(func() bool {
				node, err := f.ClientSet.CoreV1().Nodes().Get(framework.TestContext.NodeName, metav1.GetOptions{})
				framework.ExpectNoError(err)
				return numberOfDevicesCapacity(node, resourceName) == devsLen &&
					numberOfDevicesAllocatable(node, resourceName) == devsLen
			}, 30*time.Second, framework.Poll).Should(BeTrue())

			By("Creating one pod on node with at least one fake-device")
			podRECMD := "devs=$(ls /tmp/ | egrep '^Dev-[0-9]+$') && echo stub devices: $devs"
			pod1 := f.PodClient().CreateSync(makeBusyboxPod(resourceName, podRECMD))
			deviceIDRE := "stub devices: (Dev-[0-9]+)"
			devId1 := parseLog(f, pod1.Name, pod1.Name, deviceIDRE)
			Expect(devId1).To(Not(Equal("")))

			pod1, err = f.PodClient().Get(pod1.Name, metav1.GetOptions{})
			framework.ExpectNoError(err)

			ensurePodContainerRestart(f, pod1.Name, pod1.Name)

			By("Confirming that device assignment persists even after container restart")
			devIdAfterRestart := parseLog(f, pod1.Name, pod1.Name, deviceIDRE)
			Expect(devIdAfterRestart).To(Equal(devId1))

			restartTime := time.Now()
			By("Restarting Kubelet")
			restartKubelet()

			// We need to wait for node to be ready before re-registering stub device plugin.
			// Otherwise, Kubelet DeviceManager may remove the re-registered sockets after it starts.
			By("Wait for node is ready")
			Eventually(func() bool {
				node, err := f.ClientSet.CoreV1().Nodes().Get(framework.TestContext.NodeName, metav1.GetOptions{})
				framework.ExpectNoError(err)
				for _, cond := range node.Status.Conditions {
					if cond.Type == v1.NodeReady && cond.Status == v1.ConditionTrue && cond.LastHeartbeatTime.After(restartTime) {
						return true
					}
				}
				return false
			}, 5*time.Minute, framework.Poll).Should(BeTrue())

			By("Re-Register resources")
			dp1 = dm.NewDevicePluginStub(devs, socketPath, resourceName, false)
			dp1.SetAllocFunc(stubAllocFunc)
			err = dp1.Start()
			framework.ExpectNoError(err)

			err = dp1.Register(pluginapi.KubeletSocket, resourceName, pluginSockDir)
			framework.ExpectNoError(err)

			ensurePodContainerRestart(f, pod1.Name, pod1.Name)
			By("Confirming that after a kubelet restart, fake-device assignement is kept")
			devIdRestart1 := parseLog(f, pod1.Name, pod1.Name, deviceIDRE)
			Expect(devIdRestart1).To(Equal(devId1))

			By("Waiting for resource to become available on the local node after re-registration")
			Eventually(func() bool {
				node, err := f.ClientSet.CoreV1().Nodes().Get(framework.TestContext.NodeName, metav1.GetOptions{})
				framework.ExpectNoError(err)
				return numberOfDevicesCapacity(node, resourceName) == devsLen &&
					numberOfDevicesAllocatable(node, resourceName) == devsLen
			}, 30*time.Second, framework.Poll).Should(BeTrue())

			By("Creating another pod")
			pod2 := f.PodClient().CreateSync(makeBusyboxPod(resourceName, podRECMD))

			By("Checking that pod got a different fake device")
			devId2 := parseLog(f, pod2.Name, pod2.Name, deviceIDRE)

			Expect(devId1).To(Not(Equal(devId2)))

			By("Deleting device plugin.")
			err = dp1.Stop()
			framework.ExpectNoError(err)

			By("Waiting for stub device plugin to become unhealthy on the local node")
			Eventually(func() int64 {
				node, err := f.ClientSet.CoreV1().Nodes().Get(framework.TestContext.NodeName, metav1.GetOptions{})
				framework.ExpectNoError(err)
				return numberOfDevicesAllocatable(node, resourceName)
			}, 30*time.Second, framework.Poll).Should(Equal(int64(0)))

			By("Checking that scheduled pods can continue to run even after we delete device plugin.")
			ensurePodContainerRestart(f, pod1.Name, pod1.Name)
			devIdRestart1 = parseLog(f, pod1.Name, pod1.Name, deviceIDRE)
			Expect(devIdRestart1).To(Equal(devId1))

			ensurePodContainerRestart(f, pod2.Name, pod2.Name)
			devIdRestart2 := parseLog(f, pod2.Name, pod2.Name, deviceIDRE)
			Expect(devIdRestart2).To(Equal(devId2))

			By("Re-register resources")
			dp1 = dm.NewDevicePluginStub(devs, socketPath, resourceName, false)
			dp1.SetAllocFunc(stubAllocFunc)
			err = dp1.Start()
			framework.ExpectNoError(err)

			err = dp1.Register(pluginapi.KubeletSocket, resourceName, pluginSockDir)
			framework.ExpectNoError(err)

			By("Waiting for the resource exported by the stub device plugin to become healthy on the local node")
			Eventually(func() int64 {
				node, err := f.ClientSet.CoreV1().Nodes().Get(framework.TestContext.NodeName, metav1.GetOptions{})
				framework.ExpectNoError(err)
				return numberOfDevicesAllocatable(node, resourceName)
			}, 30*time.Second, framework.Poll).Should(Equal(devsLen))

			By("Deleting device plugin again.")
			err = dp1.Stop()
			framework.ExpectNoError(err)

			By("Waiting for stub device plugin to become unavailable on the local node")
			Eventually(func() bool {
				node, err := f.ClientSet.CoreV1().Nodes().Get(framework.TestContext.NodeName, metav1.GetOptions{})
				framework.ExpectNoError(err)
				return numberOfDevicesCapacity(node, resourceName) <= 0
			}, 10*time.Minute, framework.Poll).Should(BeTrue())

			// Cleanup
			f.PodClient().DeleteSync(pod1.Name, &metav1.DeleteOptions{}, framework.DefaultPodDeletionTimeout)
			f.PodClient().DeleteSync(pod2.Name, &metav1.DeleteOptions{}, framework.DefaultPodDeletionTimeout)
		})
	})
}

// makeBusyboxPod returns a simple Pod spec with a busybox container
// that requests resourceName and runs the specified command.
func makeBusyboxPod(resourceName, cmd string) *v1.Pod {
	podName := "device-plugin-test-" + string(uuid.NewUUID())
	rl := v1.ResourceList{v1.ResourceName(resourceName): *resource.NewQuantity(1, resource.DecimalSI)}

	return &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{Name: podName},
		Spec: v1.PodSpec{
			RestartPolicy: v1.RestartPolicyAlways,
			Containers: []v1.Container{{
				Image: busyboxImage,
				Name:  podName,
				// Runs the specified command in the test pod.
				Command: []string{"sh", "-c", cmd},
				Resources: v1.ResourceRequirements{
					Limits:   rl,
					Requests: rl,
				},
			}},
		},
	}
}

// ensurePodContainerRestart confirms that pod container has restarted at least once
func ensurePodContainerRestart(f *framework.Framework, podName string, contName string) {
	var initialCount int32
	var currentCount int32
	p, err := f.PodClient().Get(podName, metav1.GetOptions{})
	if err != nil || len(p.Status.ContainerStatuses) < 1 {
		framework.Failf("ensurePodContainerRestart failed for pod %q: %v", podName, err)
	}
	initialCount = p.Status.ContainerStatuses[0].RestartCount
	Eventually(func() bool {
		p, err = f.PodClient().Get(podName, metav1.GetOptions{})
		if err != nil || len(p.Status.ContainerStatuses) < 1 {
			return false
		}
		currentCount = p.Status.ContainerStatuses[0].RestartCount
		framework.Logf("initial %v, current %v", initialCount, currentCount)
		return currentCount > initialCount
	}, 5*time.Minute, framework.Poll).Should(BeTrue())
}

// parseLog returns the matching string for the specified regular expression parsed from the container logs.
func parseLog(f *framework.Framework, podName string, contName string, re string) string {
	logs, err := framework.GetPodLogs(f.ClientSet, f.Namespace.Name, podName, contName)
	if err != nil {
		framework.Failf("GetPodLogs for pod %q failed: %v", podName, err)
	}

	framework.Logf("got pod logs: %v", logs)
	regex := regexp.MustCompile(re)
	matches := regex.FindStringSubmatch(logs)
	if len(matches) < 2 {
		return ""
	}

	return matches[1]
}

// numberOfDevicesCapacity returns the number of devices of resourceName advertised by a node capacity
func numberOfDevicesCapacity(node *v1.Node, resourceName string) int64 {
	val, ok := node.Status.Capacity[v1.ResourceName(resourceName)]
	if !ok {
		return 0
	}

	return val.Value()
}

// numberOfDevicesAllocatable returns the number of devices of resourceName advertised by a node allocatable
func numberOfDevicesAllocatable(node *v1.Node, resourceName string) int64 {
	val, ok := node.Status.Allocatable[v1.ResourceName(resourceName)]
	if !ok {
		return 0
	}

	return val.Value()
}

// stubAllocFunc will pass to stub device plugin
func stubAllocFunc(r *pluginapi.AllocateRequest, devs map[string]pluginapi.Device) (*pluginapi.AllocateResponse, error) {
	var responses pluginapi.AllocateResponse
	for _, req := range r.ContainerRequests {
		response := &pluginapi.ContainerAllocateResponse{}
		for _, requestID := range req.DevicesIDs {
			dev, ok := devs[requestID]
			if !ok {
				return nil, fmt.Errorf("invalid allocation request with non-existing device %s", requestID)
			}

			if dev.Health != pluginapi.Healthy {
				return nil, fmt.Errorf("invalid allocation request with unhealthy device: %s", requestID)
			}

			// create fake device file
			fpath := filepath.Join("/tmp", dev.ID)

			// clean first
			os.RemoveAll(fpath)
			f, err := os.Create(fpath)
			if err != nil && !os.IsExist(err) {
				return nil, fmt.Errorf("failed to create fake device file: %s", err)
			}

			f.Close()

			response.Mounts = append(response.Mounts, &pluginapi.Mount{
				ContainerPath: fpath,
				HostPath:      fpath,
			})
		}
		responses.ContainerResponses = append(responses.ContainerResponses, response)
	}

	return &responses, nil
}
<|endoftext|>


<commit_before>// Copyright 2016 The prometheus-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package framework

import (
	"encoding/json"
	"fmt"
	"os"
	"time"

	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/apimachinery/pkg/util/yaml"

	"github.com/coreos/prometheus-operator/pkg/alertmanager"
	monitoringv1 "github.com/coreos/prometheus-operator/pkg/client/monitoring/v1"
	"github.com/coreos/prometheus-operator/pkg/client/monitoring/v1alpha1"
	"github.com/pkg/errors"
)

var ValidAlertmanagerConfig = `global:
  resolve_timeout: 5m
route:
  group_by: ['job']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 12h
  receiver: 'webhook'
receivers:
- name: 'webhook'
  webhook_configs:
  - url: 'http://alertmanagerwh:30500/'
`

func (f *Framework) MakeBasicAlertmanager(name string, replicas int32) *monitoringv1.Alertmanager {
	return &monitoringv1.Alertmanager{
		ObjectMeta: metav1.ObjectMeta{
			Name: name,
		},
		Spec: monitoringv1.AlertmanagerSpec{
			Replicas: &replicas,
		},
	}
}

func (f *Framework) MakeBasicAlertmanagerV1alpha1(name string, replicas int32) *v1alpha1.Alertmanager {
	return &v1alpha1.Alertmanager{
		ObjectMeta: metav1.ObjectMeta{
			Name: name,
		},
		Spec: v1alpha1.AlertmanagerSpec{
			Replicas: &replicas,
		},
	}
}

func (f *Framework) MakeAlertmanagerService(name, group string, serviceType v1.ServiceType) *v1.Service {
	service := &v1.Service{
		ObjectMeta: metav1.ObjectMeta{
			Name: fmt.Sprintf("alertmanager-%s", name),
			Labels: map[string]string{
				"group": group,
			},
		},
		Spec: v1.ServiceSpec{
			Type: serviceType,
			Ports: []v1.ServicePort{
				v1.ServicePort{
					Name:       "web",
					Port:       9093,
					TargetPort: intstr.FromString("web"),
				},
			},
			Selector: map[string]string{
				"alertmanager": name,
			},
		},
	}

	return service
}

func (f *Framework) SecretFromYaml(filepath string) (*v1.Secret, error) {
	manifest, err := os.Open(filepath)
	if err != nil {
		return nil, err
	}

	s := v1.Secret{}
	err = yaml.NewYAMLOrJSONDecoder(manifest, 100).Decode(&s)
	if err != nil {
		return nil, err
	}

	return &s, nil
}

func (f *Framework) AlertmanagerConfigSecret(name string) (*v1.Secret, error) {
	s, err := f.SecretFromYaml("../../contrib/kube-prometheus/manifests/alertmanager/alertmanager-config.yaml")
	if err != nil {
		return nil, err
	}

	s.Name = name
	return s, nil
}

func (f *Framework) CreateAlertmanagerAndWaitUntilReady(ns string, a *monitoringv1.Alertmanager) error {
	amConfigSecretName := fmt.Sprintf("alertmanager-%s", a.Name)
	s, err := f.AlertmanagerConfigSecret(amConfigSecretName)
	if err != nil {
		return errors.Wrap(err, fmt.Sprintf("making alertmanager config secret %v failed", amConfigSecretName))
	}
	_, err = f.KubeClient.CoreV1().Secrets(ns).Create(s)
	if err != nil {
		return errors.Wrap(err, fmt.Sprintf("creating alertmanager config secret %v failed", s.Name))
	}

	_, err = f.MonClient.Alertmanagers(ns).Create(a)
	if err != nil {
		return errors.Wrap(err, fmt.Sprintf("creating alertmanager %v failed", a.Name))
	}

	return f.WaitForAlertmanagerReady(ns, a.Name, int(*a.Spec.Replicas))
}

func (f *Framework) WaitForAlertmanagerReady(ns, name string, replicas int) error {
	err := WaitForPodsReady(
		f.KubeClient,
		ns,
		5*time.Minute,
		replicas,
		alertmanager.ListOptions(name),
	)

	return errors.Wrap(err, fmt.Sprintf("failed to create an Alertmanager cluster (%s) with %d instances", name, replicas))
}

func (f *Framework) UpdateAlertmanagerAndWaitUntilReady(ns string, a *monitoringv1.Alertmanager) error {
	_, err := f.MonClient.Alertmanagers(ns).Update(a)
	if err != nil {
		return err
	}

	err = WaitForPodsReady(
		f.KubeClient,
		ns,
		5*time.Minute,
		int(*a.Spec.Replicas),
		alertmanager.ListOptions(a.Name),
	)
	if err != nil {
		return fmt.Errorf("failed to update %d Alertmanager instances (%s): %v", a.Spec.Replicas, a.Name, err)
	}

	return nil
}

func (f *Framework) DeleteAlertmanagerAndWaitUntilGone(ns, name string) error {
	_, err := f.MonClient.Alertmanagers(ns).Get(name, metav1.GetOptions{})
	if err != nil {
		return errors.Wrap(err, fmt.Sprintf("requesting Alertmanager tpr %v failed", name))
	}

	if err := f.MonClient.Alertmanagers(ns).Delete(name, nil); err != nil {
		return errors.Wrap(err, fmt.Sprintf("deleting Alertmanager tpr %v failed", name))
	}

	if err := WaitForPodsReady(
		f.KubeClient,
		ns,
		f.DefaultTimeout,
		0,
		alertmanager.ListOptions(name),
	); err != nil {
		return errors.Wrap(err, fmt.Sprintf("waiting for Alertmanager tpr (%s) to vanish timed out", name))
	}

	return f.KubeClient.CoreV1().Secrets(ns).Delete(fmt.Sprintf("alertmanager-%s", name), nil)
}

func amImage(version string) string {
	return fmt.Sprintf("quay.io/prometheus/alertmanager:%s", version)
}

func (f *Framework) WaitForAlertmanagerInitializedMesh(ns, name string, amountPeers int) error {
	return wait.Poll(time.Second, time.Second*20, func() (bool, error) {
		amStatus, err := f.GetAlertmanagerConfig(ns, name)
		if err != nil {
			return false, err
		}
		if len(amStatus.Data.MeshStatus.Peers) == amountPeers {
			return true, nil
		}

		return false, nil
	})
}

func (f *Framework) GetAlertmanagerConfig(ns, n string) (alertmanagerStatus, error) {
	var amStatus alertmanagerStatus
	request := ProxyGetPod(f.KubeClient, ns, n, "9093", "/api/v1/status")
	resp, err := request.DoRaw()
	if err != nil {
		return amStatus, err
	}

	if err := json.Unmarshal(resp, &amStatus); err != nil {
		return amStatus, err
	}

	return amStatus, nil
}

func (f *Framework) WaitForSpecificAlertmanagerConfig(ns, amName string, expectedConfig string) error {
	return wait.Poll(time.Second, time.Minute*5, func() (bool, error) {
		config, err := f.GetAlertmanagerConfig(ns, "alertmanager-"+amName+"-0")
		if err != nil {
			return false, err
		}

		if config.Data.ConfigYAML == expectedConfig {
			return true, nil
		}

		return false, nil
	})
}

type alertmanagerStatus struct {
	Data alertmanagerStatusData `json:"data"`
}

type alertmanagerStatusData struct {
	MeshStatus meshStatus `json:"meshStatus"`
	ConfigYAML string     `json:"configYAML"`
}

type meshStatus struct {
	Peers []interface{} `json:"peers"`
}
<commit_msg>test/alertmanager: print expected and found alertmanager config<commit_after>// Copyright 2016 The prometheus-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package framework

import (
	"encoding/json"
	"fmt"
	"log"
	"os"
	"time"

	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/apimachinery/pkg/util/yaml"

	"github.com/coreos/prometheus-operator/pkg/alertmanager"
	monitoringv1 "github.com/coreos/prometheus-operator/pkg/client/monitoring/v1"
	"github.com/coreos/prometheus-operator/pkg/client/monitoring/v1alpha1"
	"github.com/pkg/errors"
)

var ValidAlertmanagerConfig = `global:
  resolve_timeout: 5m
route:
  group_by: ['job']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 12h
  receiver: 'webhook'
receivers:
- name: 'webhook'
  webhook_configs:
  - url: 'http://alertmanagerwh:30500/'
`

func (f *Framework) MakeBasicAlertmanager(name string, replicas int32) *monitoringv1.Alertmanager {
	return &monitoringv1.Alertmanager{
		ObjectMeta: metav1.ObjectMeta{
			Name: name,
		},
		Spec: monitoringv1.AlertmanagerSpec{
			Replicas: &replicas,
		},
	}
}

func (f *Framework) MakeBasicAlertmanagerV1alpha1(name string, replicas int32) *v1alpha1.Alertmanager {
	return &v1alpha1.Alertmanager{
		ObjectMeta: metav1.ObjectMeta{
			Name: name,
		},
		Spec: v1alpha1.AlertmanagerSpec{
			Replicas: &replicas,
		},
	}
}

func (f *Framework) MakeAlertmanagerService(name, group string, serviceType v1.ServiceType) *v1.Service {
	service := &v1.Service{
		ObjectMeta: metav1.ObjectMeta{
			Name: fmt.Sprintf("alertmanager-%s", name),
			Labels: map[string]string{
				"group": group,
			},
		},
		Spec: v1.ServiceSpec{
			Type: serviceType,
			Ports: []v1.ServicePort{
				v1.ServicePort{
					Name:       "web",
					Port:       9093,
					TargetPort: intstr.FromString("web"),
				},
			},
			Selector: map[string]string{
				"alertmanager": name,
			},
		},
	}

	return service
}

func (f *Framework) SecretFromYaml(filepath string) (*v1.Secret, error) {
	manifest, err := os.Open(filepath)
	if err != nil {
		return nil, err
	}

	s := v1.Secret{}
	err = yaml.NewYAMLOrJSONDecoder(manifest, 100).Decode(&s)
	if err != nil {
		return nil, err
	}

	return &s, nil
}

func (f *Framework) AlertmanagerConfigSecret(name string) (*v1.Secret, error) {
	s, err := f.SecretFromYaml("../../contrib/kube-prometheus/manifests/alertmanager/alertmanager-config.yaml")
	if err != nil {
		return nil, err
	}

	s.Name = name
	return s, nil
}

func (f *Framework) CreateAlertmanagerAndWaitUntilReady(ns string, a *monitoringv1.Alertmanager) error {
	amConfigSecretName := fmt.Sprintf("alertmanager-%s", a.Name)
	s, err := f.AlertmanagerConfigSecret(amConfigSecretName)
	if err != nil {
		return errors.Wrap(err, fmt.Sprintf("making alertmanager config secret %v failed", amConfigSecretName))
	}
	_, err = f.KubeClient.CoreV1().Secrets(ns).Create(s)
	if err != nil {
		return errors.Wrap(err, fmt.Sprintf("creating alertmanager config secret %v failed", s.Name))
	}

	_, err = f.MonClient.Alertmanagers(ns).Create(a)
	if err != nil {
		return errors.Wrap(err, fmt.Sprintf("creating alertmanager %v failed", a.Name))
	}

	return f.WaitForAlertmanagerReady(ns, a.Name, int(*a.Spec.Replicas))
}

func (f *Framework) WaitForAlertmanagerReady(ns, name string, replicas int) error {
	err := WaitForPodsReady(
		f.KubeClient,
		ns,
		5*time.Minute,
		replicas,
		alertmanager.ListOptions(name),
	)

	return errors.Wrap(err, fmt.Sprintf("failed to create an Alertmanager cluster (%s) with %d instances", name, replicas))
}

func (f *Framework) UpdateAlertmanagerAndWaitUntilReady(ns string, a *monitoringv1.Alertmanager) error {
	_, err := f.MonClient.Alertmanagers(ns).Update(a)
	if err != nil {
		return err
	}

	err = WaitForPodsReady(
		f.KubeClient,
		ns,
		5*time.Minute,
		int(*a.Spec.Replicas),
		alertmanager.ListOptions(a.Name),
	)
	if err != nil {
		return fmt.Errorf("failed to update %d Alertmanager instances (%s): %v", a.Spec.Replicas, a.Name, err)
	}

	return nil
}

func (f *Framework) DeleteAlertmanagerAndWaitUntilGone(ns, name string) error {
	_, err := f.MonClient.Alertmanagers(ns).Get(name, metav1.GetOptions{})
	if err != nil {
		return errors.Wrap(err, fmt.Sprintf("requesting Alertmanager tpr %v failed", name))
	}

	if err := f.MonClient.Alertmanagers(ns).Delete(name, nil); err != nil {
		return errors.Wrap(err, fmt.Sprintf("deleting Alertmanager tpr %v failed", name))
	}

	if err := WaitForPodsReady(
		f.KubeClient,
		ns,
		f.DefaultTimeout,
		0,
		alertmanager.ListOptions(name),
	); err != nil {
		return errors.Wrap(err, fmt.Sprintf("waiting for Alertmanager tpr (%s) to vanish timed out", name))
	}

	return f.KubeClient.CoreV1().Secrets(ns).Delete(fmt.Sprintf("alertmanager-%s", name), nil)
}

func amImage(version string) string {
	return fmt.Sprintf("quay.io/prometheus/alertmanager:%s", version)
}

func (f *Framework) WaitForAlertmanagerInitializedMesh(ns, name string, amountPeers int) error {
	return wait.Poll(time.Second, time.Second*20, func() (bool, error) {
		amStatus, err := f.GetAlertmanagerConfig(ns, name)
		if err != nil {
			return false, err
		}
		if len(amStatus.Data.MeshStatus.Peers) == amountPeers {
			return true, nil
		}

		return false, nil
	})
}

func (f *Framework) GetAlertmanagerConfig(ns, n string) (alertmanagerStatus, error) {
	var amStatus alertmanagerStatus
	request := ProxyGetPod(f.KubeClient, ns, n, "9093", "/api/v1/status")
	resp, err := request.DoRaw()
	if err != nil {
		return amStatus, err
	}

	if err := json.Unmarshal(resp, &amStatus); err != nil {
		return amStatus, err
	}

	return amStatus, nil
}

func (f *Framework) WaitForSpecificAlertmanagerConfig(ns, amName string, expectedConfig string) error {
	return wait.Poll(10*time.Second, time.Minute*5, func() (bool, error) {
		config, err := f.GetAlertmanagerConfig(ns, "alertmanager-"+amName+"-0")
		if err != nil {
			return false, err
		}

		if config.Data.ConfigYAML == expectedConfig {
			return true, nil
		}

		log.Printf("\n\nFound:\n\n%#+v\n\nExpected:\n\n%#+v\n\n", config.Data.ConfigYAML, expectedConfig)

		return false, nil
	})
}

type alertmanagerStatus struct {
	Data alertmanagerStatusData `json:"data"`
}

type alertmanagerStatusData struct {
	MeshStatus meshStatus `json:"meshStatus"`
	ConfigYAML string     `json:"configYAML"`
}

type meshStatus struct {
	Peers []interface{} `json:"peers"`
}
<|endoftext|>


<commit_before>package file

import (
	"os"

	"github.com/antham/doc-hunt/util"
)

//go:generate stringer -type=ItemStatus

// ItemStatus represents a source file status
type ItemStatus int

// Source files status
const (
	INONE ItemStatus = iota
	IFAILED
	IADDED
	IUPDATED
	IDELETED
)

var statusCache map[string]ItemStatus

// BuildStatus retrieves sources file status
func BuildStatus() (*[]Result, error) {
	results := []Result{}

	configs, err := ListConfig()

	if err != nil {
		return nil, err
	}

	for _, config := range *configs {
		for _, source := range config.Sources {
			result := Result{
				Status: map[ItemStatus][]string{},
			}

			result.Doc = config.Doc
			result.Source = source

			switch source.Category {
			case SFILE:
				status, err := retrieveFileItem(&source)

				if err != nil {
					return nil, err
				}

				result.Status[status] = append(result.Status[status], source.Identifier)
			case SFOLDER:
				folderItems, err := retrieveFolderItems(&source)

				if err != nil {
					return nil, err
				}

				for identifier, status := range *folderItems {
					result.Status[status] = append(result.Status[status], identifier)
				}
			}

			results = append(results, result)
		}
	}

	return &results, nil
}

func retrieveFolderItems(source *Source) (*map[string]ItemStatus, error) {
	items := map[string]ItemStatus{}
	dbItems, err := getItems(source)

	if err != nil {
		return nil, err
	}

	for _, item := range *dbItems {
		items[item.Identifier] = getFileStatus(item.Identifier, item.Fingerprint)
	}

	files, err := util.ExtractFolderFiles(source.Identifier)

	switch err.(type) {
	case nil:
	case *os.PathError:
	default:
		return &items, err
	}

	for _, file := range *files {
		if _, ok := items[file]; !ok {
			items[file] = IADDED
		}
	}

	return &items, nil
}

func retrieveFileItem(source *Source) (ItemStatus, error) {
	items, err := getItems(source)

	if err != nil {
		return INONE, err
	}

	return getFileStatus((*items)[0].Identifier, (*items)[0].Fingerprint), nil
}

func getFileStatus(path string, origFingerprint string) ItemStatus {
	filename := util.GetAbsPath(path)

	_, err := os.Stat(filename)

	if err != nil {
		return IDELETED
	}

	fingerprint, err := calculateFingerprint(path)

	if err != nil {
		return IFAILED
	}

	if hasChanged(fingerprint, origFingerprint) {
		return IUPDATED
	}

	return INONE
}
<commit_msg>fix(file/status) : remove useless stmt<commit_after>package file

import (
	"os"

	"github.com/antham/doc-hunt/util"
)

//go:generate stringer -type=ItemStatus

// ItemStatus represents a source file status
type ItemStatus int

// Source files status
const (
	INONE ItemStatus = iota
	IFAILED
	IADDED
	IUPDATED
	IDELETED
)

// BuildStatus retrieves sources file status
func BuildStatus() (*[]Result, error) {
	results := []Result{}

	configs, err := ListConfig()

	if err != nil {
		return nil, err
	}

	for _, config := range *configs {
		for _, source := range config.Sources {
			result := Result{
				Status: map[ItemStatus][]string{},
			}

			result.Doc = config.Doc
			result.Source = source

			switch source.Category {
			case SFILE:
				status, err := retrieveFileItem(&source)

				if err != nil {
					return nil, err
				}

				result.Status[status] = append(result.Status[status], source.Identifier)
			case SFOLDER:
				folderItems, err := retrieveFolderItems(&source)

				if err != nil {
					return nil, err
				}

				for identifier, status := range *folderItems {
					result.Status[status] = append(result.Status[status], identifier)
				}
			}

			results = append(results, result)
		}
	}

	return &results, nil
}

func retrieveFolderItems(source *Source) (*map[string]ItemStatus, error) {
	items := map[string]ItemStatus{}
	dbItems, err := getItems(source)

	if err != nil {
		return nil, err
	}

	for _, item := range *dbItems {
		items[item.Identifier] = getFileStatus(item.Identifier, item.Fingerprint)
	}

	files, err := util.ExtractFolderFiles(source.Identifier)

	switch err.(type) {
	case nil:
	case *os.PathError:
	default:
		return &items, err
	}

	for _, file := range *files {
		if _, ok := items[file]; !ok {
			items[file] = IADDED
		}
	}

	return &items, nil
}

func retrieveFileItem(source *Source) (ItemStatus, error) {
	items, err := getItems(source)

	if err != nil {
		return INONE, err
	}

	return getFileStatus((*items)[0].Identifier, (*items)[0].Fingerprint), nil
}

func getFileStatus(path string, origFingerprint string) ItemStatus {
	filename := util.GetAbsPath(path)

	_, err := os.Stat(filename)

	if err != nil {
		return IDELETED
	}

	fingerprint, err := calculateFingerprint(path)

	if err != nil {
		return IFAILED
	}

	if hasChanged(fingerprint, origFingerprint) {
		return IUPDATED
	}

	return INONE
}
<|endoftext|>


<commit_before>package ftime

// A fast timing counter accessing the cheapest fastest
// counter your cpu can provide. Guaranteed to increase monotonically
// across successive calls on the same CPU core.
// On AMD64 CPUs we use the RDTSC instruction
func Counter() (count int64)

// Indicates whether the results returned from a call to Counter()
// increase at a uniform rate, independent of the actual clock speed
// of the CPU it is running on.
// On AMD64 CPUs we test for the 'Invariant TSC' property using CPUID
func IsCounterSteady() bool {
	_, _, ecx, _ := cpuid(0X80000007)
	return ecx&(1<<8) != 0
}

// Indicates whether the results returned from a call to Counter()
// is guaranteed to be monotonically increasing per CPU and across
// multiple CPUs on the same socket. No guarantee is made across CPU
// sockets
// On AMD64 CPUs we test for the 'Invariant TSC' property using CPUID
func IsCounterSMPMonotonic() bool {
	_, _, ecx, _ := cpuid(0X80000007)
	return ecx&(1<<8) != 0
}

func cpuid(eaxi uint32) (eax, ebx, ecx, edx uint32)
<commit_msg>Now identifying TSC_Invariant in the edx register<commit_after>package ftime

// A fast timing counter accessing the cheapest fastest
// counter your cpu can provide. Guaranteed to increase monotonically
// across successive calls on the same CPU core.
// On AMD64 CPUs we use the RDTSC instruction
func Counter() (count int64)

// Indicates whether the results returned from a call to Counter()
// increase at a uniform rate, independent of the actual clock speed
// of the CPU it is running on.
// On AMD64 CPUs we test for the 'Invariant TSC' property using CPUID
func IsCounterSteady() bool {
	_, _, _, edx := cpuid(0X80000007)
	return edx&(1<<8) != 0
}

// Indicates whether the results returned from a call to Counter()
// is guaranteed to be monotonically increasing per CPU and across
// multiple CPUs on the same socket. No guarantee is made across CPU
// sockets
// On AMD64 CPUs we test for the 'Invariant TSC' property using CPUID
func IsCounterSMPMonotonic() bool {
	_, _, _, edx := cpuid(0X80000007)
	return edx&(1<<8) != 0
}

func cpuid(eaxi uint32) (eax, ebx, ecx, edx uint32)
<|endoftext|>


<commit_before>// 29 june 2015
package main

import (
	"fmt"
	"os"
	"io"
	"path/filepath"
	"github.com/andlabs/nointrochk/clrmamepro"
)

// fields in dat file
const (
	fGame = "game"
	fROM = "rom"
	fFilename = "name"
	fSize = "size"
	fCRC32 = "crc"
	fMD5 = "md5"
	fSHA1 = "sha1"
)

var nroms, ngood, nbad, nmiss uint

func checksumsPass(romname string, rom *clrmamepro.Block, f io.Reader) bool {
	// TODO
	return true
}

func check(b *clrmamepro.Block, folder string) {
	if b.Name != fGame {
		return
	}
	nroms++
	rom := b.Blocks[fROM]
	if rom == nil {
		die("checking ROM: game block without rom block: %#v", b)
	}
	romname := rom.Texts[fFilename]
	if !exists(romname) {
		alert("MISSING", romname)
		nmiss++
		return
	}
	defer markProcessed(romname)

	filename := filepath.Join(folder, romname)
	f, err := os.Open(filename)
	if err != nil {
		die("opening ROM %q to check it: %v", filename, err)
	}
	defer f.Close()

	// first check size
	s, err := f.Stat()
	if err != nil {
		die("getting ROM %q stats to check size: %v", romname, err)
	}
	if fmt.Sprint(s.Size()) != rom.Texts[fSize] {
		alert("BAD SIZE", romname)
		nbad++
		return
	}

	// now check checksums
	if !checksumsPass(romname, rom, f) {
		nbad++
		return
	}

	// otherwise it all checked out
	alert("GOOD", romname)
	ngood++
}
<commit_msg>Added the actual checksumming.<commit_after>// 29 june 2015
package main

import (
	"fmt"
	"os"
	"io"
	"path/filepath"
	"hash"
	"hash/crc32"
	"crypto/md5"
	"crypto/sha1"
	"strings"
	"github.com/andlabs/nointrochk/clrmamepro"
)

// fields in dat file
const (
	fGame = "game"
	fROM = "rom"
	fFilename = "name"
	fSize = "size"
	fCRC32 = "crc"
	fMD5 = "md5"
	fSHA1 = "sha1"
)

var nroms, ngood, nbad, nmiss uint

func checksumsPass(romname string, rom *clrmamepro.Block, f io.Reader) bool {
	crc := crc32.NewIEEE()
	md := md5.New()
	sha := sha1.New()
	checksums := io.MultiWriter(crc, md, sha)

	_, err := io.Copy(checksums, f)
	if err != nil {
		die("checksumming ROM %q: %v", romname, err)
	}

	compare := func(h hash.Hash, expected string) bool {
		return strings.ToUpper(fmt.Sprintf("%x", h.Sum(nil))) == strings.ToUpper(expected)
	}
	if !compare(crc, rom.Texts[fCRC32]) {
		alert("BAD CRC32", romname)
		return false
	}
	if !compare(md, rom.Texts[fMD5]) {
		alert("BAD MD5", romname)
		return false
	}
	if !compare(sha, rom.Texts[fSHA1]) {
		alert("BAD SHA1", romname)
		return false
	}

	return true
}

func check(b *clrmamepro.Block, folder string) {
	if b.Name != fGame {
		return
	}
	nroms++
	rom := b.Blocks[fROM]
	if rom == nil {
		die("checking ROM: game block without rom block: %#v", b)
	}
	romname := rom.Texts[fFilename]
	if !exists(romname) {
		alert("MISSING", romname)
		nmiss++
		return
	}
	defer markProcessed(romname)

	filename := filepath.Join(folder, romname)
	f, err := os.Open(filename)
	if err != nil {
		die("opening ROM %q to check it: %v", filename, err)
	}
	defer f.Close()

	// first check size
	s, err := f.Stat()
	if err != nil {
		die("getting ROM %q stats to check size: %v", romname, err)
	}
	if fmt.Sprint(s.Size()) != rom.Texts[fSize] {
		alert("BAD SIZE", romname)
		nbad++
		return
	}

	// now check checksums
	if !checksumsPass(romname, rom, f) {
		nbad++
		return
	}

	// otherwise it all checked out
	alert("GOOD", romname)
	ngood++
}
<|endoftext|>


<commit_before>package log

import (
	"io"

	"github.com/inconshreveable/log15"
)

var _ io.Writer = Writer{}

type Writer struct {
	Log   log15.Logger
	Level log15.Lvl
	Msg   string
}

func (lw Writer) Write(msg []byte) (int, error) {
	// goddamnit, expose your fucking `write` method
	switch lw.Level {
	case log15.LvlDebug:
		lw.Log.Debug(lw.Msg, "chunk", string(msg))
	case log15.LvlInfo:
		lw.Log.Info(lw.Msg, "chunk", string(msg))
	case log15.LvlWarn:
		lw.Log.Warn(lw.Msg, "chunk", string(msg))
	case log15.LvlError:
		lw.Log.Error(lw.Msg, "chunk", string(msg))
	case log15.LvlCrit:
		lw.Log.Crit(lw.Msg, "chunk", string(msg))
	}
	return len(msg), nil
}
<commit_msg>Add log handler for print methods.<commit_after>package log

import (
	"io"

	"github.com/inconshreveable/log15"
)

var _ io.Writer = Writer{}

/*
	Exposes a logger as an `io.Writer`, so you can have any system output tagged logs.
*/
type Writer struct {
	Log   log15.Logger
	Level log15.Lvl
	Msg   string
}

func (lw Writer) Write(msg []byte) (int, error) {
	// goddamnit, expose your fucking `write` method
	switch lw.Level {
	case log15.LvlDebug:
		lw.Log.Debug(lw.Msg, "chunk", string(msg))
	case log15.LvlInfo:
		lw.Log.Info(lw.Msg, "chunk", string(msg))
	case log15.LvlWarn:
		lw.Log.Warn(lw.Msg, "chunk", string(msg))
	case log15.LvlError:
		lw.Log.Error(lw.Msg, "chunk", string(msg))
	case log15.LvlCrit:
		lw.Log.Crit(lw.Msg, "chunk", string(msg))
	}
	return len(msg), nil
}

type Printer func(a ...interface{}) (n int, err error)

/*
	Dumps logs to a printer.  Any regular `Printer` will do -- `fmt.Print`
	qualifies.
*/
func PrinterHandler(printer Printer, fmtr log15.Format) log15.Handler {
	return log15.FuncHandler(func(r *log15.Record) error {
		printer(string(fmtr.Format(r)))
		return nil
	})
}
<|endoftext|>


<commit_before>package crypto

import (
	"github.com/dedis/crypto/abstract"
	"github.com/lbarman/prifi/prifi-lib/config"
	"testing"

	"fmt"
	"github.com/dedis/crypto/random"
	"strconv"
)

func TestSchnorr(t *testing.T) {

	pub, priv := NewKeyPair()
	pub2, priv2 := NewKeyPair()

	//with empty data
	data := make([]byte, 0)
	sig := SchnorrSign(config.CryptoSuite, random.Stream, data, priv)
	err := SchnorrVerify(config.CryptoSuite, data, pub, sig)

	if err != nil {
		t.Error("Should validate with nil message, err is " + err.Error())
	}

	//with empty data
	data = random.Bits(100, false, random.Stream)
	sig = SchnorrSign(config.CryptoSuite, random.Stream, data, priv)
	err = SchnorrVerify(config.CryptoSuite, data, pub, sig)

	if err != nil {
		t.Error("Should validate with random message, err is " + err.Error())
	}

	//should trivially not validate with other keys
	data = random.Bits(100, false, random.Stream)
	sig = SchnorrSign(config.CryptoSuite, random.Stream, data, priv2)
	err = SchnorrVerify(config.CryptoSuite, data, pub, sig)

	if err == nil {
		t.Error("Should not validate with wrong keys")
	}
	data = random.Bits(100, false, random.Stream)
	sig = SchnorrSign(config.CryptoSuite, random.Stream, data, priv)
	err = SchnorrVerify(config.CryptoSuite, data, pub2, sig)

	if err == nil {
		t.Error("Should not validate with wrong keys")
	}

}

func TestSchnorrHash(t *testing.T) {

	pub, _ := NewKeyPair()
	data := random.Bits(100, false, random.Stream)
	secret := hashSchnorr(config.CryptoSuite, data, pub)

	if secret == nil {
		t.Error("Secret should not be nil")
	}
}

func TestNeffShuffle(t *testing.T) {

	//output distribution testing.
	nClientsRange := []int{2, 3, 4, 5, 10}
	repetition := 100
	base := config.CryptoSuite.Point().Base()
	maxUnfairness := 30 //30% of difference between the shuffle's homogeneity

	for _, nClients := range nClientsRange {
		fmt.Println("Testing shuffle for ", nClients, " clients.")

		//build the client's public key array
		clientPks := make([]abstract.Point, nClients)
		clientPrivKeys := make([]abstract.Scalar, nClients)
		for i := 0; i < nClients; i++ {
			pub, priv := NewKeyPair()
			clientPks[i] = pub
			clientPrivKeys[i] = priv
		}

		//shuffle
		shuffledKeys, newBase, secretCoeff, proof, err := NeffShuffle(clientPks, base, config.CryptoSuite, true)

		if err != nil {
			t.Error(err)
		}
		if shuffledKeys == nil {
			t.Error("ShuffledKeys is nil")
		}
		if newBase == nil {
			t.Error("newBase is nil")
		}
		if secretCoeff == nil {
			t.Error("secretCoeff is nil")
		}
		if proof == nil {
			t.Error("proof is nil")
		}

		//now test that the shuffled keys are indeed the old keys in the new base
		transformedKeys := make([]abstract.Point, nClients)
		for i := 0; i < nClients; i++ {
			transformedKeys[i] = config.CryptoSuite.Point().Mul(newBase, clientPrivKeys[i])
		}

		//for every key, check that it exists in the remaining array
		for _, v := range transformedKeys {
			found := false
			for i := 0; i < nClients; i++ {
				if shuffledKeys[i].Equal(v) {
					found = true
				}
			}

			if !found {
				t.Error("Public key not found in outbound array")
			}
		}

		//test the distribution
		mappingDistrib := make([][]float64, nClients)
		for k := 0; k < nClients; k++ {
			mappingDistrib[k] = make([]float64, nClients)
		}
		fmt.Print("Testing distribution for ", nClients, " clients.")
		for i := 0; i < repetition; i++ {
			shuffledKeys, newBase, secretCoeff, proof, err = NeffShuffle(clientPks, base, config.CryptoSuite, true)

			mapping := make([]int, nClients)
			transformedKeys := make([]abstract.Point, nClients)
			for i := 0; i < nClients; i++ {
				transformedKeys[i] = config.CryptoSuite.Point().Mul(newBase, clientPrivKeys[i])
			}
			for k, v := range transformedKeys {
				for i := 0; i < nClients; i++ {
					if shuffledKeys[i].Equal(v) {
						mapping[k] = i
					}
				}
			}

			for clientID, slot := range mapping {
				mappingDistrib[clientID][slot] += float64(1)
			}
		}
		maxDeviation := float64(-1)
		for clientID, _ := range mappingDistrib {
			for slot := 0; slot < nClients; slot++ {
				//compute deviation
				expectedValue := float64(100) / float64(len(mappingDistrib[clientID]))
				mappingDistrib[clientID][slot] -= expectedValue
				if mappingDistrib[clientID][slot] < 0 {
					mappingDistrib[clientID][slot] = -mappingDistrib[clientID][slot]
				}

				//store max deviation
				if mappingDistrib[clientID][slot] > maxDeviation {
					maxDeviation = mappingDistrib[clientID][slot]
				}
			}
		}
		fmt.Printf("+-%d%%\n", int(maxDeviation))
		if int(maxDeviation) > maxUnfairness {
			t.Error("Max allowed distribution biais is " + strconv.Itoa(maxUnfairness) + " percent.")
		}

	}

}
<commit_msg>Improves coverage of prifi-lib/crypto<commit_after>package crypto

import (
	"github.com/dedis/crypto/abstract"
	"github.com/lbarman/prifi/prifi-lib/config"
	"testing"

	"fmt"
	"github.com/dedis/crypto/random"
	"strconv"
)

func TestSchnorr(t *testing.T) {

	pub, priv := NewKeyPair()
	pub2, priv2 := NewKeyPair()

	//with empty data
	data := make([]byte, 0)
	sig := SchnorrSign(config.CryptoSuite, random.Stream, data, priv)
	err := SchnorrVerify(config.CryptoSuite, data, pub, sig)

	if err != nil {
		t.Error("Should validate with nil message, err is " + err.Error())
	}

	//with empty data
	data = random.Bits(100, false, random.Stream)
	sig = SchnorrSign(config.CryptoSuite, random.Stream, data, priv)
	err = SchnorrVerify(config.CryptoSuite, data, pub, sig)

	if err != nil {
		t.Error("Should validate with random message, err is " + err.Error())
	}

	//should trivially not validate with other keys
	data = random.Bits(100, false, random.Stream)
	sig = SchnorrSign(config.CryptoSuite, random.Stream, data, priv2)
	err = SchnorrVerify(config.CryptoSuite, data, pub, sig)

	if err == nil {
		t.Error("Should not validate with wrong keys")
	}
	data = random.Bits(100, false, random.Stream)
	sig = SchnorrSign(config.CryptoSuite, random.Stream, data, priv)
	err = SchnorrVerify(config.CryptoSuite, data, pub2, sig)

	if err == nil {
		t.Error("Should not validate with wrong keys")
	}

}

func TestNeffErrors(t *testing.T) {

	nClients := 2
	base := config.CryptoSuite.Point().Base()

	//build the client's public key array
	clientPks := make([]abstract.Point, nClients)
	clientPrivKeys := make([]abstract.Scalar, nClients)
	for i := 0; i < nClients; i++ {
		pub, priv := NewKeyPair()
		clientPks[i] = pub
		clientPrivKeys[i] = priv
	}

	//each of those call should fail
	_, _, _, _, err := NeffShuffle(nil, base, config.CryptoSuite, true)
	if err == nil {
		t.Error("NeffShuffle without a public key array should fail")
	}
	_, _, _, _, err = NeffShuffle(clientPks, nil, config.CryptoSuite, true)
	if err == nil {
		t.Error("NeffShuffle without a base should fail")
	}
	_, _, _, _, err = NeffShuffle(clientPks, base, nil, true)
	if err == nil {
		t.Error("NeffShuffle without a suite should fail")
	}
	_, _, _, _, err = NeffShuffle(make([]abstract.Point, 0), base, config.CryptoSuite, true)
	if err == nil {
		t.Error("NeffShuffle with 0 public keys should fail")
	}

}

func TestSchnorrHash(t *testing.T) {

	pub, _ := NewKeyPair()
	data := random.Bits(100, false, random.Stream)
	secret := hashSchnorr(config.CryptoSuite, data, pub)

	if secret == nil {
		t.Error("Secret should not be nil")
	}
}

func TestNeffShuffle(t *testing.T) {

	//output distribution testing.
	nClientsRange := []int{2, 3, 4, 5, 10}
	repetition := 100
	base := config.CryptoSuite.Point().Base()
	maxUnfairness := 30 //30% of difference between the shuffle's homogeneity

	for _, nClients := range nClientsRange {
		fmt.Println("Testing shuffle for ", nClients, " clients.")

		//build the client's public key array
		clientPks := make([]abstract.Point, nClients)
		clientPrivKeys := make([]abstract.Scalar, nClients)
		for i := 0; i < nClients; i++ {
			pub, priv := NewKeyPair()
			clientPks[i] = pub
			clientPrivKeys[i] = priv
		}

		//shuffle
		shuffledKeys, newBase, secretCoeff, proof, err := NeffShuffle(clientPks, base, config.CryptoSuite, true)

		if err != nil {
			t.Error(err)
		}
		if shuffledKeys == nil {
			t.Error("ShuffledKeys is nil")
		}
		if newBase == nil {
			t.Error("newBase is nil")
		}
		if secretCoeff == nil {
			t.Error("secretCoeff is nil")
		}
		if proof == nil {
			t.Error("proof is nil")
		}

		//now test that the shuffled keys are indeed the old keys in the new base
		transformedKeys := make([]abstract.Point, nClients)
		for i := 0; i < nClients; i++ {
			transformedKeys[i] = config.CryptoSuite.Point().Mul(newBase, clientPrivKeys[i])
		}

		//for every key, check that it exists in the remaining array
		for _, v := range transformedKeys {
			found := false
			for i := 0; i < nClients; i++ {
				if shuffledKeys[i].Equal(v) {
					found = true
				}
			}

			if !found {
				t.Error("Public key not found in outbound array")
			}
		}

		//test the distribution
		mappingDistrib := make([][]float64, nClients)
		for k := 0; k < nClients; k++ {
			mappingDistrib[k] = make([]float64, nClients)
		}
		fmt.Print("Testing distribution for ", nClients, " clients.")
		for i := 0; i < repetition; i++ {
			shuffledKeys, newBase, secretCoeff, proof, err = NeffShuffle(clientPks, base, config.CryptoSuite, true)

			mapping := make([]int, nClients)
			transformedKeys := make([]abstract.Point, nClients)
			for i := 0; i < nClients; i++ {
				transformedKeys[i] = config.CryptoSuite.Point().Mul(newBase, clientPrivKeys[i])
			}
			for k, v := range transformedKeys {
				for i := 0; i < nClients; i++ {
					if shuffledKeys[i].Equal(v) {
						mapping[k] = i
					}
				}
			}

			for clientID, slot := range mapping {
				mappingDistrib[clientID][slot] += float64(1)
			}
		}
		maxDeviation := float64(-1)
		for clientID, _ := range mappingDistrib {
			for slot := 0; slot < nClients; slot++ {
				//compute deviation
				expectedValue := float64(100) / float64(len(mappingDistrib[clientID]))
				mappingDistrib[clientID][slot] -= expectedValue
				if mappingDistrib[clientID][slot] < 0 {
					mappingDistrib[clientID][slot] = -mappingDistrib[clientID][slot]
				}

				//store max deviation
				if mappingDistrib[clientID][slot] > maxDeviation {
					maxDeviation = mappingDistrib[clientID][slot]
				}
			}
		}
		fmt.Printf("+-%d%%\n", int(maxDeviation))
		if int(maxDeviation) > maxUnfairness {
			t.Error("Max allowed distribution biais is " + strconv.Itoa(maxUnfairness) + " percent.")
		}

	}

}
<|endoftext|>


<commit_before>package nzb

import (
    "sync"
)

// Chunk of data to download
type Chunk struct {
    Groups []string
    Segment *Segment
}

// Chunks list of all chunks within NZB
type Chunks struct {
    c       []*Chunk
    mu      *sync.Mutex
    Total   int
    Marker  int
}

func (c *Chunks) addChunks(groups []string, segments []*Segment) {
    for _, s := range segments {
        cnk := &Chunk {
            Groups: groups,
            Segment: s,
        }

        c.c = append(c.c, cnk)
    }
}

// GetChunks limited by max
func (c *Chunks) GetChunks(max int) []*Chunk {
    c.mu.Lock()
    defer c.mu.Unlock()

    chunks := make([]*Chunk, max)

    if c.Marker < len(c.c) {
        for i := 0; i < max; i++ {
            chunks[i] = c.c[c.Marker]
            c.Marker++
        }
    }

    return chunks
}

// GetNext get next chunk from Chunks
func (c *Chunks) GetNext() *Chunk {
    c.mu.Lock()
    defer c.mu.Unlock()

    if c.Marker < len(c.c) {
        cnk := c.c[c.Marker]
        c.Marker++
        return cnk
    }

    return nil
}

// EOF
<commit_msg>added Remove(id string) to Chunks, allows to remove previous downloaded chunk based on article id<commit_after>package nzb

import (
    "fmt"
    "sync"
)

// Chunk of data to download
type Chunk struct {
    Groups []string
    Segment *Segment
}

// Chunks list of all chunks within NZB
type Chunks struct {
    c       []*Chunk
    mu      *sync.Mutex
    Total   int
    Marker  int
}

func (c *Chunks) addChunks(groups []string, segments []*Segment) {
    for _, s := range segments {
        cnk := &Chunk {
            Groups: groups,
            Segment: s,
        }

        c.c = append(c.c, cnk)
    }
}

// GetChunks limited by max
func (c *Chunks) GetChunks(max int) []*Chunk {
    c.mu.Lock()
    defer c.mu.Unlock()

    chunks := make([]*Chunk, max)

    if c.Marker < len(c.c) {
        for i := 0; i < max; i++ {
            chunks[i] = c.c[c.Marker]
            c.Marker++
        }
    }

    return chunks
}

// GetNext get next chunk from Chunks
func (c *Chunks) GetNext() *Chunk {
    c.mu.Lock()
    defer c.mu.Unlock()

    if c.Marker < len(c.c) {
        cnk := c.c[c.Marker]
        c.Marker++
        return cnk
    }

    return nil
}

// Remove chunk from chunks based on article id
// This allows to re-continue a download
// and remove already downloaded chunks from the chunk list
func (c *Chunks) Remove(id string) error {

    c.mu.Lock()
    defer c.mu.Unlock()

    rm := -1
    for i, cnk := range c.c {
        if cnk.Segment.ID == id {
            // Found chunk to remove
            rm = i
        }
    }

    // Remove element
    if rm > -1 {
        c.c = append(c.c[:rm], c.c[rm+1:]...)
        // Update Total
        c.Total = len(c.c)

        return nil
    }

    return fmt.Errorf("id does not exists")
}

// EOF
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/LK4D4/vndr/build"
	"github.com/LK4D4/vndr/godl"
)

func isCDir(fis []os.FileInfo) bool {
	var hFound bool
	for _, fi := range fis {
		ext := filepath.Ext(fi.Name())
		if ext == ".cc" || ext == ".cpp" || ext == ".py" {
			return false
		}
		if ext == ".h" {
			hFound = true
		}
	}
	return hFound
}

func isPBDir(fis []os.FileInfo) bool {
	var pbFound bool
	for _, fi := range fis {
		if fi.IsDir() {
			continue
		}
		ext := filepath.Ext(fi.Name())
		if ext != ".proto" {
			return false
		}
		pbFound = true
	}
	return pbFound
}

func isInterestingDir(path string) bool {
	fis, err := ioutil.ReadDir(path)
	if err != nil {
		return false
	}
	return isCDir(fis) || isPBDir(fis)
}

func isGoFile(path string) bool {
	ext := filepath.Ext(path)
	return ext == ".go" || ext == ".c" || ext == ".h" || ext == ".s" || ext == ".proto"
}

var licenseFiles = map[string]bool{
	"LICENSE": true,
	"COPYING": true,
	"PATENTS": true,
	"NOTICE":  true,
}

// cleanVendor removes files from unused pacakges and non-go files
func cleanVendor(vendorDir string, realDeps []*build.Package) error {
	realPaths := make(map[string]bool)
	for _, pkg := range realDeps {
		realPaths[pkg.Dir] = true
	}
	var paths []string
	err := filepath.Walk(vendorDir, func(path string, i os.FileInfo, err error) error {
		if path == vendorDir {
			return nil
		}
		if err != nil {
			return nil
		}
		if strings.HasPrefix(i.Name(), ".") || strings.HasPrefix(i.Name(), "_") {
			return os.RemoveAll(path)
		}
		if i.IsDir() {
			if i.Name() == "testdata" {
				return os.RemoveAll(path)
			}
			if isInterestingDir(path) {
				realPaths[path] = true
				return nil
			}
			if !realPaths[path] {
				paths = append(paths, path)
			}
			return nil
		}

		// keep files for licenses
		if licenseFiles[i.Name()] {
			return nil
		}
		// remove files from non-deps, non-go files and test files
		if !realPaths[filepath.Dir(path)] || !isGoFile(path) || strings.HasSuffix(path, "_test.go") {
			return os.Remove(path)
		}
		return nil
	})
	if err != nil {
		return err
	}
	sort.Sort(sort.Reverse(sort.StringSlice(paths)))
	// iterate over paths (longest first)
	for _, p := range paths {
		// at this point we cleaned all files from unused deps dirs
		lst, err := ioutil.ReadDir(p)
		if err != nil {
			return err
		}
		// remove licenses if it's only files
		var onlyLicenses = true
		for _, fi := range lst {
			if !licenseFiles[fi.Name()] {
				onlyLicenses = false
				break
			}
		}
		if !onlyLicenses {
			continue
		}
		// remove all directories if they're not in dependency paths
		if err := os.RemoveAll(p); err != nil {
			return err
		}
	}
	return nil
}

func cleanVCS(v *godl.VCS) error {
	if err := os.RemoveAll(filepath.Join(v.Root, "."+v.Type)); err != nil {
		return err
	}
	return filepath.Walk(v.Root, func(path string, i os.FileInfo, err error) error {
		if path == vendorDir {
			return nil
		}
		if !i.IsDir() {
			return nil
		}
		name := i.Name()
		if name == "vendor" || name == "Godeps" || name == "_vendor" {
			if err := os.RemoveAll(path); err != nil {
				return err
			}
			return filepath.SkipDir
		}
		return nil
	})
}
<commit_msg>clean.go: detect more license files<commit_after>package main

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	"github.com/LK4D4/vndr/build"
	"github.com/LK4D4/vndr/godl"
)

func isCDir(fis []os.FileInfo) bool {
	var hFound bool
	for _, fi := range fis {
		ext := filepath.Ext(fi.Name())
		if ext == ".cc" || ext == ".cpp" || ext == ".py" {
			return false
		}
		if ext == ".h" {
			hFound = true
		}
	}
	return hFound
}

func isPBDir(fis []os.FileInfo) bool {
	var pbFound bool
	for _, fi := range fis {
		if fi.IsDir() {
			continue
		}
		ext := filepath.Ext(fi.Name())
		if ext != ".proto" {
			return false
		}
		pbFound = true
	}
	return pbFound
}

func isInterestingDir(path string) bool {
	fis, err := ioutil.ReadDir(path)
	if err != nil {
		return false
	}
	return isCDir(fis) || isPBDir(fis)
}

func isGoFile(path string) bool {
	ext := filepath.Ext(path)
	return ext == ".go" || ext == ".c" || ext == ".h" || ext == ".s" || ext == ".proto"
}

// licenseFilesRegexp is a regexp of file names that are likely to contain licensing information
var licenseFilesRegexp = regexp.MustCompile(`(?i).*(LICENSE|COPYING|PATENT|NOTICE|README).*`)

// cleanVendor removes files from unused pacakges and non-go files
func cleanVendor(vendorDir string, realDeps []*build.Package) error {
	realPaths := make(map[string]bool)
	for _, pkg := range realDeps {
		realPaths[pkg.Dir] = true
	}
	var paths []string
	err := filepath.Walk(vendorDir, func(path string, i os.FileInfo, err error) error {
		if path == vendorDir {
			return nil
		}
		if err != nil {
			return nil
		}
		if strings.HasPrefix(i.Name(), ".") || strings.HasPrefix(i.Name(), "_") {
			return os.RemoveAll(path)
		}
		if i.IsDir() {
			if i.Name() == "testdata" {
				return os.RemoveAll(path)
			}
			if isInterestingDir(path) {
				realPaths[path] = true
				return nil
			}
			if !realPaths[path] {
				paths = append(paths, path)
			}
			return nil
		}

		// keep files for licenses
		if licenseFilesRegexp.MatchString(i.Name()) {
			return nil
		}
		// remove files from non-deps, non-go files and test files
		if !realPaths[filepath.Dir(path)] || !isGoFile(path) || strings.HasSuffix(path, "_test.go") {
			return os.Remove(path)
		}
		return nil
	})
	if err != nil {
		return err
	}
	sort.Sort(sort.Reverse(sort.StringSlice(paths)))
	// iterate over paths (longest first)
	for _, p := range paths {
		// at this point we cleaned all files from unused deps dirs
		lst, err := ioutil.ReadDir(p)
		if err != nil {
			return err
		}
		// remove licenses if it's only files
		var onlyLicenses = true
		for _, fi := range lst {
			if !licenseFilesRegexp.MatchString(fi.Name()) {
				onlyLicenses = false
				break
			}
		}
		if !onlyLicenses {
			continue
		}
		// remove all directories if they're not in dependency paths
		if err := os.RemoveAll(p); err != nil {
			return err
		}
	}
	return nil
}

func cleanVCS(v *godl.VCS) error {
	if err := os.RemoveAll(filepath.Join(v.Root, "."+v.Type)); err != nil {
		return err
	}
	return filepath.Walk(v.Root, func(path string, i os.FileInfo, err error) error {
		if path == vendorDir {
			return nil
		}
		if !i.IsDir() {
			return nil
		}
		name := i.Name()
		if name == "vendor" || name == "Godeps" || name == "_vendor" {
			if err := os.RemoveAll(path); err != nil {
				return err
			}
			return filepath.SkipDir
		}
		return nil
	})
}
<|endoftext|>


<commit_before>// +build integration

/*
Copyright 2020 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"context"
	"os/exec"
	"strings"
	"testing"
)

func TestPause(t *testing.T) {
	MaybeParallel(t)

	type validateFunc func(context.Context, *testing.T, string)
	profile := UniqueProfileName("pause")
	ctx, cancel := context.WithTimeout(context.Background(), Minutes(30))
	defer CleanupWithLogs(t, profile, cancel)

	// Serial tests
	t.Run("serial", func(t *testing.T) {
		tests := []struct {
			name      string
			validator validateFunc
		}{
			{"Start", validateFreshStart},
			{"SecondStartNoReset", validateStartNoReset},
			{"Pause", validatePause},
			{"Unpause", validateUnpause},
			{"PauseAgain", validatePause},
			{"DeletePaused", validateDelete},
		}
		for _, tc := range tests {
			tc := tc
			t.Run(tc.name, func(t *testing.T) {
				tc.validator(ctx, t, profile)
			})
		}
	})
}

func validateFreshStart(ctx context.Context, t *testing.T, profile string) {
	args := append([]string{"start", "-p", profile, "--memory=1800", "--install-addons=false", "--wait=false"}, StartArgs()...)
	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Fatalf("failed to start minikube with args: %q : %v", rr.Command(), err)
	}
}

// validateStartNoReset validates that starting a running cluster won't invoke a reset
func validateStartNoReset(ctx context.Context, t *testing.T, profile string) {
	args := []string{"start", "-p", profile, "--alsologtostderr", "-v=5"}
	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Fatalf("failed to second start a running minikube with args: %q : %v", rr.Command(), err)
	}
	if !NoneDriver() {
		softLog := "The running cluster does not need a reset"
		if !strings.Contains(rr.Output(), softLog) {
			t.Errorf("expected the second start log outputs to include %q but got: %s", softLog, rr.Output())
		}
	}

}

func validatePause(ctx context.Context, t *testing.T, profile string) {
	args := []string{"pause", "-p", profile, "--alsologtostderr", "-v=5"}
	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Errorf("failed to pause minikube with args: %q : %v", rr.Command(), err)
	}
}

func validateUnpause(ctx context.Context, t *testing.T, profile string) {
	args := []string{"unpause", "-p", profile, "--alsologtostderr", "-v=5"}
	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Errorf("failed to unpause minikube with args: %q : %v", rr.Command(), err)
	}
}

func validateDelete(ctx context.Context, t *testing.T, profile string) {
	// vervose logging because this might go wrong, if container get stuck
	args := []string{"delete", "-p", profile, "--alsologtostderr", "-v=5"}
	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Errorf("failed to delete minikube with args: %q : %v", rr.Command(), err)
	}
}
<commit_msg>wait true<commit_after>// +build integration

/*
Copyright 2020 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"context"
	"os/exec"
	"strings"
	"testing"
)

func TestPause(t *testing.T) {
	MaybeParallel(t)

	type validateFunc func(context.Context, *testing.T, string)
	profile := UniqueProfileName("pause")
	ctx, cancel := context.WithTimeout(context.Background(), Minutes(30))
	defer CleanupWithLogs(t, profile, cancel)

	// Serial tests
	t.Run("serial", func(t *testing.T) {
		tests := []struct {
			name      string
			validator validateFunc
		}{
			{"Start", validateFreshStart},
			{"SecondStartNoReset", validateStartNoReset},
			{"Pause", validatePause},
			{"Unpause", validateUnpause},
			{"PauseAgain", validatePause},
			{"DeletePaused", validateDelete},
		}
		for _, tc := range tests {
			tc := tc
			t.Run(tc.name, func(t *testing.T) {
				tc.validator(ctx, t, profile)
			})
		}
	})
}

func validateFreshStart(ctx context.Context, t *testing.T, profile string) {
	args := append([]string{"start", "-p", profile, "--memory=1800", "--install-addons=false", "--wait=true"}, StartArgs()...)
	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Fatalf("failed to start minikube with args: %q : %v", rr.Command(), err)
	}
}

// validateStartNoReset validates that starting a running cluster won't invoke a reset
func validateStartNoReset(ctx context.Context, t *testing.T, profile string) {
	args := []string{"start", "-p", profile, "--alsologtostderr", "-v=5"}
	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Fatalf("failed to second start a running minikube with args: %q : %v", rr.Command(), err)
	}
	if !NoneDriver() {
		softLog := "The running cluster does not need a reset"
		if !strings.Contains(rr.Output(), softLog) {
			t.Errorf("expected the second start log outputs to include %q but got: %s", softLog, rr.Output())
		}
	}

}

func validatePause(ctx context.Context, t *testing.T, profile string) {
	args := []string{"pause", "-p", profile, "--alsologtostderr", "-v=5"}
	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Errorf("failed to pause minikube with args: %q : %v", rr.Command(), err)
	}
}

func validateUnpause(ctx context.Context, t *testing.T, profile string) {
	args := []string{"unpause", "-p", profile, "--alsologtostderr", "-v=5"}
	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Errorf("failed to unpause minikube with args: %q : %v", rr.Command(), err)
	}
}

func validateDelete(ctx context.Context, t *testing.T, profile string) {
	// vervose logging because this might go wrong, if container get stuck
	args := []string{"delete", "-p", profile, "--alsologtostderr", "-v=5"}
	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
	if err != nil {
		t.Errorf("failed to delete minikube with args: %q : %v", rr.Command(), err)
	}
}
<|endoftext|>