<commit_before>package sparse

import "net"
import "github.com/kp6/alphorn/log"
import "encoding/gob"
import "os"
import "errors"
import "strconv"

// TCPEndPoint tcp connection address
type TCPEndPoint struct {
	Host string
	Port int16
}

const connectionRetries = 5

// SyncFile synchronizes local file to remote host
func SyncFile(localPath string, addr TCPEndPoint, remotePath string) error {
	file, err := os.Open(localPath)
	if err != nil {
		log.Error("Failed to open local source file:", localPath)
		return err
	}
	defer file.Close()

	size, errSize := file.Seek(0, os.SEEK_END)
	if errSize != nil {
		log.Error("Failed to get size of local source file:", localPath, errSize)
		return err
	}

	conn := connect(addr.Host, strconv.Itoa(int(addr.Port)))
	if nil == conn {
		log.Error("Failed to connect to", addr)
		return err
	}
	defer conn.Close()

	encoder := gob.NewEncoder(conn)
	decoder := gob.NewDecoder(conn)
	status := sendSyncRequest(encoder, decoder, remotePath, size)
	if !status {
		return err
	}

	localLayout, err := getLocalFileLayout(file)
	if err != nil {
		log.Error("Failed to retrieve local file layout:", err)
		return err
	}
	remoteLayout, err := getRemoteFileLayout(decoder)
	if err != nil {
		log.Error("Failed to retrieve remote file layout:", err)
		return err
	}
	return processDiff(encoder, decoder, localLayout, remoteLayout, file)
}

func connect(host, port string) net.Conn {
	// connect to this socket
	endpoint := host + ":" + port
	for retries := 1; retries <= connectionRetries; retries++ {
		conn, err := net.Dial("tcp", endpoint)
		if err == nil {
			return conn
		}
		log.Warn("Failed connection to", endpoint, "Retrying...")
	}
	return nil
}

func sendSyncRequest(encoder *gob.Encoder, decoder *gob.Decoder, path string, size int64) bool {
	err := encoder.Encode(requestHeader{requestMagic, syncRequestCode})
	if err != nil {
		log.Error("Client protocol encoder error:", err)
		return false
	}
	err = encoder.Encode(path)
	if err != nil {
		log.Error("Client protocol encoder error:", err)
		return false
	}
	err = encoder.Encode(size)
	if err != nil {
		log.Error("Client protocol encoder error:", err)
		return false
	}

	var ack bool
	err = decoder.Decode(&ack)
	if err != nil {
		log.Error("Client protocol decoder error:", err)
		return false
	}

	return ack
}

func getLocalFileLayout(file *os.File) ([]FileInterval, error) {
	size, err := file.Seek(0, os.SEEK_END)
	if err != nil {
		log.Fatal("cannot retrieve local source file size", err)
		return nil, err
	}
	return RetrieveLayout(file, Interval{0, size})
}

func getRemoteFileLayout(decoder *gob.Decoder) ([]FileInterval, error) {
	var layout []FileInterval
	err := decoder.Decode(&layout)
	if err != nil {
		log.Fatal("Cient protocol error:", err)
		return nil, err
	}
	log.Debug("Received layout:", layout)
	return layout, nil
}

// file reading chunk
type fileChunk struct {
	eof    bool // end of stream: stop reader
	header FileInterval
}

// network transfer chunk
type diffChunk struct {
	status bool // read file or network send error yield false
	header FileInterval
	data   interface{}
}

func processDiff(encoder *gob.Encoder, decoder *gob.Decoder, local, remote []FileInterval, file *os.File) error {
	// Local:   __ _*
	// Remote:  *_ **
	const concurrentReaders = 4
	netStream := make(chan diffChunk, 128)
	netStatus := make(chan bool)
	go networkSender(netStream, encoder, netStatus)
	fileStream := make(chan fileChunk, 128)
	fileStatus := make(chan bool)
	for i := 0; i < concurrentReaders; i++ {
		if 0 == i {
			go fileReader(i, file, fileStream, netStream, fileStatus)
		} else {
			f, _ := os.Open(file.Name())
			go fileReader(i, f, fileStream, netStream, fileStatus)
		}
	}

	for i, j := 0, 0; i < len(local); {
		if j >= len(remote) {
			// Copy local tail
			processFileInterval(local[i], fileStream, netStream)
			i++
			continue
		}
		// Diff
		lrange := local[i]
		rrange := remote[j]
		if lrange.Begin == rrange.Begin {
			if lrange.End > rrange.End {
				local[i].End = rrange.End
				if SparseData == local[i].Kind || local[i].Kind != remote[j].Kind {
					processFileInterval(local[i], fileStream, netStream)
				}
				local[i].Begin = rrange.End
				local[i].End = lrange.End
				j++
				continue
			} else if lrange.End < rrange.End {
				if SparseData == local[i].Kind || local[i].Kind != remote[j].Kind {
					processFileInterval(local[i], fileStream, netStream)
				}
				remote[j].Begin = lrange.End
				i++
				continue
			}
			if SparseData == local[i].Kind || local[i].Kind != remote[j].Kind {
				processFileInterval(local[i], fileStream, netStream)
			}
			i++
			j++
		} else {
			// Should never happen
			log.Fatal("internal error")
			return errors.New("internal error")
		}
	}
	log.Info("Finished processing file diff")

	// stop file readers
	for i := 0; i < concurrentReaders; i++ {
		fileStream <- fileChunk{true, FileInterval{SparseHole, Interval{0, 0}}}
		<-fileStatus // wait for reader completion
	}
	// Send end of transmission
	netStream <- diffChunk{true, FileInterval{SparseHole, Interval{0, 0}}, nil}

	// get network sender status
	status := <-netStatus
	if !status {
		return errors.New("netwoek transfer failure")
	}

	var statusRemote bool
	err := decoder.Decode(&statusRemote)
	if err != nil {
		log.Fatal("Cient protocol remote status error:", err)
		return err
	}
	if !statusRemote {
		return errors.New("failure on remote sync site")
	}
	return nil
}

func processFileInterval(r FileInterval, fileStream chan<- fileChunk, netStream chan<- diffChunk) {
	const batch = 128 * Blocks

	// Process hole
	if SparseHole == r.Kind {
		netStream <- diffChunk{true, r, nil}
		return
	}

	// Process data in chunks
	for offset := r.Begin; offset < r.End; {
		size := batch
		if offset+size > r.End {
			size = r.End - offset
		}
		fileStream <- fileChunk{false, FileInterval{SparseData, Interval{offset, offset + size}}}
		offset += size
	}
}

func networkSender(netStream <-chan diffChunk, encoder *gob.Encoder, netStatus chan<- bool) {
	status := true
	for {
		chunk := <-netStream
		if 0 == chunk.header.Len() {
			// eof: last 0 len header
			err := encoder.Encode(chunk.header)
			if err != nil {
				log.Error("Client protocol encoder error:", err)
				status = false
			}
			break
		}

		if !status {
			// network error
			continue // discard the chunk
		}
		if !chunk.status {
			// read error
			status = false
			continue // discard the chunk
		}

		// Encode and send data to the network
		err := encoder.Encode(chunk.header)
		if err != nil {
			log.Error("Client protocol encoder error:", err)
			status = false
			continue
		}
		if nil == chunk.data {
			continue
		}
		err = encoder.Encode(chunk.data)
		if err != nil {
			log.Error("Client protocol encoder error:", err)
			status = false
			continue
		}
	}
	log.Info("Finished sending file diff, status =", status)
	netStatus <- status
}

func fileReader(id int, file *os.File, fileStream <-chan fileChunk, netStream chan<- diffChunk, fileStatus chan<- bool) {
	for {
		chunk := <-fileStream
		if chunk.eof {
			break
		}

		// Check interval type
		r := chunk.header
		if SparseData != r.Kind {
			log.Fatal("internal error: noles should be send directly to netStream")
		}

		// Read file data
		data := make([]byte, r.Len())
		status := true
		n, err := file.ReadAt(data, r.Begin)
		if err != nil {
			log.Error("File read error")
			status = false
		} else if int64(n) != r.Len() {
			log.Error("File read underrun")
			status = false
		}

		// Send file data
		netStream <- diffChunk{status, r, data}
	}
	log.Info("Finished reading file")
	fileStatus <- true
}
<commit_msg>sync: added rudimentary client side channel tracing - the tracing is controller by traceChannelLoad const<commit_after>package sparse

import "net"
import "github.com/kp6/alphorn/log"
import "encoding/gob"
import "os"
import "errors"
import "strconv"
import "fmt"

// TCPEndPoint tcp connection address
type TCPEndPoint struct {
	Host string
	Port int16
}

const connectionRetries = 5

// SyncFile synchronizes local file to remote host
func SyncFile(localPath string, addr TCPEndPoint, remotePath string) error {
	file, err := os.Open(localPath)
	if err != nil {
		log.Error("Failed to open local source file:", localPath)
		return err
	}
	defer file.Close()

	size, errSize := file.Seek(0, os.SEEK_END)
	if errSize != nil {
		log.Error("Failed to get size of local source file:", localPath, errSize)
		return err
	}

	conn := connect(addr.Host, strconv.Itoa(int(addr.Port)))
	if nil == conn {
		log.Error("Failed to connect to", addr)
		return err
	}
	defer conn.Close()

	encoder := gob.NewEncoder(conn)
	decoder := gob.NewDecoder(conn)
	status := sendSyncRequest(encoder, decoder, remotePath, size)
	if !status {
		return err
	}

	localLayout, err := getLocalFileLayout(file)
	if err != nil {
		log.Error("Failed to retrieve local file layout:", err)
		return err
	}
	remoteLayout, err := getRemoteFileLayout(decoder)
	if err != nil {
		log.Error("Failed to retrieve remote file layout:", err)
		return err
	}
	return processDiff(encoder, decoder, localLayout, remoteLayout, file)
}

func connect(host, port string) net.Conn {
	// connect to this socket
	endpoint := host + ":" + port
	for retries := 1; retries <= connectionRetries; retries++ {
		conn, err := net.Dial("tcp", endpoint)
		if err == nil {
			return conn
		}
		log.Warn("Failed connection to", endpoint, "Retrying...")
	}
	return nil
}

func sendSyncRequest(encoder *gob.Encoder, decoder *gob.Decoder, path string, size int64) bool {
	err := encoder.Encode(requestHeader{requestMagic, syncRequestCode})
	if err != nil {
		log.Error("Client protocol encoder error:", err)
		return false
	}
	err = encoder.Encode(path)
	if err != nil {
		log.Error("Client protocol encoder error:", err)
		return false
	}
	err = encoder.Encode(size)
	if err != nil {
		log.Error("Client protocol encoder error:", err)
		return false
	}

	var ack bool
	err = decoder.Decode(&ack)
	if err != nil {
		log.Error("Client protocol decoder error:", err)
		return false
	}

	return ack
}

func getLocalFileLayout(file *os.File) ([]FileInterval, error) {
	size, err := file.Seek(0, os.SEEK_END)
	if err != nil {
		log.Fatal("cannot retrieve local source file size", err)
		return nil, err
	}
	return RetrieveLayout(file, Interval{0, size})
}

func getRemoteFileLayout(decoder *gob.Decoder) ([]FileInterval, error) {
	var layout []FileInterval
	err := decoder.Decode(&layout)
	if err != nil {
		log.Fatal("Cient protocol error:", err)
		return nil, err
	}
	log.Debug("Received layout:", layout)
	return layout, nil
}

// file reading chunk
type fileChunk struct {
	eof    bool // end of stream: stop reader
	header FileInterval
}

// network transfer chunk
type diffChunk struct {
	status bool // read file or network send error yield false
	header FileInterval
	data   interface{}
}

func processDiff(encoder *gob.Encoder, decoder *gob.Decoder, local, remote []FileInterval, file *os.File) error {
	// Local:   __ _*
	// Remote:  *_ **
	const concurrentReaders = 4
	netStream := make(chan diffChunk, 128)
	netStatus := make(chan bool)
	go networkSender(netStream, encoder, netStatus)
	fileStream := make(chan fileChunk, 128)
	fileStatus := make(chan bool)
	for i := 0; i < concurrentReaders; i++ {
		if 0 == i {
			go fileReader(i, file, fileStream, netStream, fileStatus)
		} else {
			f, _ := os.Open(file.Name())
			go fileReader(i, f, fileStream, netStream, fileStatus)
		}
	}

	for i, j := 0, 0; i < len(local); {
		if j >= len(remote) {
			// Copy local tail
			processFileInterval(local[i], fileStream, netStream)
			i++
			continue
		}
		// Diff
		lrange := local[i]
		rrange := remote[j]
		if lrange.Begin == rrange.Begin {
			if lrange.End > rrange.End {
				local[i].End = rrange.End
				if SparseData == local[i].Kind || local[i].Kind != remote[j].Kind {
					processFileInterval(local[i], fileStream, netStream)
				}
				local[i].Begin = rrange.End
				local[i].End = lrange.End
				j++
				continue
			} else if lrange.End < rrange.End {
				if SparseData == local[i].Kind || local[i].Kind != remote[j].Kind {
					processFileInterval(local[i], fileStream, netStream)
				}
				remote[j].Begin = lrange.End
				i++
				continue
			}
			if SparseData == local[i].Kind || local[i].Kind != remote[j].Kind {
				processFileInterval(local[i], fileStream, netStream)
			}
			i++
			j++
		} else {
			// Should never happen
			log.Fatal("internal error")
			return errors.New("internal error")
		}
	}
	log.Info("Finished processing file diff")

	// stop file readers
	for i := 0; i < concurrentReaders; i++ {
		fileStream <- fileChunk{true, FileInterval{SparseHole, Interval{0, 0}}}
		<-fileStatus // wait for reader completion
	}
	// Send end of transmission
	netStream <- diffChunk{true, FileInterval{SparseHole, Interval{0, 0}}, nil}

	// get network sender status
	status := <-netStatus
	if !status {
		return errors.New("netwoek transfer failure")
	}

	var statusRemote bool
	err := decoder.Decode(&statusRemote)
	if err != nil {
		log.Fatal("Cient protocol remote status error:", err)
		return err
	}
	if !statusRemote {
		return errors.New("failure on remote sync site")
	}
	return nil
}

func processFileInterval(r FileInterval, fileStream chan<- fileChunk, netStream chan<- diffChunk) {
	const batch = 128 * Blocks

	// Process hole
	if SparseHole == r.Kind {
		netStream <- diffChunk{true, r, nil}
		return
	}

	// Process data in chunks
	for offset := r.Begin; offset < r.End; {
		size := batch
		if offset+size > r.End {
			size = r.End - offset
		}
		fileStream <- fileChunk{false, FileInterval{SparseData, Interval{offset, offset + size}}}
		offset += size
	}
}

// prints chan codes and lengths to trace
// - sequence and interleaving of chan processing
// - how much of the chan buffer is used
const traceChannelLoad = false

func networkSender(netStream <-chan diffChunk, encoder *gob.Encoder, netStatus chan<- bool) {
	status := true
	for {
		chunk := <-netStream
		if 0 == chunk.header.Len() {
			// eof: last 0 len header
			err := encoder.Encode(chunk.header)
			if err != nil {
				log.Error("Client protocol encoder error:", err)
				status = false
			}
			break
		}

		if !status {
			// network error
			continue // discard the chunk
		}
		if !chunk.status {
			// read error
			status = false
			continue // discard the chunk
		}

		if traceChannelLoad {
			fmt.Fprint(os.Stderr, len(netStream), "n")
		}
		// Encode and send data to the network
		err := encoder.Encode(chunk.header)
		if err != nil {
			log.Error("Client protocol encoder error:", err)
			status = false
			continue
		}
		if nil == chunk.data {
			continue
		}
		err = encoder.Encode(chunk.data)
		if err != nil {
			log.Error("Client protocol encoder error:", err)
			status = false
			continue
		}
		if traceChannelLoad {
			fmt.Fprint(os.Stderr, "N\n")
		}
	}
	log.Info("Finished sending file diff, status =", status)
	netStatus <- status
}

func fileReader(id int, file *os.File, fileStream <-chan fileChunk, netStream chan<- diffChunk, fileStatus chan<- bool) {
	idBeg := map[int]string{0: "a", 1: "b", 2: "c", 3: "d"}
	idEnd := map[int]string{0: "A", 1: "B", 2: "C", 3: "D"}
	for {
		chunk := <-fileStream
		if chunk.eof {
			break
		}
		if traceChannelLoad {
			fmt.Fprint(os.Stderr, len(fileStream), idBeg[id])
		}
		// Check interval type
		r := chunk.header
		if SparseData != r.Kind {
			log.Fatal("internal error: noles should be send directly to netStream")
		}

		// Read file data
		data := make([]byte, r.Len())
		status := true
		n, err := file.ReadAt(data, r.Begin)
		if err != nil {
			log.Error("File read error")
			status = false
		} else if int64(n) != r.Len() {
			log.Error("File read underrun")
			status = false
		}

		// Send file data
		if traceChannelLoad {
			fmt.Fprint(os.Stderr, idEnd[id])
		}
		netStream <- diffChunk{status, r, data}
	}
	log.Info("Finished reading file")
	fileStatus <- true
}
<|endoftext|>


<commit_before>package fortuna

import (
	"fmt"
	"io"
	"os"
	"testing"
)

func TestNilRNG(t *testing.T) {
	var rng *Fortuna
	if rng.Initialised() {
		fmt.Fprintf(os.Stderr, "fortuna: PRNG should not have reported it was initialised\n")
		t.FailNow()
	}
}

func TestNotSeeded(t *testing.T) {
	var p []byte
	rng := New(nil)
	if _, err := rng.Read(p); err == nil {
		fmt.Fprintf(os.Stderr, "fortuna: PRNG should report it is not seeded")
		t.FailNow()
	}
}

func TestPRNGEmptyRead(t *testing.T) {
	var p []byte
	rng := New(nil)
	rng.reseed()

	n, err := rng.Read(p)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		t.FailNow()
	} else if n != 0 {
		fmt.Fprintf(os.Stderr, "fortuna: no data should have been read\n")
		t.FailNow()
	}
}

func TestInvalidEvents(t *testing.T) {
	var p = make([]byte, 1)
	rng := New(nil)
	err := rng.AddRandomEvent(0, 33, p)
	if err != ErrInvalidEvent {
		fmt.Fprintf(os.Stderr, "fortuna: random event should be invalid\n")
		t.FailNow()
	}

	err = rng.AddRandomEvent(0, -1, p)
	if err != ErrInvalidEvent {
		fmt.Fprintf(os.Stderr, "fortuna: random event should be invalid\n")
		t.FailNow()
	}

	p = nil
	err = rng.AddRandomEvent(0, 0, p)
	if err != ErrInvalidEvent {
		fmt.Fprintf(os.Stderr, "fortuna: random event should be invalid\n")
		t.FailNow()
	}

	p = make([]byte, MaxEventSize+1)
	err = rng.AddRandomEvent(0, 0, p)
	if err != ErrInvalidEvent {
		fmt.Fprintf(os.Stderr, "fortuna: random event should be invalid\n")
		t.FailNow()
	}
}

func TestSeed(t *testing.T) {
	rng := New(nil)
	sw := NewSourceWriter(rng, 0)

	f, err := os.Open("/dev/zero")
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		t.FailNow()
	}

	io.CopyN(sw, f, 4096)
	seed, err := rng.Seed()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		t.FailNow()
	} else if len(seed) != SeedFileLength {
		fmt.Fprintf(os.Stderr, "fortuna: bad seed file length\n")
		t.FailNow()
	}

}
<commit_msg>Add more seed tests.<commit_after>package fortuna

import (
	"fmt"
	"io"
	"os"
	"testing"
)

func TestNilRNG(t *testing.T) {
	var rng *Fortuna
	if rng.Initialised() {
		fmt.Fprintf(os.Stderr, "fortuna: PRNG should not have reported it was initialised\n")
		t.FailNow()
	}
}

func TestNotSeeded(t *testing.T) {
	var p []byte
	rng := New(nil)
	if _, err := rng.Read(p); err == nil {
		fmt.Fprintf(os.Stderr, "fortuna: PRNG should report it is not seeded")
		t.FailNow()
	}
}

func TestPRNGEmptyRead(t *testing.T) {
	var p []byte
	rng := New(nil)
	rng.reseed()

	n, err := rng.Read(p)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		t.FailNow()
	} else if n != 0 {
		fmt.Fprintf(os.Stderr, "fortuna: no data should have been read\n")
		t.FailNow()
	}
}

func TestInvalidEvents(t *testing.T) {
	var p = make([]byte, 1)
	rng := New(nil)
	err := rng.AddRandomEvent(0, 33, p)
	if err != ErrInvalidEvent {
		fmt.Fprintf(os.Stderr, "fortuna: random event should be invalid\n")
		t.FailNow()
	}

	err = rng.AddRandomEvent(0, -1, p)
	if err != ErrInvalidEvent {
		fmt.Fprintf(os.Stderr, "fortuna: random event should be invalid\n")
		t.FailNow()
	}

	p = nil
	err = rng.AddRandomEvent(0, 0, p)
	if err != ErrInvalidEvent {
		fmt.Fprintf(os.Stderr, "fortuna: random event should be invalid\n")
		t.FailNow()
	}

	p = make([]byte, MaxEventSize+1)
	err = rng.AddRandomEvent(0, 0, p)
	if err != ErrInvalidEvent {
		fmt.Fprintf(os.Stderr, "fortuna: random event should be invalid\n")
		t.FailNow()
	}
}

var seed []byte

func TestSeed(t *testing.T) {
	rng := New(nil)
	sw := NewSourceWriter(rng, 0)

	_, err := rng.Seed()
	if err != ErrNotSeeded {
		fmt.Fprintf(os.Stderr, "fortuna: PRNG seed() should fail for unseeded PRNG")
		t.FailNow()
	}

	f, err := os.Open("/dev/zero")
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		t.FailNow()
	}

	io.CopyN(sw, f, 4096)
	seed, err = rng.Seed()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		t.FailNow()
	} else if len(seed) != SeedFileLength {
		fmt.Fprintf(os.Stderr, "fortuna: bad seed file length\n")
		t.FailNow()
	}
}

func TestReadSeed(t *testing.T) {
	rng := New(nil)
	err := rng.ReadSeed(seed)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		t.FailNow()
	}

	seed = nil
	err = rng.ReadSeed(seed)
	if err == nil {
		fmt.Fprintf(os.Stderr, "fortuna: ReadSeed should fail\n")
		t.FailNow()
	}

	seed = make([]byte, SeedFileLength-1)
	if err = rng.ReadSeed(seed); err == nil {
		fmt.Fprintf(os.Stderr, "fortuna: ReadSeed should fail\n")
		t.FailNow()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"archive/tar"
	"bytes"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/moby/src/initrd"
)

// Process the build arguments and execute build
func build(args []string) {
	buildCmd := flag.NewFlagSet("build", flag.ExitOnError)
	buildCmd.Usage = func() {
		fmt.Printf("USAGE: %s build [options] <file>[.yml]\n\n", os.Args[0])
		fmt.Printf("Options:\n")
		buildCmd.PrintDefaults()
	}
	buildName := buildCmd.String("name", "", "Name to use for output files")
	buildPull := buildCmd.Bool("pull", false, "Always pull images")

	if err := buildCmd.Parse(args); err != nil {
		log.Fatal("Unable to parse args")
	}
	remArgs := buildCmd.Args()

	if len(remArgs) == 0 {
		fmt.Println("Please specify a configuration file")
		buildCmd.Usage()
		os.Exit(1)
	}
	conf := remArgs[0]
	if !(filepath.Ext(conf) == ".yml" || filepath.Ext(conf) == ".yaml") {
		conf = conf + ".yml"
	}

	buildInternal(*buildName, *buildPull, conf)
}

func initrdAppend(iw *initrd.Writer, r io.Reader) {
	_, err := initrd.Copy(iw, r)
	if err != nil {
		log.Fatalf("initrd write error: %v", err)
	}
}

func enforceContentTrust(fullImageName string, config *TrustConfig) bool {
	for _, img := range config.Image {
		// First check for an exact name match
		if img == fullImageName {
			return true
		}
		// Also check for an image name only match
		// by removing a possible tag (with possibly added digest):
		if img == strings.TrimSuffix(fullImageName, ":") {
			return true
		}
		// and by removing a possible digest:
		if img == strings.TrimSuffix(fullImageName, "@sha256:") {
			return true
		}
	}

	for _, org := range config.Org {
		if strings.HasPrefix(fullImageName, org+"/") {
			return true
		}
	}
	return false
}

// Perform the actual build process
func buildInternal(name string, pull bool, conf string) {
	if name == "" {
		name = filepath.Base(conf)
		ext := filepath.Ext(conf)
		if ext != "" {
			name = name[:len(name)-len(ext)]
		}
	}

	config, err := ioutil.ReadFile(conf)
	if err != nil {
		log.Fatalf("Cannot open config file: %v", err)
	}

	m, err := NewConfig(config)
	if err != nil {
		log.Fatalf("Invalid config: %v", err)
	}

	w := new(bytes.Buffer)
	iw := initrd.NewWriter(w)

	if pull || enforceContentTrust(m.Kernel.Image, &m.Trust) {
		log.Infof("Pull kernel image: %s", m.Kernel.Image)
		err := dockerPull(m.Kernel.Image, enforceContentTrust(m.Kernel.Image, &m.Trust))
		if err != nil {
			log.Fatalf("Could not pull image %s: %v", m.Kernel.Image, err)
		}
	}
	// get kernel bzImage and initrd tarball from container
	// TODO examine contents to see what names they might have
	log.Infof("Extract kernel image: %s", m.Kernel.Image)
	const (
		bzimageName = "bzImage"
		ktarName    = "kernel.tar"
	)
	out, err := dockerRun(m.Kernel.Image, "tar", "cf", "-", bzimageName, ktarName)
	if err != nil {
		log.Fatalf("Failed to extract kernel image and tarball: %v", err)
	}
	buf := bytes.NewBuffer(out)
	bzimage, ktar, err := untarKernel(buf, bzimageName, ktarName)
	if err != nil {
		log.Fatalf("Could not extract bzImage and kernel filesystem from tarball. %v", err)
	}
	initrdAppend(iw, ktar)

	// convert init images to tarballs
	log.Infof("Add init containers:")
	for _, ii := range m.Init {
		if pull || enforceContentTrust(ii, &m.Trust) {
			log.Infof("Pull init image: %s", ii)
			err := dockerPull(ii, enforceContentTrust(ii, &m.Trust))
			if err != nil {
				log.Fatalf("Could not pull image %s: %v", ii, err)
			}
		}
		log.Infof("Process init image: %s", ii)
		init, err := ImageExtract(ii, "")
		if err != nil {
			log.Fatalf("Failed to build init tarball from %s: %v", ii, err)
		}
		buffer := bytes.NewBuffer(init)
		initrdAppend(iw, buffer)
	}

	log.Infof("Add onboot containers:")
	for i, image := range m.Onboot {
		if pull || enforceContentTrust(image.Image, &m.Trust) {
			log.Infof("  Pull: %s", image.Image)
			err := dockerPull(image.Image, enforceContentTrust(image.Image, &m.Trust))
			if err != nil {
				log.Fatalf("Could not pull image %s: %v", image.Image, err)
			}
		}
		log.Infof("  Create OCI config for %s", image.Image)
		config, err := ConfigToOCI(&image)
		if err != nil {
			log.Fatalf("Failed to create config.json for %s: %v", image.Image, err)
		}
		so := fmt.Sprintf("%03d", i)
		path := "containers/onboot/" + so + "-" + image.Name
		out, err := ImageBundle(path, image.Image, config)
		if err != nil {
			log.Fatalf("Failed to extract root filesystem for %s: %v", image.Image, err)
		}
		buffer := bytes.NewBuffer(out)
		initrdAppend(iw, buffer)
	}

	log.Infof("Add service containers:")
	for _, image := range m.Services {
		if pull || enforceContentTrust(image.Image, &m.Trust) {
			log.Infof("  Pull: %s", image.Image)
			err := dockerPull(image.Image, enforceContentTrust(image.Image, &m.Trust))
			if err != nil {
				log.Fatalf("Could not pull image %s: %v", image.Image, err)
			}
		}
		log.Infof("  Create OCI config for %s", image.Image)
		config, err := ConfigToOCI(&image)
		if err != nil {
			log.Fatalf("Failed to create config.json for %s: %v", image.Image, err)
		}
		path := "containers/services/" + image.Name
		out, err := ImageBundle(path, image.Image, config)
		if err != nil {
			log.Fatalf("Failed to extract root filesystem for %s: %v", image.Image, err)
		}
		buffer := bytes.NewBuffer(out)
		initrdAppend(iw, buffer)
	}

	// add files
	buffer, err := filesystem(m)
	if err != nil {
		log.Fatalf("failed to add filesystem parts: %v", err)
	}
	initrdAppend(iw, buffer)
	err = iw.Close()
	if err != nil {
		log.Fatalf("initrd close error: %v", err)
	}

	log.Infof("Create outputs:")
	err = outputs(m, name, bzimage.Bytes(), w.Bytes())
	if err != nil {
		log.Fatalf("Error writing outputs: %v", err)
	}
}

func untarKernel(buf *bytes.Buffer, bzimageName, ktarName string) (*bytes.Buffer, *bytes.Buffer, error) {
	tr := tar.NewReader(buf)

	var bzimage, ktar *bytes.Buffer

	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Fatalln(err)
		}
		switch hdr.Name {
		case bzimageName:
			bzimage = new(bytes.Buffer)
			_, err := io.Copy(bzimage, tr)
			if err != nil {
				return nil, nil, err
			}
		case ktarName:
			ktar = new(bytes.Buffer)
			_, err := io.Copy(ktar, tr)
			if err != nil {
				return nil, nil, err
			}
		default:
			continue
		}
	}

	if ktar == nil || bzimage == nil {
		return nil, nil, errors.New("did not find bzImage and kernel.tar in tarball")
	}

	return bzimage, ktar, nil
}
<commit_msg>Rename package initrd in build.go<commit_after>package main

import (
	"archive/tar"
	"bytes"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/linuxkit/linuxkit/src/initrd"
)

// Process the build arguments and execute build
func build(args []string) {
	buildCmd := flag.NewFlagSet("build", flag.ExitOnError)
	buildCmd.Usage = func() {
		fmt.Printf("USAGE: %s build [options] <file>[.yml]\n\n", os.Args[0])
		fmt.Printf("Options:\n")
		buildCmd.PrintDefaults()
	}
	buildName := buildCmd.String("name", "", "Name to use for output files")
	buildPull := buildCmd.Bool("pull", false, "Always pull images")

	if err := buildCmd.Parse(args); err != nil {
		log.Fatal("Unable to parse args")
	}
	remArgs := buildCmd.Args()

	if len(remArgs) == 0 {
		fmt.Println("Please specify a configuration file")
		buildCmd.Usage()
		os.Exit(1)
	}
	conf := remArgs[0]
	if !(filepath.Ext(conf) == ".yml" || filepath.Ext(conf) == ".yaml") {
		conf = conf + ".yml"
	}

	buildInternal(*buildName, *buildPull, conf)
}

func initrdAppend(iw *initrd.Writer, r io.Reader) {
	_, err := initrd.Copy(iw, r)
	if err != nil {
		log.Fatalf("initrd write error: %v", err)
	}
}

func enforceContentTrust(fullImageName string, config *TrustConfig) bool {
	for _, img := range config.Image {
		// First check for an exact name match
		if img == fullImageName {
			return true
		}
		// Also check for an image name only match
		// by removing a possible tag (with possibly added digest):
		if img == strings.TrimSuffix(fullImageName, ":") {
			return true
		}
		// and by removing a possible digest:
		if img == strings.TrimSuffix(fullImageName, "@sha256:") {
			return true
		}
	}

	for _, org := range config.Org {
		if strings.HasPrefix(fullImageName, org+"/") {
			return true
		}
	}
	return false
}

// Perform the actual build process
func buildInternal(name string, pull bool, conf string) {
	if name == "" {
		name = filepath.Base(conf)
		ext := filepath.Ext(conf)
		if ext != "" {
			name = name[:len(name)-len(ext)]
		}
	}

	config, err := ioutil.ReadFile(conf)
	if err != nil {
		log.Fatalf("Cannot open config file: %v", err)
	}

	m, err := NewConfig(config)
	if err != nil {
		log.Fatalf("Invalid config: %v", err)
	}

	w := new(bytes.Buffer)
	iw := initrd.NewWriter(w)

	if pull || enforceContentTrust(m.Kernel.Image, &m.Trust) {
		log.Infof("Pull kernel image: %s", m.Kernel.Image)
		err := dockerPull(m.Kernel.Image, enforceContentTrust(m.Kernel.Image, &m.Trust))
		if err != nil {
			log.Fatalf("Could not pull image %s: %v", m.Kernel.Image, err)
		}
	}
	// get kernel bzImage and initrd tarball from container
	// TODO examine contents to see what names they might have
	log.Infof("Extract kernel image: %s", m.Kernel.Image)
	const (
		bzimageName = "bzImage"
		ktarName    = "kernel.tar"
	)
	out, err := dockerRun(m.Kernel.Image, "tar", "cf", "-", bzimageName, ktarName)
	if err != nil {
		log.Fatalf("Failed to extract kernel image and tarball: %v", err)
	}
	buf := bytes.NewBuffer(out)
	bzimage, ktar, err := untarKernel(buf, bzimageName, ktarName)
	if err != nil {
		log.Fatalf("Could not extract bzImage and kernel filesystem from tarball. %v", err)
	}
	initrdAppend(iw, ktar)

	// convert init images to tarballs
	log.Infof("Add init containers:")
	for _, ii := range m.Init {
		if pull || enforceContentTrust(ii, &m.Trust) {
			log.Infof("Pull init image: %s", ii)
			err := dockerPull(ii, enforceContentTrust(ii, &m.Trust))
			if err != nil {
				log.Fatalf("Could not pull image %s: %v", ii, err)
			}
		}
		log.Infof("Process init image: %s", ii)
		init, err := ImageExtract(ii, "")
		if err != nil {
			log.Fatalf("Failed to build init tarball from %s: %v", ii, err)
		}
		buffer := bytes.NewBuffer(init)
		initrdAppend(iw, buffer)
	}

	log.Infof("Add onboot containers:")
	for i, image := range m.Onboot {
		if pull || enforceContentTrust(image.Image, &m.Trust) {
			log.Infof("  Pull: %s", image.Image)
			err := dockerPull(image.Image, enforceContentTrust(image.Image, &m.Trust))
			if err != nil {
				log.Fatalf("Could not pull image %s: %v", image.Image, err)
			}
		}
		log.Infof("  Create OCI config for %s", image.Image)
		config, err := ConfigToOCI(&image)
		if err != nil {
			log.Fatalf("Failed to create config.json for %s: %v", image.Image, err)
		}
		so := fmt.Sprintf("%03d", i)
		path := "containers/onboot/" + so + "-" + image.Name
		out, err := ImageBundle(path, image.Image, config)
		if err != nil {
			log.Fatalf("Failed to extract root filesystem for %s: %v", image.Image, err)
		}
		buffer := bytes.NewBuffer(out)
		initrdAppend(iw, buffer)
	}

	log.Infof("Add service containers:")
	for _, image := range m.Services {
		if pull || enforceContentTrust(image.Image, &m.Trust) {
			log.Infof("  Pull: %s", image.Image)
			err := dockerPull(image.Image, enforceContentTrust(image.Image, &m.Trust))
			if err != nil {
				log.Fatalf("Could not pull image %s: %v", image.Image, err)
			}
		}
		log.Infof("  Create OCI config for %s", image.Image)
		config, err := ConfigToOCI(&image)
		if err != nil {
			log.Fatalf("Failed to create config.json for %s: %v", image.Image, err)
		}
		path := "containers/services/" + image.Name
		out, err := ImageBundle(path, image.Image, config)
		if err != nil {
			log.Fatalf("Failed to extract root filesystem for %s: %v", image.Image, err)
		}
		buffer := bytes.NewBuffer(out)
		initrdAppend(iw, buffer)
	}

	// add files
	buffer, err := filesystem(m)
	if err != nil {
		log.Fatalf("failed to add filesystem parts: %v", err)
	}
	initrdAppend(iw, buffer)
	err = iw.Close()
	if err != nil {
		log.Fatalf("initrd close error: %v", err)
	}

	log.Infof("Create outputs:")
	err = outputs(m, name, bzimage.Bytes(), w.Bytes())
	if err != nil {
		log.Fatalf("Error writing outputs: %v", err)
	}
}

func untarKernel(buf *bytes.Buffer, bzimageName, ktarName string) (*bytes.Buffer, *bytes.Buffer, error) {
	tr := tar.NewReader(buf)

	var bzimage, ktar *bytes.Buffer

	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Fatalln(err)
		}
		switch hdr.Name {
		case bzimageName:
			bzimage = new(bytes.Buffer)
			_, err := io.Copy(bzimage, tr)
			if err != nil {
				return nil, nil, err
			}
		case ktarName:
			ktar = new(bytes.Buffer)
			_, err := io.Copy(ktar, tr)
			if err != nil {
				return nil, nil, err
			}
		default:
			continue
		}
	}

	if ktar == nil || bzimage == nil {
		return nil, nil, errors.New("did not find bzImage and kernel.tar in tarball")
	}

	return bzimage, ktar, nil
}
<|endoftext|>


<commit_before>package file

import (
	"bufio"
	"errors"
	"github.com/luopengift/golibs/logger"
	"io"
	"os"
	"strings"
	"time"
)

type Tail struct {
	*File
	cname    string //config name
	line     chan *string
	reader   *bufio.Reader
	interval int64
	Handler  // file name handle interface
	//EOF
	//@ true: stop
	//@ false: wait
	endstop bool
}

func NewTail(cname string, handler Handler) *Tail {
	name := handler.Handle(cname)
	file := NewFile(name, os.O_RDONLY)

	tail := new(Tail)
	tail.File = NewFile(name, os.O_RDONLY)
	tail.cname = cname
	tail.line = make(chan *string)
	tail.reader = bufio.NewReader(file.fd)
	tail.interval = 1000 //ms
	tail.endstop = false
	tail.Handler = handler
	return tail
}

func (self *Tail) EndStop(b bool) {
	self.endstop = b
}

func (self *Tail) Close() error {
	close(self.line)
	return self.File.Close()
}

func (self *Tail) ReOpen() error {
	if err := self.File.Close(); err != nil {
		logger.Error("<file %v close fail:%v>", self.name, err)
	}
	self.name = self.Handler.Handle(self.cname)
	err := self.Open()
	if err != nil {
		return err
	}
	self.reader = bufio.NewReader(self.fd)
	return nil
}

func (self *Tail) Stop() {
	self.File.Close()
	close(self.line)
}

func (self *Tail) ReadLine() {
	go func() {

		offset, err := self.TrancateOffsetByLF(self.seek)
		if err != nil {
			logger.Error("<Trancate offset:%d,Error:%+v>", self.seek, err)
		}
		err = self.Seek(offset)
		if err != nil {
			logger.Error("<seek offset[%d] error:%+v>", self.seek, err)
		}

		for {
			line, err := self.reader.ReadString('\n')
			switch {
			case err == io.EOF:
				if self.endstop {
					logger.Warn("<file %s is END:%+v>", self.name, err)
					self.Stop()
					return
				}
				time.Sleep(time.Duration(self.interval) * time.Millisecond)
				if self.name == self.cname {
					if inode, err := Inode(self.name); err != nil { //检测是否需要重新打开新的文件
						continue
					} else {
						if inode != self.inode {
							self.ReOpen()
						}
					}
				} else {
					if self.name == self.Handler.Handle(self.cname) { //检测是否需要按时间轮转新文件
						continue
					} else {
						self.ReOpen()
					}
				}

			case err != nil && err != io.EOF:
				time.Sleep(time.Duration(self.interval) * time.Millisecond)
				logger.Error("<Read file error:%v,%v>", line, err)
				self.ReOpen()
				continue
			default:
				msg := strings.TrimRight(line, "\n")
				self.line <- &msg
				self.seek += int64(len(line))
			}
		}
	}()
}

func (self *Tail) NextLine() chan *string {
	return self.line
}

func (self *Tail) Read(p []byte) (int, error) {
	msg := <-self.line
	if len(*msg) > len(p) {
		return 0, errors.New("message is large than buf")
	}
	copy(p, []byte(*msg))
	return len(*msg), nil
}
<commit_msg>Fix file endstop return EOF<commit_after>package file

import (
	"bufio"
	"errors"
	"github.com/luopengift/golibs/logger"
	"io"
	"os"
	"strings"
	"time"
)

type Tail struct {
	*File
	cname    string //config name
	line     chan *string
	reader   *bufio.Reader
	interval int64
	Handler  // file name handle interface
	//EOF
	//@ true: stop
	//@ false: wait
	endstop bool
}

func NewTail(cname string, handler Handler) *Tail {
	name := handler.Handle(cname)
	file := NewFile(name, os.O_RDONLY)

	tail := new(Tail)
	tail.File = NewFile(name, os.O_RDONLY)
	tail.cname = cname
	tail.line = make(chan *string)
	tail.reader = bufio.NewReader(file.fd)
	tail.interval = 1000 //ms
	tail.endstop = false
	tail.Handler = handler
	return tail
}

func (self *Tail) EndStop(b bool) {
	self.endstop = b
}

func (self *Tail) Close() error {
	close(self.line)
	return self.File.Close()
}

func (self *Tail) ReOpen() error {
	if err := self.File.Close(); err != nil {
		logger.Error("<file %v close fail:%v>", self.name, err)
	}
	self.name = self.Handler.Handle(self.cname)
	err := self.Open()
	if err != nil {
		return err
	}
	self.reader = bufio.NewReader(self.fd)
	return nil
}

func (self *Tail) Stop() {
	self.File.Close()
	close(self.line)
}

func (self *Tail) ReadLine() {
	go func() {

		offset, err := self.TrancateOffsetByLF(self.seek)
		if err != nil {
			logger.Error("<Trancate offset:%d,Error:%+v>", self.seek, err)
		}
		err = self.Seek(offset)
		if err != nil {
			logger.Error("<seek offset[%d] error:%+v>", self.seek, err)
		}

		for {
			line, err := self.reader.ReadString('\n')
			switch {
			case err == io.EOF:
				if self.endstop {
					logger.Warn("<file %s is END:%+v>", self.name, err)
					self.Stop()
					return
				}
				time.Sleep(time.Duration(self.interval) * time.Millisecond)
				if self.name == self.cname {
					if inode, err := Inode(self.name); err != nil { //检测是否需要重新打开新的文件
						continue
					} else {
						if inode != self.inode {
							self.ReOpen()
						}
					}
				} else {
					if self.name == self.Handler.Handle(self.cname) { //检测是否需要按时间轮转新文件
						continue
					} else {
						self.ReOpen()
					}
				}

			case err != nil && err != io.EOF:
				time.Sleep(time.Duration(self.interval) * time.Millisecond)
				logger.Error("<Read file error:%v,%v>", line, err)
				self.ReOpen()
				continue
			default:
				msg := strings.TrimRight(line, "\n")
				self.line <- &msg
				self.seek += int64(len(line))
			}
		}
	}()
}

func (self *Tail) NextLine() chan *string {
	return self.line
}

func (self *Tail) Read(p []byte) (int, error) {
	msg,ok := <-self.line
	if !ok {
		return 0,fmt.Errorf("file is closed")
	}
	if len(*msg) > len(p) {
		return 0, errors.New("message is large than buf")
	}
	copy(p, []byte(*msg))
	return len(*msg), nil
}
<|endoftext|>


<commit_before>// SPDX-License-Identifier: MIT

package utils

import (
	"io/ioutil"
	"path/filepath"
	"testing"

	"github.com/issue9/assert"
)

func TestFileExists(t *testing.T) {
	a := assert.New(t)

	// 测试文件
	a.True(FileExists("utils.go"))
	a.False(FileExists("unknown.go"))

	// 测试文件夹
	a.True(FileExists("./"))
	a.False(FileExists("./unknown_dir/"))
}

func TestDumpGoFile(t *testing.T) {
	a := assert.New(t)

	a.NotError(DumpGoFile("./testdata/go.go", "var x=1"))
	content, err := ioutil.ReadFile("./testdata/go.go")
	a.NotError(err)
	a.Equal(string(content), "var x = 1")
}

func TestCurrentPath(t *testing.T) {
	a := assert.New(t)

	dir, err := filepath.Abs("./file.go")
	a.NotError(err).NotEmpty(dir)

	d, err := filepath.Abs(CurrentPath("./file.go"))
	a.NotError(err).NotEmpty(d)

	a.Equal(d, dir)
}

func TestCurrentDir(t *testing.T) {
	a := assert.New(t)

	dir, err := filepath.Abs("./")
	a.NotError(err).NotEmpty(dir)

	a.Equal(CurrentDir(), dir)
}

func TestCurrentFile(t *testing.T) {
	a := assert.New(t)

	filename, err := filepath.Abs("./file_test.go")
	a.NotError(err).NotEmpty(filename)

	a.Equal(CurrentFile(), filename)
}

func TestCurrentFunction(t *testing.T) {
	a := assert.New(t)

	a.Equal(CurrentFunction(), "TestCurrentFunction")
}

func TestCurrentLine(t *testing.T) {
	a := assert.New(t)

	a.Equal(CurrentLine(), 73)
}
<commit_msg>test: 修正 windows 的测试错误<commit_after>// SPDX-License-Identifier: MIT

package utils

import (
	"io/ioutil"
	"path/filepath"
	"testing"

	"github.com/issue9/assert"
)

func TestFileExists(t *testing.T) {
	a := assert.New(t)

	// 测试文件
	a.True(FileExists("utils.go"))
	a.False(FileExists("unknown.go"))

	// 测试文件夹
	a.True(FileExists("./"))
	a.False(FileExists("./unknown_dir/"))
}

func TestDumpGoFile(t *testing.T) {
	a := assert.New(t)

	a.NotError(DumpGoFile("./testdata/go.go", "var x=1"))
	content, err := ioutil.ReadFile("./testdata/go.go")
	a.NotError(err)
	a.Equal(string(content), "var x = 1")
}

func TestCurrentPath(t *testing.T) {
	a := assert.New(t)

	dir, err := filepath.Abs("./file.go")
	a.NotError(err).NotEmpty(dir)

	d, err := filepath.Abs(CurrentPath("./file.go"))
	a.NotError(err).NotEmpty(d)

	a.Equal(d, dir)
}

func TestCurrentDir(t *testing.T) {
	a := assert.New(t)

	dir, err := filepath.Abs("./")
	a.NotError(err).NotEmpty(dir)

	a.Equal(CurrentDir(), dir)
}

func TestCurrentFile(t *testing.T) {
	a := assert.New(t)

	filename, err := filepath.Abs("./file_test.go")
	a.NotError(err).NotEmpty(filename)

	a.Equal(CurrentFile(), filepath.FromSlash(filename))
}

func TestCurrentFunction(t *testing.T) {
	a := assert.New(t)

	a.Equal(CurrentFunction(), "TestCurrentFunction")
}

func TestCurrentLine(t *testing.T) {
	a := assert.New(t)

	a.Equal(CurrentLine(), 73)
}
<|endoftext|>


<commit_before>// Package files provides a set type to track local/remote files with newness checks.
package files

import (
	"sync"

	"github.com/calmh/syncthing/cid"
	"github.com/calmh/syncthing/lamport"
	"github.com/calmh/syncthing/protocol"
	"github.com/calmh/syncthing/scanner"
)

type fileRecord struct {
	Usage int
	File  scanner.File
}

type bitset uint64

type Set struct {
	sync.Mutex
	files              map[key]fileRecord
	remoteKey          [64]map[string]key
	changes            [64]uint64
	globalAvailability map[string]bitset
	globalKey          map[string]key
}

func NewSet() *Set {
	var m = Set{
		files:              make(map[key]fileRecord),
		globalAvailability: make(map[string]bitset),
		globalKey:          make(map[string]key),
	}
	return &m
}

func (m *Set) Replace(id uint, fs []scanner.File) {
	if debug {
		dlog.Printf("Replace(%d, [%d])", id, len(fs))
	}
	if id > 63 {
		panic("Connection ID must be in the range 0 - 63 inclusive")
	}

	m.Lock()
	if len(fs) == 0 || !m.equals(id, fs) {
		m.changes[id]++
		m.replace(id, fs)
	}
	m.Unlock()
}

func (m *Set) ReplaceWithDelete(id uint, fs []scanner.File) {
	if debug {
		dlog.Printf("ReplaceWithDelete(%d, [%d])", id, len(fs))
	}
	if id > 63 {
		panic("Connection ID must be in the range 0 - 63 inclusive")
	}

	m.Lock()
	if len(fs) == 0 || !m.equals(id, fs) {
		m.changes[id]++

		var nf = make(map[string]key, len(fs))
		for _, f := range fs {
			nf[f.Name] = keyFor(f)
		}

		// For previously existing files not in the list, add them to the list
		// with the relevant delete flags etc set. Previously existing files
		// with the delete bit already set are not modified.

		for _, ck := range m.remoteKey[cid.LocalID] {
			if _, ok := nf[ck.Name]; !ok {
				cf := m.files[ck].File
				if cf.Flags&protocol.FlagDeleted != protocol.FlagDeleted {
					cf.Flags |= protocol.FlagDeleted
					cf.Blocks = nil
					cf.Size = 0
					cf.Version = lamport.Default.Tick(cf.Version)
				}
				fs = append(fs, cf)
				if debug {
					dlog.Println("deleted:", ck.Name)
				}
			}
		}

		m.replace(id, fs)
	}
	m.Unlock()
}

func (m *Set) Update(id uint, fs []scanner.File) {
	if debug {
		dlog.Printf("Update(%d, [%d])", id, len(fs))
	}
	m.Lock()
	m.update(id, fs)
	m.changes[id]++
	m.Unlock()
}

func (m *Set) Need(id uint) []scanner.File {
	if debug {
		dlog.Printf("Need(%d)", id)
	}
	var fs = make([]scanner.File, 0, len(m.globalKey)/2) // Just a guess, but avoids too many reallocations
	m.Lock()
	rkID := m.remoteKey[id]
	for name, gk := range m.globalKey {
		if gk.newerThan(rkID[name]) {
			file := m.files[gk].File
			if file.Flags&protocol.FlagDirectory == 0 || // Regular file
				file.Flags&(protocol.FlagDirectory|protocol.FlagDeleted) == protocol.FlagDirectory { // Non-deleted directory
				fs = append(fs, file)
			}
		}
	}
	m.Unlock()
	return fs
}

func (m *Set) Have(id uint) []scanner.File {
	if debug {
		dlog.Printf("Have(%d)", id)
	}
	var fs = make([]scanner.File, 0, len(m.remoteKey[id]))
	m.Lock()
	for _, rk := range m.remoteKey[id] {
		fs = append(fs, m.files[rk].File)
	}
	m.Unlock()
	return fs
}

func (m *Set) Global() []scanner.File {
	if debug {
		dlog.Printf("Global()")
	}
	var fs = make([]scanner.File, 0, len(m.globalKey))
	m.Lock()
	for _, rk := range m.globalKey {
		fs = append(fs, m.files[rk].File)
	}
	m.Unlock()
	return fs
}

func (m *Set) Get(id uint, file string) scanner.File {
	m.Lock()
	defer m.Unlock()
	if debug {
		dlog.Printf("Get(%d, %q)", id, file)
	}
	return m.files[m.remoteKey[id][file]].File
}

func (m *Set) GetGlobal(file string) scanner.File {
	m.Lock()
	defer m.Unlock()
	if debug {
		dlog.Printf("GetGlobal(%q)", file)
	}
	return m.files[m.globalKey[file]].File
}

func (m *Set) Availability(name string) bitset {
	m.Lock()
	defer m.Unlock()
	av := m.globalAvailability[name]
	if debug {
		dlog.Printf("Availability(%q) = %0x", name, av)
	}
	return av
}

func (m *Set) Changes(id uint) uint64 {
	m.Lock()
	defer m.Unlock()
	if debug {
		dlog.Printf("Changes(%d)", id)
	}
	return m.changes[id]
}

func (m *Set) equals(id uint, fs []scanner.File) bool {
	curWithoutDeleted := make(map[string]key)
	for _, k := range m.remoteKey[id] {
		f := m.files[k].File
		if f.Flags&protocol.FlagDeleted == 0 {
			curWithoutDeleted[f.Name] = k
		}
	}
	if len(curWithoutDeleted) != len(fs) {
		return false
	}
	for _, f := range fs {
		if curWithoutDeleted[f.Name] != keyFor(f) {
			return false
		}
	}
	return true
}

func (m *Set) update(cid uint, fs []scanner.File) {
	remFiles := m.remoteKey[cid]
	for _, f := range fs {
		n := f.Name
		fk := keyFor(f)

		if ck, ok := remFiles[n]; ok && ck == fk {
			// The remote already has exactly this file, skip it
			continue
		}

		remFiles[n] = fk

		// Keep the block list or increment the usage
		if br, ok := m.files[fk]; !ok {
			m.files[fk] = fileRecord{
				Usage: 1,
				File:  f,
			}
		} else {
			br.Usage++
			m.files[fk] = br
		}

		// Update global view
		gk, ok := m.globalKey[n]
		switch {
		case ok && fk == gk:
			av := m.globalAvailability[n]
			av |= 1 << cid
			m.globalAvailability[n] = av
		case fk.newerThan(gk):
			m.globalKey[n] = fk
			m.globalAvailability[n] = 1 << cid
		}
	}
}

func (m *Set) replace(cid uint, fs []scanner.File) {
	// Decrement usage for all files belonging to this remote, and remove
	// those that are no longer needed.
	for _, fk := range m.remoteKey[cid] {
		br, ok := m.files[fk]
		switch {
		case ok && br.Usage == 1:
			delete(m.files, fk)
		case ok && br.Usage > 1:
			br.Usage--
			m.files[fk] = br
		}
	}

	// Clear existing remote remoteKey
	m.remoteKey[cid] = make(map[string]key)

	// Recalculate global based on all remaining remoteKey
	for n := range m.globalKey {
		var nk key    // newest key
		var na bitset // newest availability

		for i, rem := range m.remoteKey {
			if rk, ok := rem[n]; ok {
				switch {
				case rk == nk:
					na |= 1 << uint(i)
				case rk.newerThan(nk):
					nk = rk
					na = 1 << uint(i)
				}
			}
		}

		if na != 0 {
			// Someone had the file
			m.globalKey[n] = nk
			m.globalAvailability[n] = na
		} else {
			// Noone had the file
			delete(m.globalKey, n)
			delete(m.globalAvailability, n)
		}
	}

	// Add new remote remoteKey to the mix
	m.update(cid, fs)
}
<commit_msg>Speed up Global() at the price of Update() in fileset<commit_after>// Package files provides a set type to track local/remote files with newness checks.
package files

import (
	"sync"

	"github.com/calmh/syncthing/cid"
	"github.com/calmh/syncthing/lamport"
	"github.com/calmh/syncthing/protocol"
	"github.com/calmh/syncthing/scanner"
)

type fileRecord struct {
	File   scanner.File
	Usage  int
	Global bool
}

type bitset uint64

type Set struct {
	sync.Mutex
	files              map[key]fileRecord
	remoteKey          [64]map[string]key
	changes            [64]uint64
	globalAvailability map[string]bitset
	globalKey          map[string]key
}

func NewSet() *Set {
	var m = Set{
		files:              make(map[key]fileRecord),
		globalAvailability: make(map[string]bitset),
		globalKey:          make(map[string]key),
	}
	return &m
}

func (m *Set) Replace(id uint, fs []scanner.File) {
	if debug {
		dlog.Printf("Replace(%d, [%d])", id, len(fs))
	}
	if id > 63 {
		panic("Connection ID must be in the range 0 - 63 inclusive")
	}

	m.Lock()
	if len(fs) == 0 || !m.equals(id, fs) {
		m.changes[id]++
		m.replace(id, fs)
	}
	m.Unlock()
}

func (m *Set) ReplaceWithDelete(id uint, fs []scanner.File) {
	if debug {
		dlog.Printf("ReplaceWithDelete(%d, [%d])", id, len(fs))
	}
	if id > 63 {
		panic("Connection ID must be in the range 0 - 63 inclusive")
	}

	m.Lock()
	if len(fs) == 0 || !m.equals(id, fs) {
		m.changes[id]++

		var nf = make(map[string]key, len(fs))
		for _, f := range fs {
			nf[f.Name] = keyFor(f)
		}

		// For previously existing files not in the list, add them to the list
		// with the relevant delete flags etc set. Previously existing files
		// with the delete bit already set are not modified.

		for _, ck := range m.remoteKey[cid.LocalID] {
			if _, ok := nf[ck.Name]; !ok {
				cf := m.files[ck].File
				if cf.Flags&protocol.FlagDeleted != protocol.FlagDeleted {
					cf.Flags |= protocol.FlagDeleted
					cf.Blocks = nil
					cf.Size = 0
					cf.Version = lamport.Default.Tick(cf.Version)
				}
				fs = append(fs, cf)
				if debug {
					dlog.Println("deleted:", ck.Name)
				}
			}
		}

		m.replace(id, fs)
	}
	m.Unlock()
}

func (m *Set) Update(id uint, fs []scanner.File) {
	if debug {
		dlog.Printf("Update(%d, [%d])", id, len(fs))
	}
	m.Lock()
	m.update(id, fs)
	m.changes[id]++
	m.Unlock()
}

func (m *Set) Need(id uint) []scanner.File {
	if debug {
		dlog.Printf("Need(%d)", id)
	}
	var fs = make([]scanner.File, 0, len(m.globalKey)/2) // Just a guess, but avoids too many reallocations
	m.Lock()
	rkID := m.remoteKey[id]
	for gk, gf := range m.files {
		if !gf.Global {
			continue
		}

		file := gf.File
		switch {
		case file.Flags&protocol.FlagDirectory == 0 && gk.newerThan(rkID[gk.Name]):
			fs = append(fs, file)
		case file.Flags&(protocol.FlagDirectory|protocol.FlagDeleted) == protocol.FlagDirectory && gk.newerThan(rkID[gk.Name]):
			fs = append(fs, file)
		}
	}
	m.Unlock()
	return fs
}

func (m *Set) Have(id uint) []scanner.File {
	if debug {
		dlog.Printf("Have(%d)", id)
	}
	var fs = make([]scanner.File, 0, len(m.remoteKey[id]))
	m.Lock()
	for _, rk := range m.remoteKey[id] {
		fs = append(fs, m.files[rk].File)
	}
	m.Unlock()
	return fs
}

func (m *Set) Global() []scanner.File {
	if debug {
		dlog.Printf("Global()")
	}
	var fs = make([]scanner.File, 0, len(m.globalKey))
	m.Lock()
	for _, file := range m.files {
		if file.Global {
			fs = append(fs, file.File)
		}
	}
	m.Unlock()
	return fs
}

func (m *Set) Get(id uint, file string) scanner.File {
	m.Lock()
	defer m.Unlock()
	if debug {
		dlog.Printf("Get(%d, %q)", id, file)
	}
	return m.files[m.remoteKey[id][file]].File
}

func (m *Set) GetGlobal(file string) scanner.File {
	m.Lock()
	defer m.Unlock()
	if debug {
		dlog.Printf("GetGlobal(%q)", file)
	}
	return m.files[m.globalKey[file]].File
}

func (m *Set) Availability(name string) bitset {
	m.Lock()
	defer m.Unlock()
	av := m.globalAvailability[name]
	if debug {
		dlog.Printf("Availability(%q) = %0x", name, av)
	}
	return av
}

func (m *Set) Changes(id uint) uint64 {
	m.Lock()
	defer m.Unlock()
	if debug {
		dlog.Printf("Changes(%d)", id)
	}
	return m.changes[id]
}

func (m *Set) equals(id uint, fs []scanner.File) bool {
	curWithoutDeleted := make(map[string]key)
	for _, k := range m.remoteKey[id] {
		f := m.files[k].File
		if f.Flags&protocol.FlagDeleted == 0 {
			curWithoutDeleted[f.Name] = k
		}
	}
	if len(curWithoutDeleted) != len(fs) {
		return false
	}
	for _, f := range fs {
		if curWithoutDeleted[f.Name] != keyFor(f) {
			return false
		}
	}
	return true
}

func (m *Set) update(cid uint, fs []scanner.File) {
	remFiles := m.remoteKey[cid]
	for _, f := range fs {
		n := f.Name
		fk := keyFor(f)

		if ck, ok := remFiles[n]; ok && ck == fk {
			// The remote already has exactly this file, skip it
			continue
		}

		remFiles[n] = fk

		// Keep the block list or increment the usage
		if br, ok := m.files[fk]; !ok {
			m.files[fk] = fileRecord{
				Usage: 1,
				File:  f,
			}
		} else {
			br.Usage++
			m.files[fk] = br
		}

		// Update global view
		gk, ok := m.globalKey[n]
		switch {
		case ok && fk == gk:
			av := m.globalAvailability[n]
			av |= 1 << cid
			m.globalAvailability[n] = av
		case fk.newerThan(gk):
			if ok {
				f := m.files[gk]
				f.Global = false
				m.files[gk] = f
			}
			f := m.files[fk]
			f.Global = true
			m.files[fk] = f
			m.globalKey[n] = fk
			m.globalAvailability[n] = 1 << cid
		}
	}
}

func (m *Set) replace(cid uint, fs []scanner.File) {
	// Decrement usage for all files belonging to this remote, and remove
	// those that are no longer needed.
	for _, fk := range m.remoteKey[cid] {
		br, ok := m.files[fk]
		switch {
		case ok && br.Usage == 1:
			delete(m.files, fk)
		case ok && br.Usage > 1:
			br.Usage--
			m.files[fk] = br
		}
	}

	// Clear existing remote remoteKey
	m.remoteKey[cid] = make(map[string]key)

	// Recalculate global based on all remaining remoteKey
	for n := range m.globalKey {
		var nk key    // newest key
		var na bitset // newest availability

		for i, rem := range m.remoteKey {
			if rk, ok := rem[n]; ok {
				switch {
				case rk == nk:
					na |= 1 << uint(i)
				case rk.newerThan(nk):
					nk = rk
					na = 1 << uint(i)
				}
			}
		}

		if na != 0 {
			// Someone had the file
			m.globalKey[n] = nk
			m.globalAvailability[n] = na
		} else {
			// Noone had the file
			delete(m.globalKey, n)
			delete(m.globalAvailability, n)
		}
	}

	// Add new remote remoteKey to the mix
	m.update(cid, fs)
}
<|endoftext|>


<commit_before>package ov

import (
	"fmt"
	"github.com/HewlettPackard/oneview-golang/ov"
	"github.com/docker/machine/libmachine/log"
	"github.com/stretchr/testify/assert"
	"os"
	"testing"
)

func TestCreateEthernetNetwork(t *testing.T) {
	var (
		d        *OVTest
		c        *ov.OVClient
		testName string
	)
	if os.Getenv("ONEVIEW_TEST_ACCEPTANCE") == "true" {
		d, c = getTestDriverA("test_ethernet_network")
		if c == nil {
			t.Fatalf("Failed to execute getTestDriver() ")
		}
		// find out if the test ethernet network already exist
		testName = d.Tc.GetTestData(d.Env, "Name").(string)

		testEthNet, err := c.GetEthernetNetworkByName(testName)
		assert.NoError(t, err, "CreateEthernetNetwork get the EthernetNetwork error -> %s", err)

		if testEthNet.URI.IsNil() {
			testEthNet = ov.EthernetNetwork{
				Name:                testName,
				VlanId:              7,
				Purpose:             d.Tc.GetTestData(d.Env, "Purpose").(string),
				SmartLink:           d.Tc.GetTestData(d.Env, "SmartLink").(bool),
				PrivateNetwork:      d.Tc.GetTestData(d.Env, "PrivateNetwork").(bool),
				EthernetNetworkType: d.Tc.GetTestData(d.Env, "EthernetNetworkType").(string),
				Type:                d.Tc.GetTestData(d.Env, "Type").(string),
			}
			err := c.CreateEthernetNetwork(testEthNet)
			assert.NoError(t, err, "CreateEthernetNetwork error -> %s", err)

			err = c.CreateEthernetNetwork(testEthNet)
			assert.Error(t, err, "CreateEthernetNetwork should error because the EthernetNetwork already exists, err-> %s", err)

		} else {
			log.Warnf("The ethernetNetwork already exist, so skipping CreateEthernetNetwork test for %s", testName)
		}

		// reload the test profile that we just created
		testEthNet, err = c.GetEthernetNetworkByName(testName)
		assert.NoError(t, err, "GetEthernetNetwork error -> %s", err)
	}

}

func TestGetEthernetNetworkByName(t *testing.T) {
	var (
		d        *OVTest
		c        *ov.OVClient
		testName string
	)
	if os.Getenv("ONEVIEW_TEST_ACCEPTANCE") == "true" {
		d, c = getTestDriverA("test_ethernet_network")
		if c == nil {
			t.Fatalf("Failed to execute getTestDriver() ")
		}
		testName = d.Tc.GetTestData(d.Env, "Name").(string)

		testEthNet, err := c.GetEthernetNetworkByName(testName)
		assert.NoError(t, err, "GetEthernetNetworkByName thew an error -> %s", err)
		assert.Equal(t, testName, testEthNet.Name)

		testEthNet, err = c.GetEthernetNetworkByName("bad")
		assert.NoError(t, err, "GetEthernetNetworkByName with fake name -> %s", err)
		assert.Equal(t, "", testEthNet.Name)

	} else {
		d, c = getTestDriverU("test_ethernet_network")
		testName = d.Tc.GetTestData(d.Env, "Name").(string)
		data, err := c.GetEthernetNetworkByName(testName)
		assert.Error(t, err, fmt.Sprintf("ALL ok, no error, caught as expected: %s,%+v\n", err, data))
	}
}

func TestGetEthernetNetworks(t *testing.T) {
	var (
		c *ov.OVClient
	)
	if os.Getenv("ONEVIEW_TEST_ACCEPTANCE") == "true" {
		_, c = getTestDriverA("test_ethernet_network")
		if c == nil {
			t.Fatalf("Failed to execute getTestDriver() ")
		}
		ethernetNetworks, err := c.GetEthernetNetworks("", "")
		assert.NoError(t, err, "GetEthernetNetworks threw error -> %s, %+v\n", err, ethernetNetworks)

		ethernetNetworks, err = c.GetEthernetNetworks("", "name:asc")
		assert.NoError(t, err, "GetEthernetNetworks name:asc error -> %s, %+v\n", err, ethernetNetworks)

	} else {
		_, c = getTestDriverU("test_ethernet_network")
		data, err := c.GetEthernetNetworks("", "")
		assert.Error(t, err, fmt.Sprintf("ALL ok, no error, caught as expected: %s,%+v\n", err, data))
	}
}

func TestDeleteEthernetNetworkNotFound(t *testing.T) {
	var (
		c          *ov.OVClient
		testName   = "fake"
		testEthNet ov.EthernetNetwork
	)
	if os.Getenv("ONEVIEW_TEST_ACCEPTANCE") == "true" {
		_, c = getTestDriverA("test_ethernet_network")
		if c == nil {
			t.Fatalf("Failed to execute getTestDriver() ")
		}

		err := c.DeleteEthernetNetwork(testName)
		assert.NoError(t, err, "DeleteEthernetNetwork err-> %s", err)

		testEthNet, err = c.GetEthernetNetworkByName(testName)
		assert.NoError(t, err, "GetEthernetNetworkByName with deleted ethernet network -> %+v", err)
		assert.Equal(t, "", testEthNet.Name, fmt.Sprintf("Problem getting ethernet name, %+v", testEthNet))
	} else {
		_, c = getTestDriverU("test_ethernet_network")
		err := c.DeleteEthernetNetwork(testName)
		assert.Error(t, err, fmt.Sprintf("All ok, no error, caught as expected: %s,%+v\n", err, testEthNet))
	}
}

func TestDeleteEthernetNetwork(t *testing.T) {
	var (
		d          *OVTest
		c          *ov.OVClient
		testName   string
		testEthNet ov.EthernetNetwork
	)
	if os.Getenv("ONEVIEW_TEST_ACCEPTANCE") == "true" {
		d, c = getTestDriverA("test_ethernet_network")
		if c == nil {
			t.Fatalf("Failed to execute getTestDriver() ")
		}
		testName = d.Tc.GetTestData(d.Env, "Name").(string)

		err := c.DeleteEthernetNetwork(testName)
		assert.NoError(t, err, "DeleteEthernetNetwork err-> %s", err)

		testEthNet, err = c.GetEthernetNetworkByName(testName)
		assert.NoError(t, err, "GetEthernetNetworkByName with deleted ethernet network-> %+v", err)
		assert.Equal(t, "", testEthNet.Name, fmt.Sprintf("Problem getting ethnet name, %+v", testEthNet))
	} else {
		_, c = getTestDriverU("test_ethernet_network")
		err := c.DeleteEthernetNetwork("footest")
		assert.Error(t, err, fmt.Sprintf("ALL ok, no error, caught as expected: %s,%+v\n", err, testEthNet))
	}

}
<commit_msg>added query parameter changes to test file<commit_after>package ov

import (
	"fmt"
	"github.com/HewlettPackard/oneview-golang/ov"
	"github.com/docker/machine/libmachine/log"
	"github.com/stretchr/testify/assert"
	"os"
	"testing"
)

func TestCreateEthernetNetwork(t *testing.T) {
	var (
		d        *OVTest
		c        *ov.OVClient
		testName string
	)
	if os.Getenv("ONEVIEW_TEST_ACCEPTANCE") == "true" {
		d, c = getTestDriverA("test_ethernet_network")
		if c == nil {
			t.Fatalf("Failed to execute getTestDriver() ")
		}
		// find out if the test ethernet network already exist
		testName = d.Tc.GetTestData(d.Env, "Name").(string)

		testEthNet, err := c.GetEthernetNetworkByName(testName)
		assert.NoError(t, err, "CreateEthernetNetwork get the EthernetNetwork error -> %s", err)

		if testEthNet.URI.IsNil() {
			testEthNet = ov.EthernetNetwork{
				Name:                testName,
				VlanId:              7,
				Purpose:             d.Tc.GetTestData(d.Env, "Purpose").(string),
				SmartLink:           d.Tc.GetTestData(d.Env, "SmartLink").(bool),
				PrivateNetwork:      d.Tc.GetTestData(d.Env, "PrivateNetwork").(bool),
				EthernetNetworkType: d.Tc.GetTestData(d.Env, "EthernetNetworkType").(string),
				Type:                d.Tc.GetTestData(d.Env, "Type").(string),
			}
			err := c.CreateEthernetNetwork(testEthNet)
			assert.NoError(t, err, "CreateEthernetNetwork error -> %s", err)

			err = c.CreateEthernetNetwork(testEthNet)
			assert.Error(t, err, "CreateEthernetNetwork should error because the EthernetNetwork already exists, err-> %s", err)

		} else {
			log.Warnf("The ethernetNetwork already exist, so skipping CreateEthernetNetwork test for %s", testName)
		}

		// reload the test profile that we just created
		testEthNet, err = c.GetEthernetNetworkByName(testName)
		assert.NoError(t, err, "GetEthernetNetwork error -> %s", err)
	}

}

func TestGetEthernetNetworkByName(t *testing.T) {
	var (
		d        *OVTest
		c        *ov.OVClient
		testName string
	)
	if os.Getenv("ONEVIEW_TEST_ACCEPTANCE") == "true" {
		d, c = getTestDriverA("test_ethernet_network")
		if c == nil {
			t.Fatalf("Failed to execute getTestDriver() ")
		}
		testName = d.Tc.GetTestData(d.Env, "Name").(string)

		testEthNet, err := c.GetEthernetNetworkByName(testName)
		assert.NoError(t, err, "GetEthernetNetworkByName thew an error -> %s", err)
		assert.Equal(t, testName, testEthNet.Name)

		testEthNet, err = c.GetEthernetNetworkByName("bad")
		assert.NoError(t, err, "GetEthernetNetworkByName with fake name -> %s", err)
		assert.Equal(t, "", testEthNet.Name)

	} else {
		d, c = getTestDriverU("test_ethernet_network")
		testName = d.Tc.GetTestData(d.Env, "Name").(string)
		data, err := c.GetEthernetNetworkByName(testName)
		assert.Error(t, err, fmt.Sprintf("ALL ok, no error, caught as expected: %s,%+v\n", err, data))
	}
}

func TestGetEthernetNetworks(t *testing.T) {
	var (
		c *ov.OVClient
	)
	if os.Getenv("ONEVIEW_TEST_ACCEPTANCE") == "true" {
		_, c = getTestDriverA("test_ethernet_network")
		if c == nil {
			t.Fatalf("Failed to execute getTestDriver() ")
		}
		ethernetNetworks, err := c.GetEthernetNetworks("", "", "", "")
		assert.NoError(t, err, "GetEthernetNetworks threw error -> %s, %+v\n", err, ethernetNetworks)

		ethernetNetworks, err = c.GetEthernetNetworks("", "", "", "name:asc")
		assert.NoError(t, err, "GetEthernetNetworks name:asc error -> %s, %+v\n", err, ethernetNetworks)

	} else {
		_, c = getTestDriverU("test_ethernet_network")
		data, err := c.GetEthernetNetworks("", "", "", "")
		assert.Error(t, err, fmt.Sprintf("ALL ok, no error, caught as expected: %s,%+v\n", err, data))
	}
}

func TestDeleteEthernetNetworkNotFound(t *testing.T) {
	var (
		c          *ov.OVClient
		testName   = "fake"
		testEthNet ov.EthernetNetwork
	)
	if os.Getenv("ONEVIEW_TEST_ACCEPTANCE") == "true" {
		_, c = getTestDriverA("test_ethernet_network")
		if c == nil {
			t.Fatalf("Failed to execute getTestDriver() ")
		}

		err := c.DeleteEthernetNetwork(testName)
		assert.NoError(t, err, "DeleteEthernetNetwork err-> %s", err)

		testEthNet, err = c.GetEthernetNetworkByName(testName)
		assert.NoError(t, err, "GetEthernetNetworkByName with deleted ethernet network -> %+v", err)
		assert.Equal(t, "", testEthNet.Name, fmt.Sprintf("Problem getting ethernet name, %+v", testEthNet))
	} else {
		_, c = getTestDriverU("test_ethernet_network")
		err := c.DeleteEthernetNetwork(testName)
		assert.Error(t, err, fmt.Sprintf("All ok, no error, caught as expected: %s,%+v\n", err, testEthNet))
	}
}

func TestDeleteEthernetNetwork(t *testing.T) {
	var (
		d          *OVTest
		c          *ov.OVClient
		testName   string
		testEthNet ov.EthernetNetwork
	)
	if os.Getenv("ONEVIEW_TEST_ACCEPTANCE") == "true" {
		d, c = getTestDriverA("test_ethernet_network")
		if c == nil {
			t.Fatalf("Failed to execute getTestDriver() ")
		}
		testName = d.Tc.GetTestData(d.Env, "Name").(string)

		err := c.DeleteEthernetNetwork(testName)
		assert.NoError(t, err, "DeleteEthernetNetwork err-> %s", err)

		testEthNet, err = c.GetEthernetNetworkByName(testName)
		assert.NoError(t, err, "GetEthernetNetworkByName with deleted ethernet network-> %+v", err)
		assert.Equal(t, "", testEthNet.Name, fmt.Sprintf("Problem getting ethnet name, %+v", testEthNet))
	} else {
		_, c = getTestDriverU("test_ethernet_network")
		err := c.DeleteEthernetNetwork("footest")
		assert.Error(t, err, fmt.Sprintf("ALL ok, no error, caught as expected: %s,%+v\n", err, testEthNet))
	}

}
<|endoftext|>


<commit_before>package oschina

import (
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"

	"appengine"
	"appengine/urlfetch"
)

const (
	//catalog:1 for all
	//pageIndex: The first page.
	//pageSize:20
	API = "http://www.oschina.net/action/api/news_list?catalog=1&pageIndex=%d&pageSize=20"
)

func NewNewsList() (p *NewsList) {
	p = new(NewsList)
	return
}

type NewsList struct {
	XMLName     xml.Name    `xml:"oschina"`
	NewsEntries NewsEntries `xml:"newslist"`
}

type NewsEntries struct {
	Array []NewsEntry `xml:"news"`
}

type NewsEntry struct {
	Id           int    `xml:"id"`
	Title        string `xml:"title"`
	CommentCount int    `xml:"commentCount"`
	Author       string `xml:"author"`
	AuthorId     int    `xml:"authorid"`
	PubDate      string `xml:"pubDate"`
	Url          string `xml:"url"` //Might be empty then the news-type should be used to build a url
	UrlMobile    string
	NewsType     NewsType `xml:"newstype"`
}

type NewsType struct {
	Type int `xml:"type"`
}

//Create a news-list and return a json-feeds to client through channel.
func (self *NewsList) Create(cxt appengine.Context, page int, chJsonStr chan *string) {
	client := urlfetch.Client(cxt)
	if r, e := http.NewRequest("GET", fmt.Sprintf(API, page), nil); e == nil {
		if resp, e := client.Do(r); e == nil {
			if resp != nil {
				defer resp.Body.Close()
			}

			pNewsList := new(NewsList)
			if bytes, e := ioutil.ReadAll(resp.Body); e == nil {
				if e := xml.Unmarshal(bytes, pNewsList); e == nil {
					s := "[" //Start making a json result.
					for _, v := range pNewsList.NewsEntries.Array {
						v.Title = strings.Replace(v.Title, "\"", "'", -1)
						if v.Url == "" { //A Url might be null then we need change it self associated with its type.
							v.Url = fmt.Sprintf("http://www.oschina.net/news/%d", v.Id)
							v.UrlMobile = fmt.Sprintf("http://m.oschina.net/news/%d", v.Id)
						} else {
							v.UrlMobile = v.Url
						}
						json := fmt.Sprintf(`{"title" : "%s", "desc" : "%s", "url" : "%s", "url_mobile" : "%s",  "pubDate" : "%s" },`, v.Title, "", v.Url, v.UrlMobile, v.PubDate)
						cxt.Infof("Json: %s", json)
						s += json
					}
					s = s[:len(s)-1] //Remove last ","
					s += "]"         //Stop making json
					chJsonStr <- &s
				} else {
					chJsonStr <- nil
					cxt.Errorf("Error but still going: %v", e)
				}
			} else {
				chJsonStr <- nil
				panic(e)
			}
		} else {
			chJsonStr <- nil
			cxt.Errorf("Error but still going: %v", e)
		}
	} else {
		chJsonStr <- nil
		panic(e)
	}
}
<commit_msg>Fix for bottom of feeds of oschina.<commit_after>package oschina

import (
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"

	"appengine"
	"appengine/urlfetch"
)

const (
	//catalog:1 for all
	//pageIndex: The first page.
	//pageSize:20
	API = "http://www.oschina.net/action/api/news_list?catalog=1&pageIndex=%d&pageSize=20"
)

func NewNewsList() (p *NewsList) {
	p = new(NewsList)
	return
}

type NewsList struct {
	XMLName     xml.Name    `xml:"oschina"`
	NewsEntries NewsEntries `xml:"newslist"`
}

type NewsEntries struct {
	Array []NewsEntry `xml:"news"`
}

type NewsEntry struct {
	Id           int    `xml:"id"`
	Title        string `xml:"title"`
	CommentCount int    `xml:"commentCount"`
	Author       string `xml:"author"`
	AuthorId     int    `xml:"authorid"`
	PubDate      string `xml:"pubDate"`
	Url          string `xml:"url"` //Might be empty then the news-type should be used to build a url
	UrlMobile    string
	NewsType     NewsType `xml:"newstype"`
}

type NewsType struct {
	Type int `xml:"type"`
}

//Create a news-list and return a json-feeds to client through channel.
func (self *NewsList) Create(cxt appengine.Context, page int, chJsonStr chan *string) {
	client := urlfetch.Client(cxt)
	if r, e := http.NewRequest("GET", fmt.Sprintf(API, page), nil); e == nil {
		if resp, e := client.Do(r); e == nil {
			if resp != nil {
				defer resp.Body.Close()
			}

			pNewsList := new(NewsList)
			if bytes, e := ioutil.ReadAll(resp.Body); e == nil {
				if e := xml.Unmarshal(bytes, pNewsList); e == nil {
					s := "[" //Start making a json result.
					for _, v := range pNewsList.NewsEntries.Array {
						v.Title = strings.Replace(v.Title, "\"", "'", -1)
						if v.Url == "" { //A Url might be null then we need change it self associated with its type.
							v.Url = fmt.Sprintf("http://www.oschina.net/news/%d", v.Id)
							v.UrlMobile = fmt.Sprintf("http://m.oschina.net/news/%d", v.Id)
						} else {
							v.UrlMobile = v.Url
						}
						json := fmt.Sprintf(`{"title" : "%s", "desc" : "%s", "url" : "%s", "url_mobile" : "%s",  "pubDate" : "%s" },`, v.Title, "", v.Url, v.UrlMobile, v.PubDate)
						cxt.Infof("Json: %s", json)
						s += json
					}
					length := len(s)
					if length > 2 {
						s = s[:length-1] //Remove last ","
					}
					s += "]" //Stop making json
					chJsonStr <- &s
				} else {
					chJsonStr <- nil
					cxt.Errorf("Error but still going: %v", e)
				}
			} else {
				chJsonStr <- nil
				panic(e)
			}
		} else {
			chJsonStr <- nil
			cxt.Errorf("Error but still going: %v", e)
		}
	} else {
		chJsonStr <- nil
		panic(e)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd_test

import (
	"bytes"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"sort"
	"strings"
	"testing"

	"github.com/renstrom/dedent"
	"github.com/spf13/cobra"

	kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1"
	"k8s.io/kubernetes/cmd/kubeadm/app/cmd"
	"k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs"
	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
	"k8s.io/kubernetes/cmd/kubeadm/app/features"
	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
	utilruntime "k8s.io/kubernetes/cmd/kubeadm/app/util/runtime"
	"k8s.io/utils/exec"
	fakeexec "k8s.io/utils/exec/testing"
)

const (
	defaultNumberOfImages = 8
	// dummyKubernetesVersion is just used for unit testing, in order to not make
	// kubeadm lookup dl.k8s.io to resolve what the latest stable release is
	dummyKubernetesVersion = "v1.11.0"
)

func TestNewCmdConfigImagesList(t *testing.T) {
	var output bytes.Buffer
	mockK8sVersion := dummyKubernetesVersion
	images := cmd.NewCmdConfigImagesList(&output, &mockK8sVersion)
	images.Run(nil, nil)
	actual := strings.Split(output.String(), "\n")
	if len(actual) != defaultNumberOfImages {
		t.Fatalf("Expected %v but found %v images", defaultNumberOfImages, len(actual))
	}
}

func TestImagesListRunWithCustomConfigPath(t *testing.T) {
	testcases := []struct {
		name               string
		expectedImageCount int
		// each string provided here must appear in at least one image returned by Run
		expectedImageSubstrings []string
		configContents          []byte
	}{
		{
			name:               "set k8s version",
			expectedImageCount: defaultNumberOfImages,
			expectedImageSubstrings: []string{
				":v1.11.1",
			},
			configContents: []byte(dedent.Dedent(`
				apiVersion: kubeadm.k8s.io/v1beta1
				kind: ClusterConfiguration
				kubernetesVersion: v1.11.1
			`)),
		},
		{
			name:               "use coredns",
			expectedImageCount: defaultNumberOfImages,
			expectedImageSubstrings: []string{
				"coredns",
			},
			configContents: []byte(dedent.Dedent(`
				apiVersion: kubeadm.k8s.io/v1beta1
				kind: ClusterConfiguration
				kubernetesVersion: v1.11.0
				featureGates:
				  CoreDNS: true
			`)),
		},
	}

	for _, tc := range testcases {
		t.Run(tc.name, func(t *testing.T) {
			tmpDir, err := ioutil.TempDir("", "kubeadm-images-test")
			if err != nil {
				t.Fatalf("Unable to create temporary directory: %v", err)
			}
			defer os.RemoveAll(tmpDir)

			configFilePath := filepath.Join(tmpDir, "test-config-file")
			if err := ioutil.WriteFile(configFilePath, tc.configContents, 0644); err != nil {
				t.Fatalf("Failed writing a config file: %v", err)
			}

			i, err := cmd.NewImagesList(configFilePath, &kubeadmapiv1beta1.InitConfiguration{
				ClusterConfiguration: kubeadmapiv1beta1.ClusterConfiguration{
					KubernetesVersion: dummyKubernetesVersion,
				},
			})
			if err != nil {
				t.Fatalf("Failed getting the kubeadm images command: %v", err)
			}
			var output bytes.Buffer
			if i.Run(&output) != nil {
				t.Fatalf("Error from running the images command: %v", err)
			}
			actual := strings.Split(output.String(), "\n")
			if len(actual) != tc.expectedImageCount {
				t.Fatalf("did not get the same number of images: actual: %v expected: %v. Actual value: %v", len(actual), tc.expectedImageCount, actual)
			}

			for _, substring := range tc.expectedImageSubstrings {
				if !strings.Contains(output.String(), substring) {
					t.Errorf("Expected to find %v but did not in this list of images: %v", substring, actual)
				}
			}
		})
	}
}

func TestConfigImagesListRunWithoutPath(t *testing.T) {
	testcases := []struct {
		name           string
		cfg            kubeadmapiv1beta1.InitConfiguration
		expectedImages int
	}{
		{
			name:           "empty config",
			expectedImages: defaultNumberOfImages,
			cfg: kubeadmapiv1beta1.InitConfiguration{
				ClusterConfiguration: kubeadmapiv1beta1.ClusterConfiguration{
					KubernetesVersion: dummyKubernetesVersion,
				},
			},
		},
		{
			name: "external etcd configuration",
			cfg: kubeadmapiv1beta1.InitConfiguration{
				ClusterConfiguration: kubeadmapiv1beta1.ClusterConfiguration{
					Etcd: kubeadmapiv1beta1.Etcd{
						External: &kubeadmapiv1beta1.ExternalEtcd{
							Endpoints: []string{"https://some.etcd.com:2379"},
						},
					},
					KubernetesVersion: dummyKubernetesVersion,
				},
			},
			expectedImages: defaultNumberOfImages - 1,
		},
		{
			name: "coredns enabled",
			cfg: kubeadmapiv1beta1.InitConfiguration{
				ClusterConfiguration: kubeadmapiv1beta1.ClusterConfiguration{
					FeatureGates: map[string]bool{
						features.CoreDNS: true,
					},
					KubernetesVersion: dummyKubernetesVersion,
				},
			},
			expectedImages: defaultNumberOfImages,
		},
	}

	for _, tc := range testcases {
		t.Run(tc.name, func(t *testing.T) {
			i, err := cmd.NewImagesList("", &tc.cfg)
			if err != nil {
				t.Fatalf("did not expect an error while creating the Images command: %v", err)
			}

			var output bytes.Buffer
			if i.Run(&output) != nil {
				t.Fatalf("did not expect an error running the Images command: %v", err)
			}

			actual := strings.Split(output.String(), "\n")
			if len(actual) != tc.expectedImages {
				t.Fatalf("expected %v images but got %v", tc.expectedImages, actual)
			}
		})
	}
}

func TestImagesPull(t *testing.T) {
	fcmd := fakeexec.FakeCmd{
		CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{
			func() ([]byte, error) { return nil, nil },
			func() ([]byte, error) { return nil, nil },
			func() ([]byte, error) { return nil, nil },
			func() ([]byte, error) { return nil, nil },
			func() ([]byte, error) { return nil, nil },
		},
	}

	fexec := fakeexec.FakeExec{
		CommandScript: []fakeexec.FakeCommandAction{
			func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
			func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
			func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
			func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
			func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
		},
		LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/docker", nil },
	}

	containerRuntime, err := utilruntime.NewContainerRuntime(&fexec, kubeadmapiv1beta1.DefaultCRISocket)
	if err != nil {
		t.Errorf("unexpected NewContainerRuntime error: %v", err)
	}

	images := []string{"a", "b", "c", "d", "a"}
	ip := cmd.NewImagesPull(containerRuntime, images)

	err = ip.PullAll()
	if err != nil {
		t.Fatalf("expected nil but found %v", err)
	}

	if fcmd.CombinedOutputCalls != len(images) {
		t.Errorf("expected %d calls, got %d", len(images), fcmd.CombinedOutputCalls)
	}
}

func TestMigrate(t *testing.T) {
	cfg := []byte(dedent.Dedent(`
		# This is intentionally testing an old API version and the old kind naming and making sure the output is correct
		apiVersion: kubeadm.k8s.io/v1alpha3
		kind: InitConfiguration
		kubernetesVersion: v1.12.0
	`))
	configFile, cleanup := tempConfig(t, cfg)
	defer cleanup()

	var output bytes.Buffer
	command := cmd.NewCmdConfigMigrate(&output)
	if err := command.Flags().Set("old-config", configFile); err != nil {
		t.Fatalf("failed to set old-config flag")
	}
	newConfigPath := filepath.Join(filepath.Dir(configFile), "new-migrated-config")
	if err := command.Flags().Set("new-config", newConfigPath); err != nil {
		t.Fatalf("failed to set new-config flag")
	}
	command.Run(nil, nil)
	if _, err := configutil.ConfigFileAndDefaultsToInternalConfig(newConfigPath, &kubeadmapiv1beta1.InitConfiguration{}); err != nil {
		t.Fatalf("Could not read output back into internal type: %v", err)
	}
}

// Returns the name of the file created and a cleanup callback
func tempConfig(t *testing.T, config []byte) (string, func()) {
	t.Helper()
	tmpDir, err := ioutil.TempDir("", "kubeadm-migration-test")
	if err != nil {
		t.Fatalf("Unable to create temporary directory: %v", err)
	}
	configFilePath := filepath.Join(tmpDir, "test-config-file")
	if err := ioutil.WriteFile(configFilePath, config, 0644); err != nil {
		os.RemoveAll(tmpDir)
		t.Fatalf("Failed writing a config file: %v", err)
	}
	return configFilePath, func() {
		os.RemoveAll(tmpDir)
	}
}

func TestNewCmdConfigPrintActionDefaults(t *testing.T) {
	tests := []struct {
		name             string
		expectedKinds    []string // need to be sorted
		componentConfigs string
		cmdProc          func(out io.Writer) *cobra.Command
	}{
		{
			name: "InitConfiguration: No component configs",
			expectedKinds: []string{
				constants.ClusterConfigurationKind,
				constants.InitConfigurationKind,
			},
			cmdProc: cmd.NewCmdConfigPrintInitDefaults,
		},
		{
			name: "InitConfiguration: KubeProxyConfiguration",
			expectedKinds: []string{
				constants.ClusterConfigurationKind,
				constants.InitConfigurationKind,
				string(componentconfigs.KubeProxyConfigurationKind),
			},
			componentConfigs: "KubeProxyConfiguration",
			cmdProc:          cmd.NewCmdConfigPrintInitDefaults,
		},
		{
			name: "InitConfiguration: KubeProxyConfiguration and KubeletConfiguration",
			expectedKinds: []string{
				constants.ClusterConfigurationKind,
				constants.InitConfigurationKind,
				string(componentconfigs.KubeProxyConfigurationKind),
				string(componentconfigs.KubeletConfigurationKind),
			},
			componentConfigs: "KubeProxyConfiguration,KubeletConfiguration",
			cmdProc:          cmd.NewCmdConfigPrintInitDefaults,
		},
		{
			name: "JoinConfiguration: No component configs",
			expectedKinds: []string{
				constants.JoinConfigurationKind,
			},
			cmdProc: cmd.NewCmdConfigPrintJoinDefaults,
		},
		{
			name: "JoinConfiguration: KubeProxyConfiguration",
			expectedKinds: []string{
				constants.JoinConfigurationKind,
				string(componentconfigs.KubeProxyConfigurationKind),
			},
			componentConfigs: "KubeProxyConfiguration",
			cmdProc:          cmd.NewCmdConfigPrintJoinDefaults,
		},
		{
			name: "JoinConfiguration: KubeProxyConfiguration and KubeletConfiguration",
			expectedKinds: []string{
				constants.JoinConfigurationKind,
				string(componentconfigs.KubeProxyConfigurationKind),
				string(componentconfigs.KubeletConfigurationKind),
			},
			componentConfigs: "KubeProxyConfiguration,KubeletConfiguration",
			cmdProc:          cmd.NewCmdConfigPrintJoinDefaults,
		},
	}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			var output bytes.Buffer

			command := test.cmdProc(&output)
			if err := command.Flags().Set("component-configs", test.componentConfigs); err != nil {
				t.Fatalf("failed to set component-configs flag")
			}
			command.Run(nil, nil)

			gvkmap, err := kubeadmutil.SplitYAMLDocuments(output.Bytes())
			if err != nil {
				t.Fatalf("unexpected failure of SplitYAMLDocuments: %v", err)
			}

			gotKinds := []string{}
			for gvk := range gvkmap {
				gotKinds = append(gotKinds, gvk.Kind)
			}

			sort.Strings(gotKinds)

			if !reflect.DeepEqual(gotKinds, test.expectedKinds) {
				t.Fatalf("kinds not matching:\n\texpectedKinds: %v\n\tgotKinds: %v\n", test.expectedKinds, gotKinds)
			}
		})
	}
}
<commit_msg>Swap cmd_test to cmd package in kubeadm/app/cmd/config_test.go<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"bytes"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"sort"
	"strings"
	"testing"

	"github.com/renstrom/dedent"
	"github.com/spf13/cobra"

	kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1"
	"k8s.io/kubernetes/cmd/kubeadm/app/componentconfigs"
	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
	"k8s.io/kubernetes/cmd/kubeadm/app/features"
	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
	utilruntime "k8s.io/kubernetes/cmd/kubeadm/app/util/runtime"
	"k8s.io/utils/exec"
	fakeexec "k8s.io/utils/exec/testing"
)

const (
	defaultNumberOfImages = 8
	// dummyKubernetesVersion is just used for unit testing, in order to not make
	// kubeadm lookup dl.k8s.io to resolve what the latest stable release is
	dummyKubernetesVersion = "v1.11.0"
)

func TestNewCmdConfigImagesList(t *testing.T) {
	var output bytes.Buffer
	mockK8sVersion := dummyKubernetesVersion
	images := NewCmdConfigImagesList(&output, &mockK8sVersion)
	images.Run(nil, nil)
	actual := strings.Split(output.String(), "\n")
	if len(actual) != defaultNumberOfImages {
		t.Fatalf("Expected %v but found %v images", defaultNumberOfImages, len(actual))
	}
}

func TestImagesListRunWithCustomConfigPath(t *testing.T) {
	testcases := []struct {
		name               string
		expectedImageCount int
		// each string provided here must appear in at least one image returned by Run
		expectedImageSubstrings []string
		configContents          []byte
	}{
		{
			name:               "set k8s version",
			expectedImageCount: defaultNumberOfImages,
			expectedImageSubstrings: []string{
				":v1.11.1",
			},
			configContents: []byte(dedent.Dedent(`
				apiVersion: kubeadm.k8s.io/v1beta1
				kind: ClusterConfiguration
				kubernetesVersion: v1.11.1
			`)),
		},
		{
			name:               "use coredns",
			expectedImageCount: defaultNumberOfImages,
			expectedImageSubstrings: []string{
				"coredns",
			},
			configContents: []byte(dedent.Dedent(`
				apiVersion: kubeadm.k8s.io/v1beta1
				kind: ClusterConfiguration
				kubernetesVersion: v1.11.0
				featureGates:
				  CoreDNS: true
			`)),
		},
	}

	for _, tc := range testcases {
		t.Run(tc.name, func(t *testing.T) {
			tmpDir, err := ioutil.TempDir("", "kubeadm-images-test")
			if err != nil {
				t.Fatalf("Unable to create temporary directory: %v", err)
			}
			defer os.RemoveAll(tmpDir)

			configFilePath := filepath.Join(tmpDir, "test-config-file")
			if err := ioutil.WriteFile(configFilePath, tc.configContents, 0644); err != nil {
				t.Fatalf("Failed writing a config file: %v", err)
			}

			i, err := NewImagesList(configFilePath, &kubeadmapiv1beta1.InitConfiguration{
				ClusterConfiguration: kubeadmapiv1beta1.ClusterConfiguration{
					KubernetesVersion: dummyKubernetesVersion,
				},
			})
			if err != nil {
				t.Fatalf("Failed getting the kubeadm images command: %v", err)
			}
			var output bytes.Buffer
			if i.Run(&output) != nil {
				t.Fatalf("Error from running the images command: %v", err)
			}
			actual := strings.Split(output.String(), "\n")
			if len(actual) != tc.expectedImageCount {
				t.Fatalf("did not get the same number of images: actual: %v expected: %v. Actual value: %v", len(actual), tc.expectedImageCount, actual)
			}

			for _, substring := range tc.expectedImageSubstrings {
				if !strings.Contains(output.String(), substring) {
					t.Errorf("Expected to find %v but did not in this list of images: %v", substring, actual)
				}
			}
		})
	}
}

func TestConfigImagesListRunWithoutPath(t *testing.T) {
	testcases := []struct {
		name           string
		cfg            kubeadmapiv1beta1.InitConfiguration
		expectedImages int
	}{
		{
			name:           "empty config",
			expectedImages: defaultNumberOfImages,
			cfg: kubeadmapiv1beta1.InitConfiguration{
				ClusterConfiguration: kubeadmapiv1beta1.ClusterConfiguration{
					KubernetesVersion: dummyKubernetesVersion,
				},
			},
		},
		{
			name: "external etcd configuration",
			cfg: kubeadmapiv1beta1.InitConfiguration{
				ClusterConfiguration: kubeadmapiv1beta1.ClusterConfiguration{
					Etcd: kubeadmapiv1beta1.Etcd{
						External: &kubeadmapiv1beta1.ExternalEtcd{
							Endpoints: []string{"https://some.etcd.com:2379"},
						},
					},
					KubernetesVersion: dummyKubernetesVersion,
				},
			},
			expectedImages: defaultNumberOfImages - 1,
		},
		{
			name: "coredns enabled",
			cfg: kubeadmapiv1beta1.InitConfiguration{
				ClusterConfiguration: kubeadmapiv1beta1.ClusterConfiguration{
					FeatureGates: map[string]bool{
						features.CoreDNS: true,
					},
					KubernetesVersion: dummyKubernetesVersion,
				},
			},
			expectedImages: defaultNumberOfImages,
		},
	}

	for _, tc := range testcases {
		t.Run(tc.name, func(t *testing.T) {
			i, err := NewImagesList("", &tc.cfg)
			if err != nil {
				t.Fatalf("did not expect an error while creating the Images command: %v", err)
			}

			var output bytes.Buffer
			if i.Run(&output) != nil {
				t.Fatalf("did not expect an error running the Images command: %v", err)
			}

			actual := strings.Split(output.String(), "\n")
			if len(actual) != tc.expectedImages {
				t.Fatalf("expected %v images but got %v", tc.expectedImages, actual)
			}
		})
	}
}

func TestImagesPull(t *testing.T) {
	fcmd := fakeexec.FakeCmd{
		CombinedOutputScript: []fakeexec.FakeCombinedOutputAction{
			func() ([]byte, error) { return nil, nil },
			func() ([]byte, error) { return nil, nil },
			func() ([]byte, error) { return nil, nil },
			func() ([]byte, error) { return nil, nil },
			func() ([]byte, error) { return nil, nil },
		},
	}

	fexec := fakeexec.FakeExec{
		CommandScript: []fakeexec.FakeCommandAction{
			func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
			func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
			func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
			func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
			func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) },
		},
		LookPathFunc: func(cmd string) (string, error) { return "/usr/bin/docker", nil },
	}

	containerRuntime, err := utilruntime.NewContainerRuntime(&fexec, kubeadmapiv1beta1.DefaultCRISocket)
	if err != nil {
		t.Errorf("unexpected NewContainerRuntime error: %v", err)
	}

	images := []string{"a", "b", "c", "d", "a"}
	ip := NewImagesPull(containerRuntime, images)

	err = ip.PullAll()
	if err != nil {
		t.Fatalf("expected nil but found %v", err)
	}

	if fcmd.CombinedOutputCalls != len(images) {
		t.Errorf("expected %d calls, got %d", len(images), fcmd.CombinedOutputCalls)
	}
}

func TestMigrate(t *testing.T) {
	cfg := []byte(dedent.Dedent(`
		# This is intentionally testing an old API version and the old kind naming and making sure the output is correct
		apiVersion: kubeadm.k8s.io/v1alpha3
		kind: InitConfiguration
		kubernetesVersion: v1.12.0
	`))
	configFile, cleanup := tempConfig(t, cfg)
	defer cleanup()

	var output bytes.Buffer
	command := NewCmdConfigMigrate(&output)
	if err := command.Flags().Set("old-config", configFile); err != nil {
		t.Fatalf("failed to set old-config flag")
	}
	newConfigPath := filepath.Join(filepath.Dir(configFile), "new-migrated-config")
	if err := command.Flags().Set("new-config", newConfigPath); err != nil {
		t.Fatalf("failed to set new-config flag")
	}
	command.Run(nil, nil)
	if _, err := configutil.ConfigFileAndDefaultsToInternalConfig(newConfigPath, &kubeadmapiv1beta1.InitConfiguration{}); err != nil {
		t.Fatalf("Could not read output back into internal type: %v", err)
	}
}

// Returns the name of the file created and a cleanup callback
func tempConfig(t *testing.T, config []byte) (string, func()) {
	t.Helper()
	tmpDir, err := ioutil.TempDir("", "kubeadm-migration-test")
	if err != nil {
		t.Fatalf("Unable to create temporary directory: %v", err)
	}
	configFilePath := filepath.Join(tmpDir, "test-config-file")
	if err := ioutil.WriteFile(configFilePath, config, 0644); err != nil {
		os.RemoveAll(tmpDir)
		t.Fatalf("Failed writing a config file: %v", err)
	}
	return configFilePath, func() {
		os.RemoveAll(tmpDir)
	}
}

func TestNewCmdConfigPrintActionDefaults(t *testing.T) {
	tests := []struct {
		name             string
		expectedKinds    []string // need to be sorted
		componentConfigs string
		cmdProc          func(out io.Writer) *cobra.Command
	}{
		{
			name: "InitConfiguration: No component configs",
			expectedKinds: []string{
				constants.ClusterConfigurationKind,
				constants.InitConfigurationKind,
			},
			cmdProc: NewCmdConfigPrintInitDefaults,
		},
		{
			name: "InitConfiguration: KubeProxyConfiguration",
			expectedKinds: []string{
				constants.ClusterConfigurationKind,
				constants.InitConfigurationKind,
				string(componentconfigs.KubeProxyConfigurationKind),
			},
			componentConfigs: "KubeProxyConfiguration",
			cmdProc:          NewCmdConfigPrintInitDefaults,
		},
		{
			name: "InitConfiguration: KubeProxyConfiguration and KubeletConfiguration",
			expectedKinds: []string{
				constants.ClusterConfigurationKind,
				constants.InitConfigurationKind,
				string(componentconfigs.KubeProxyConfigurationKind),
				string(componentconfigs.KubeletConfigurationKind),
			},
			componentConfigs: "KubeProxyConfiguration,KubeletConfiguration",
			cmdProc:          NewCmdConfigPrintInitDefaults,
		},
		{
			name: "JoinConfiguration: No component configs",
			expectedKinds: []string{
				constants.JoinConfigurationKind,
			},
			cmdProc: NewCmdConfigPrintJoinDefaults,
		},
		{
			name: "JoinConfiguration: KubeProxyConfiguration",
			expectedKinds: []string{
				constants.JoinConfigurationKind,
				string(componentconfigs.KubeProxyConfigurationKind),
			},
			componentConfigs: "KubeProxyConfiguration",
			cmdProc:          NewCmdConfigPrintJoinDefaults,
		},
		{
			name: "JoinConfiguration: KubeProxyConfiguration and KubeletConfiguration",
			expectedKinds: []string{
				constants.JoinConfigurationKind,
				string(componentconfigs.KubeProxyConfigurationKind),
				string(componentconfigs.KubeletConfigurationKind),
			},
			componentConfigs: "KubeProxyConfiguration,KubeletConfiguration",
			cmdProc:          NewCmdConfigPrintJoinDefaults,
		},
	}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			var output bytes.Buffer

			command := test.cmdProc(&output)
			if err := command.Flags().Set("component-configs", test.componentConfigs); err != nil {
				t.Fatalf("failed to set component-configs flag")
			}
			command.Run(nil, nil)

			gvkmap, err := kubeadmutil.SplitYAMLDocuments(output.Bytes())
			if err != nil {
				t.Fatalf("unexpected failure of SplitYAMLDocuments: %v", err)
			}

			gotKinds := []string{}
			for gvk := range gvkmap {
				gotKinds = append(gotKinds, gvk.Kind)
			}

			sort.Strings(gotKinds)

			if !reflect.DeepEqual(gotKinds, test.expectedKinds) {
				t.Fatalf("kinds not matching:\n\texpectedKinds: %v\n\tgotKinds: %v\n", test.expectedKinds, gotKinds)
			}
		})
	}
}
<|endoftext|>


<commit_before>package main

import "github.com/coreos/cobra"

var (
	shortbreadCtl *cobra.Command
	user          string
	key           string
	baseUrl       string
)

func init() {
	shortbreadCtl = &cobra.Command{
		Use:   "shortbreadctl",
		Short: "A command line tool to interact with the CA server and issue/revoke/modify user and host certificates",
	}
	shortbreadCtl.PersistentFlags().StringVarP(&user, "username", "u", "", "username of the entity to whom the certificate is issued")
	shortbreadCtl.PersistentFlags().StringVarP(&key, "key", "k", "", "bears the path to the public key that will be signed by the CA's private key")
	shortbreadCtl.PersistentFlags().StringVarP(&baseUrl, "server", "s", "", "base url for the CA server")
}

func main() {
	shortbreadCtl.AddCommand(addUser)
	shortbreadCtl.AddCommand(revokeUser)
	shortbreadCtl.Execute()
}

// bin/shortbreadctl adduser -k /Users/shantanu/.ssh/id_rsa.pub -p /Users/shantanu/.ssh/users_ca -u shantanu -e permit-pty -c USER
<commit_msg>rolled update user and add user into a single command<commit_after>package main

import "github.com/coreos/cobra"

var (
	shortbreadCtl *cobra.Command
	user          string
	key           string
	baseUrl       string
)

func init() {
	shortbreadCtl = &cobra.Command{
		Use:   "shortbreadctl",
		Short: "A command line tool to interact with the CA server and issue/revoke/modify user and host certificates",
	}
	shortbreadCtl.PersistentFlags().StringVarP(&user, "username", "u", "", "username of the entity to whom the certificate is issued")
	shortbreadCtl.PersistentFlags().StringVarP(&key, "key", "k", "", "bears the path to the public key that will be signed by the CA's private key")
	shortbreadCtl.PersistentFlags().StringVarP(&baseUrl, "server", "s", "", "base url for the CA server")
}

func main() {
	shortbreadCtl.AddCommand(updateUser)
	shortbreadCtl.AddCommand(revokeUser)
	shortbreadCtl.Execute()
}

// bin/shortbreadctl adduser -k /Users/shantanu/.ssh/id_rsa.pub -p /Users/shantanu/.ssh/users_ca -u shantanu -e permit-pty -c USER
<|endoftext|>