<commit_before>/*
 * Copyright (C) 2013 Matthew Dawson <matthew@mjdsystems.ca>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
package main

import (
	"errors"

	"net/url"

	"sort"

	riak "github.com/tpjg/goriakpbc"
)

var FeedNotFound = errors.New("Failed to find feed in riak!")

const (
	MaximumFeedItems = 10000
)

func drainErrorChannelIntoSlice(errCh <-chan error, errorSlice *[]error, responses int) {
	for i := 0; i < responses; i++ {
		err := <-errCh
		if err != nil {
			*errorSlice = append(*errorSlice, err)
		}
	}
}

func InsertItem(con *riak.Client, itemKey ItemKey, item ParsedFeedItem) error {
	itemModel := FeedItem{
		Title:   item.Title,
		Author:  item.Author,
		Content: item.Content,
		Url:     item.Url,
		PubDate: item.PubDate,
	}
	if err := con.LoadModel(itemKey.GetRiakKey(), &itemModel); err != riak.NotFound {
		return err
	} else if err = itemModel.Save(); err != nil {
		return err
	}
	return nil
}

func UpdateItem(con *riak.Client, itemKey ItemKey, item ParsedFeedItem, itemModel *FeedItem) error {
	itemModel.Title = item.Title
	itemModel.Author = item.Author
	itemModel.Content = item.Content
	itemModel.Url = item.Url
	itemModel.PubDate = item.PubDate

	if err := itemModel.Save(); err != nil {
		return err
	}
	return nil
}

func itemDiffersFromModel(feedItem ParsedFeedItem, itemModel *FeedItem) bool {
	return itemModel.Title != feedItem.Title ||
		itemModel.Author != feedItem.Author ||
		itemModel.Content != feedItem.Content ||
		itemModel.Url != feedItem.Url ||
		itemModel.PubDate != feedItem.PubDate
}

func updateFeed(con *riak.Client, feedUrl url.URL, feedData ParsedFeedData, ids <-chan uint64) error {
	feed := &Feed{Url: feedUrl}
	if err := con.LoadModel(feed.UrlKey(), feed); err == riak.NotFound {
		return FeedNotFound
	} else if err != nil {
		return err
	}
	// First clean out inserted item keys.  This handles unfinished previous operations.
	itemsBucket, err := con.Bucket("items")
	if err != nil {
		return err
	}
	for _, itemKey := range feed.InsertedItemKeys {
		// Does this item exist?
		if ok, err := itemsBucket.Exists(itemKey.GetRiakKey()); err != nil {
			return err
		} else if ok {
			// Yep, so add it to the list.
			feed.ItemKeys = append(feed.ItemKeys, itemKey)
		}
		// Otherwise non-existent items are dropped.  This is to avoid
	}
	feed.InsertedItemKeys = nil

	// Next update the basic attributes (title basically)
	feed.Title = feedData.Title

	/* Next find all the feed items to insert/update.  If the item doesn't exist, create it's id and
	 * mark for insert.  Otherwise mark it for an read/update/store pass.  Make sure to mark for
	 * deletion items as necessary.
	 */
	// This struct holds an ItemKey and a ParsedFeedItem for later parsing.
	type ToProcess struct {
		ItemKey ItemKey
		Data    ParsedFeedItem
		Model   *FeedItem
	}
	NewItems := make([]ToProcess, 0)
	UpdatedItems := make([]ToProcess, 0)

	for _, rawItem := range feedData.Items {
		// Try to find the raw Item in the Item Keys list.
		index := feed.ItemKeys.FindRawItemId(rawItem.GenericKey)
		if index != -1 {
			// Found it!  Load the details.  Also load the model, which will be re-used later.
			p := ToProcess{
				ItemKey: feed.ItemKeys[index],
				Data:    rawItem,
				Model:   &FeedItem{},
			}

			if err := con.LoadModel(p.ItemKey.GetRiakKey(), p.Model); err != nil {
				return err
			}

			// Ok, now is this have a new pub date?  If so, pull it out of its current position, and
			// move it up the chain.  Otherwise, just update the content.  If an item has no pub date,
			// assume that it has changed if the any part of the item changed.
			if p.Model.PubDate.Equal(p.Data.PubDate) && !(p.Data.PubDate.IsZero() && itemDiffersFromModel(p.Data, p.Model)) {
				// Pub dates are the same.  Just modify the item to match what is in the feed.
				UpdatedItems = append(UpdatedItems, p)
			} else {
				// Pub dates differ.  Delete the item, and re-insert it.
				feed.DeletedItemKeys = append(feed.DeletedItemKeys, p.ItemKey)
				feed.ItemKeys.RemoveAt(index)

				// Delete the model from the to process struct.
				p.Model = &FeedItem{}

				NewItems = append(NewItems, p) // This gives us the new id.
			}
		} else {
			// Nope, lets insert it!  First, should we knock off an item?  e need to stay below MaximumFeedItems.
			for (len(feed.ItemKeys)+len(NewItems)) >= MaximumFeedItems && len(feed.ItemKeys) > 0 {
				// Need to kill an item.  So get the last key
				lastKey := feed.ItemKeys[len(feed.ItemKeys)-1]
				// insert it onto the end of the deleted item list.
				feed.DeletedItemKeys = append(feed.DeletedItemKeys, lastKey)
				// If we are updating this key, then remove it from this list.  No need to waste
				// time.
				for i, item := range UpdatedItems {
					if item.ItemKey.Equal(lastKey) {
						UpdatedItems = append(UpdatedItems[:i], UpdatedItems[i+1:]...)
					}
				}
				// And finally, pop the item
				feed.ItemKeys = feed.ItemKeys[:len(feed.ItemKeys)-1]
			}
			// Only insert if there are less then MaximumFeedItems already to be inserted.
			// This works since any later item will have been updated after.
			if len(NewItems) < MaximumFeedItems {
				NewItems = append(NewItems, ToProcess{
					Data: rawItem,
				})
			}
		}
	}

	/* Alright, any new items are mentioned in the Feed before being inserted.  In case something
	 * happens, I'd prefer not to lose an item.  Note the order is reversed so that the oldest story
	 * will get the smallest id, preserving sort order.  Inserted Item Keys needs to be sorted (well,
	 * reversed) after this so it is in correct order as well.  This loop violates ItemKeys sort
	 * order, so the sort is necessary for now. */
	for i := len(NewItems) - 1; i >= 0; i-- {
		newItem := &NewItems[i]
		newItem.ItemKey = NewItemKey(<-ids, newItem.Data.GenericKey)
		feed.InsertedItemKeys = append(feed.InsertedItemKeys, newItem.ItemKey)
	}
	sort.Sort(feed.InsertedItemKeys)

	// Ok, we must save here.  Otherwise planned changes may occur that will not be cleaned up!
	if err := feed.Save(); err != nil {
		return err
	}

	errCh := make(chan error) // All of the errors go into here, to be pulled out.

	// Good, now implement the change and update the Feed.

	// First add new items
	for _, newItem := range NewItems {
		feed.ItemKeys = append(feed.ItemKeys, newItem.ItemKey)
		go func(newItem ToProcess) {
			errCh <- InsertItem(con, newItem.ItemKey, newItem.Data)
		}(newItem)
	}
	feed.InsertedItemKeys = nil

	// Now update them.
	for _, newItem := range UpdatedItems {
		go func(newItem ToProcess) {
			errCh <- UpdateItem(con, newItem.ItemKey, newItem.Data, newItem.Model)
		}(newItem)
	}

	// Finally delete items.
	for _, deleteItemKey := range feed.DeletedItemKeys {
		go func(toDelete ItemKey) {
			if obj, err := itemsBucket.Get(toDelete.GetRiakKey()); obj == nil {
				errCh <- err
			} else {
				errCh <- obj.Destroy()
			}
		}(deleteItemKey)
	}
	deletedItemCount := len(feed.DeletedItemKeys) // Need this to drain the error channel later.
	// Ok, deleted.  So clear the list
	feed.DeletedItemKeys = nil

	sort.Sort(sort.Reverse(feed.ItemKeys)) // Just sort this.  TBD: Actually maintain this sort order to avoid this!

	//Now, collect the errors
	var errs []error
	drainErrorChannelIntoSlice(errCh, &errs, len(NewItems))
	drainErrorChannelIntoSlice(errCh, &errs, len(UpdatedItems))
	drainErrorChannelIntoSlice(errCh, &errs, deletedItemCount)
	if len(errs) != 0 {
		return MultiError(errs)
	}

	if err := feed.Save(); err != nil {
		return err
	}

	_, _ = NewItems, UpdatedItems
	return nil
}
<commit_msg>Add a comment about possible size explosion of feed items.<commit_after>/*
 * Copyright (C) 2013 Matthew Dawson <matthew@mjdsystems.ca>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
package main

import (
	"errors"

	"net/url"

	"sort"

	riak "github.com/tpjg/goriakpbc"
)

var FeedNotFound = errors.New("Failed to find feed in riak!")

const (
	MaximumFeedItems = 10000
)

func drainErrorChannelIntoSlice(errCh <-chan error, errorSlice *[]error, responses int) {
	for i := 0; i < responses; i++ {
		err := <-errCh
		if err != nil {
			*errorSlice = append(*errorSlice, err)
		}
	}
}

func InsertItem(con *riak.Client, itemKey ItemKey, item ParsedFeedItem) error {
	itemModel := FeedItem{
		Title:   item.Title,
		Author:  item.Author,
		Content: item.Content,
		Url:     item.Url,
		PubDate: item.PubDate,
	}
	if err := con.LoadModel(itemKey.GetRiakKey(), &itemModel); err != riak.NotFound {
		return err
	} else if err = itemModel.Save(); err != nil {
		return err
	}
	return nil
}

func UpdateItem(con *riak.Client, itemKey ItemKey, item ParsedFeedItem, itemModel *FeedItem) error {
	itemModel.Title = item.Title
	itemModel.Author = item.Author
	itemModel.Content = item.Content
	itemModel.Url = item.Url
	itemModel.PubDate = item.PubDate

	if err := itemModel.Save(); err != nil {
		return err
	}
	return nil
}

func itemDiffersFromModel(feedItem ParsedFeedItem, itemModel *FeedItem) bool {
	return itemModel.Title != feedItem.Title ||
		itemModel.Author != feedItem.Author ||
		itemModel.Content != feedItem.Content ||
		itemModel.Url != feedItem.Url ||
		itemModel.PubDate != feedItem.PubDate
}

func updateFeed(con *riak.Client, feedUrl url.URL, feedData ParsedFeedData, ids <-chan uint64) error {
	feed := &Feed{Url: feedUrl}
	if err := con.LoadModel(feed.UrlKey(), feed); err == riak.NotFound {
		return FeedNotFound
	} else if err != nil {
		return err
	}
	// First clean out inserted item keys.  This handles unfinished previous operations.
	itemsBucket, err := con.Bucket("items")
	if err != nil {
		return err
	}

	// Note, this insert items without caring about the 10,000 limit.  Of course, any regular inserted
	// item will force the limit back down.
	for _, itemKey := range feed.InsertedItemKeys {
		// Does this item exist?
		if ok, err := itemsBucket.Exists(itemKey.GetRiakKey()); err != nil {
			return err
		} else if ok {
			// Yep, so add it to the list.
			feed.ItemKeys = append(feed.ItemKeys, itemKey)
		}
		// Otherwise non-existent items are dropped.  This is to avoid
	}
	feed.InsertedItemKeys = nil

	// Next update the basic attributes (title basically)
	feed.Title = feedData.Title

	/* Next find all the feed items to insert/update.  If the item doesn't exist, create it's id and
	 * mark for insert.  Otherwise mark it for an read/update/store pass.  Make sure to mark for
	 * deletion items as necessary.
	 */
	// This struct holds an ItemKey and a ParsedFeedItem for later parsing.
	type ToProcess struct {
		ItemKey ItemKey
		Data    ParsedFeedItem
		Model   *FeedItem
	}
	NewItems := make([]ToProcess, 0)
	UpdatedItems := make([]ToProcess, 0)

	for _, rawItem := range feedData.Items {
		// Try to find the raw Item in the Item Keys list.
		index := feed.ItemKeys.FindRawItemId(rawItem.GenericKey)
		if index != -1 {
			// Found it!  Load the details.  Also load the model, which will be re-used later.
			p := ToProcess{
				ItemKey: feed.ItemKeys[index],
				Data:    rawItem,
				Model:   &FeedItem{},
			}

			if err := con.LoadModel(p.ItemKey.GetRiakKey(), p.Model); err != nil {
				return err
			}

			// Ok, now is this have a new pub date?  If so, pull it out of its current position, and
			// move it up the chain.  Otherwise, just update the content.  If an item has no pub date,
			// assume that it has changed if the any part of the item changed.
			if p.Model.PubDate.Equal(p.Data.PubDate) && !(p.Data.PubDate.IsZero() && itemDiffersFromModel(p.Data, p.Model)) {
				// Pub dates are the same.  Just modify the item to match what is in the feed.
				UpdatedItems = append(UpdatedItems, p)
			} else {
				// Pub dates differ.  Delete the item, and re-insert it.
				feed.DeletedItemKeys = append(feed.DeletedItemKeys, p.ItemKey)
				feed.ItemKeys.RemoveAt(index)

				// Delete the model from the to process struct.
				p.Model = &FeedItem{}

				NewItems = append(NewItems, p) // This gives us the new id.
			}
		} else {
			// Nope, lets insert it!  First, should we knock off an item?  e need to stay below MaximumFeedItems.
			for (len(feed.ItemKeys)+len(NewItems)) >= MaximumFeedItems && len(feed.ItemKeys) > 0 {
				// Need to kill an item.  So get the last key
				lastKey := feed.ItemKeys[len(feed.ItemKeys)-1]
				// insert it onto the end of the deleted item list.
				feed.DeletedItemKeys = append(feed.DeletedItemKeys, lastKey)
				// If we are updating this key, then remove it from this list.  No need to waste
				// time.
				for i, item := range UpdatedItems {
					if item.ItemKey.Equal(lastKey) {
						UpdatedItems = append(UpdatedItems[:i], UpdatedItems[i+1:]...)
					}
				}
				// And finally, pop the item
				feed.ItemKeys = feed.ItemKeys[:len(feed.ItemKeys)-1]
			}
			// Only insert if there are less then MaximumFeedItems already to be inserted.
			// This works since any later item will have been updated after.
			if len(NewItems) < MaximumFeedItems {
				NewItems = append(NewItems, ToProcess{
					Data: rawItem,
				})
			}
		}
	}

	/* Alright, any new items are mentioned in the Feed before being inserted.  In case something
	 * happens, I'd prefer not to lose an item.  Note the order is reversed so that the oldest story
	 * will get the smallest id, preserving sort order.  Inserted Item Keys needs to be sorted (well,
	 * reversed) after this so it is in correct order as well.  This loop violates ItemKeys sort
	 * order, so the sort is necessary for now. */
	for i := len(NewItems) - 1; i >= 0; i-- {
		newItem := &NewItems[i]
		newItem.ItemKey = NewItemKey(<-ids, newItem.Data.GenericKey)
		feed.InsertedItemKeys = append(feed.InsertedItemKeys, newItem.ItemKey)
	}
	sort.Sort(feed.InsertedItemKeys)

	// Ok, we must save here.  Otherwise planned changes may occur that will not be cleaned up!
	if err := feed.Save(); err != nil {
		return err
	}

	errCh := make(chan error) // All of the errors go into here, to be pulled out.

	// Good, now implement the change and update the Feed.

	// First add new items
	for _, newItem := range NewItems {
		feed.ItemKeys = append(feed.ItemKeys, newItem.ItemKey)
		go func(newItem ToProcess) {
			errCh <- InsertItem(con, newItem.ItemKey, newItem.Data)
		}(newItem)
	}
	feed.InsertedItemKeys = nil

	// Now update them.
	for _, newItem := range UpdatedItems {
		go func(newItem ToProcess) {
			errCh <- UpdateItem(con, newItem.ItemKey, newItem.Data, newItem.Model)
		}(newItem)
	}

	// Finally delete items.
	for _, deleteItemKey := range feed.DeletedItemKeys {
		go func(toDelete ItemKey) {
			if obj, err := itemsBucket.Get(toDelete.GetRiakKey()); obj == nil {
				errCh <- err
			} else {
				errCh <- obj.Destroy()
			}
		}(deleteItemKey)
	}
	deletedItemCount := len(feed.DeletedItemKeys) // Need this to drain the error channel later.
	// Ok, deleted.  So clear the list
	feed.DeletedItemKeys = nil

	sort.Sort(sort.Reverse(feed.ItemKeys)) // Just sort this.  TBD: Actually maintain this sort order to avoid this!

	//Now, collect the errors
	var errs []error
	drainErrorChannelIntoSlice(errCh, &errs, len(NewItems))
	drainErrorChannelIntoSlice(errCh, &errs, len(UpdatedItems))
	drainErrorChannelIntoSlice(errCh, &errs, deletedItemCount)
	if len(errs) != 0 {
		return MultiError(errs)
	}

	if err := feed.Save(); err != nil {
		return err
	}

	_, _ = NewItems, UpdatedItems
	return nil
}
<|endoftext|>


<commit_before>package file

import (
	"errors"
	"fmt"
	"io"
	"mime/multipart"
	"os"
	"strconv"
	"strings"

	"github.com/gin-gonic/gin"
	x "github.com/javinc/mango/module"
)

const (
	uploadPath  = "./upload/"
	uploadField = "file"
)

// UploadHandler file
func UploadHandler(c *gin.Context) {
	os.Mkdir(uploadPath, 0777)

	file, header, err := c.Request.FormFile(uploadField)
	if err != nil {
		x.Error("FILE_UPLOAD_ERROR", err.Error())

		return
	}

	id := x.GenerateHash()
	ext := getExtension(header.Filename)
	name := id + "." + ext
	filePath := uploadPath + name
	out, err := os.Create(filePath)
	if err != nil {
		x.Error("FILE_UPLOAD_CREATE_ERROR", err.Error())

		return
	}

	defer out.Close()
	_, err = io.Copy(out, file)
	if err != nil {
		x.Error("FILE_UPLOAD_COPY_ERROR", err.Error())

		return
	}

	size, err := getFileSize(file)
	if err != nil {
		x.Error("FILE_UPLOAD_SIZE_ERROR", err.Error())

		return
	}

	// check mime type
	mime := header.Header.Get("Content-Type")
	if mime == "" {
		x.Error("FILE_UPLOAD_MIME_ERROR", "cant get file content-type")

		return
	}

	// save meta on database
	payload := Object{
		ID:   id,
		Ext:  ext,
		Size: size,
		Mime: mime,
	}

	service.Create(payload)

	x.Output(payload)
}

func getExtension(filename string) string {
	raw := strings.Split(filename, ".")
	return raw[len(raw)-1]
}

func getFileSize(file multipart.File) (int64, error) {
	type size interface {
		Size() int64
	}

	var fsize string
	var i64 int64

	if sizeInterface, ok := file.(size); ok {
		sizeInterface.Size()

		fsize = fmt.Sprintf("%d", sizeInterface.Size())
	} else {
		return i64, errors.New("cant get file size")
	}

	s, err := strconv.Atoi(fsize)
	if err != nil {
		return i64, err
	}

	return int64(s), nil
}

// DisposeHandler file disposition
func DisposeHandler(c *gin.Context) {
}
<commit_msg>including extension to id<commit_after>package file

import (
	"errors"
	"fmt"
	"io"
	"mime/multipart"
	"os"
	"strconv"
	"strings"

	"github.com/gin-gonic/gin"
	x "github.com/javinc/mango/module"
)

const (
	uploadPath  = "./upload/"
	uploadField = "file"
)

// UploadHandler file
func UploadHandler(c *gin.Context) {
	os.Mkdir(uploadPath, 0777)

	file, header, err := c.Request.FormFile(uploadField)
	if err != nil {
		x.Error("FILE_UPLOAD_ERROR", err.Error())

		return
	}

	ext := getExtension(header.Filename)
	name := x.GenerateHash() + "." + ext
	filePath := uploadPath + name
	out, err := os.Create(filePath)
	if err != nil {
		x.Error("FILE_UPLOAD_CREATE_ERROR", err.Error())

		return
	}

	defer out.Close()
	_, err = io.Copy(out, file)
	if err != nil {
		x.Error("FILE_UPLOAD_COPY_ERROR", err.Error())

		return
	}

	size, err := getFileSize(file)
	if err != nil {
		x.Error("FILE_UPLOAD_SIZE_ERROR", err.Error())

		return
	}

	// check mime type
	mime := header.Header.Get("Content-Type")
	if mime == "" {
		x.Error("FILE_UPLOAD_MIME_ERROR", "cant get file content-type")

		return
	}

	// save meta on database
	payload := Object{
		ID:   name,
		Ext:  ext,
		Size: size,
		Mime: mime,
	}

	service.Create(payload)

	x.Output(payload)
}

func getExtension(filename string) string {
	raw := strings.Split(filename, ".")
	return raw[len(raw)-1]
}

func getFileSize(file multipart.File) (int64, error) {
	type size interface {
		Size() int64
	}

	var fsize string
	var i64 int64

	if sizeInterface, ok := file.(size); ok {
		sizeInterface.Size()

		fsize = fmt.Sprintf("%d", sizeInterface.Size())
	} else {
		return i64, errors.New("cant get file size")
	}

	s, err := strconv.Atoi(fsize)
	if err != nil {
		return i64, err
	}

	return int64(s), nil
}

// DisposeHandler file disposition
func DisposeHandler(c *gin.Context) {
}
<|endoftext|>


<commit_before>// Copyright (c) 2010 AFP Authors
// This source code is released under the terms of the
// MIT license. Please see the file LICENSE for license details.

package delay

import (
	"afp"
	"afp/flags"
	"os"
	"math"
)

type DelayFilter struct {
	context *afp.Context
	header afp.StreamHeader
	samplesPerSecond int
	samplesPerMillisecond int
	delayTimeInMs int
	extraSamples int
	mixBufferSize int64
	bufferSize int32
	channels int16
	bytesPerSample int16
	buffers [][][]float32
	mixBuffer [][]float32
}

func NewDelayFilter() afp.Filter {
	return &DelayFilter{}
}

func (self *DelayFilter) GetType() int {
	return afp.PIPE_LINK
}

func (self *DelayFilter) Init(ctx *afp.Context, args []string) os.Error {
	self.context = ctx

	parser := flags.FlagParser(args)
	var t *int = parser.Int("t", 125, "The delay time in milliseconds")
/*	var w *int = parser.Int("w", 40, "The wet (delayed) signal ratio: 0 (dry) to 100 (wet)")*/
	parser.Parse()

	self.delayTimeInMs = *t

	if self.delayTimeInMs <= 0 {
		panic("Delay time must be greater than zero")
	}

	return nil
}

func (self *DelayFilter) Start() {
	self.header = <-self.context.HeaderSource
	self.context.HeaderSink <- self.header

	self.samplesPerMillisecond = int(self.header.SampleRate / 1000)
	self.extraSamples = self.delayTimeInMs * self.samplesPerMillisecond;

	// the mixBuffer is a ring buffer, each subsection size is self.header.FrameSize, and has n+1 buffers
	// ie, if the delay size is <= frameSize, we have a ring buffer of size 2
	self.mixBufferSize = int64(math.Ceil(float64(self.extraSamples) / float64(self.header.FrameSize))) + 1

	self.initBuffers()
	self.process()
}

func (self *DelayFilter) process() {
	var (
		t int64 = 0
/*		d float32 = 0.75
		w float32 = 0.25
*/		mbStart int64 = 0
		mbOffset int64 = 0
	)

/*	println("mbsize: ", self.mixBufferSize,  * self.header.FrameSize)*/

	for audio := range(self.context.Source) {
		// create a destination buffer
		destBuffer := makeBuffer(self.header.FrameSize, self.header.Channels)

		// set mixBuffer to current buffer in the ring to be filled & copy the source audio into that buffer
		mixBuffer := self.mixBuffer[mbStart * int64(self.header.FrameSize):((mbStart+1)*int64(self.header.FrameSize))]
		copy(mixBuffer, audio[:])

		println("t: ", t, " mbStart: ", mbStart, " mbOffset: ", mbOffset, " from: ", mbStart * int64(self.header.FrameSize), " to: ", ((mbStart+1)*int64(self.header.FrameSize)))

		for t1,sample := range(audio) {
			for c,_ := range(sample) {
				(*destBuffer)[t1][c] = self.mixBuffer[mbOffset][c]
			}
			if t > int64(self.extraSamples) {
				mbOffset++
				mbOffset %= (self.mixBufferSize * int64(self.header.FrameSize))
			}
			t++
/*			(*destBuffer)[t1] = sample*/
/*			if t < int64(self.extraSamples) {
				for c,_ := range(sample) {
					(*destBuffer)[t1][c] = 0 * w * d // amplitude * d
				}
			} else {
				for c,_ := range(sample) {
					(*destBuffer)[t1][c] = self.mixBuffer[mbOffset][c] // + amplitude * d
				}
				mbOffset++
			}

			if (t == int64(self.extraSamples)) {
				println("Starting delay at ", t, " mbOffset: ", mbOffset)
			}

			if (mbOffset >= (self.mixBufferSize * int64(self.header.FrameSize))) {
				mbOffset = 0
			}
			t++
*/		}

/*		self.context.Sink <- mixBuffer */
		self.context.Sink <- *destBuffer
/*		self.context.Sink <- self.mixBuffer[mbStart * int64(self.header.FrameSize):((mbStart+1)*int64(self.header.FrameSize))]*/

		mbStart++
		mbStart %= self.mixBufferSize

	}

	// while incoming audio available
		// read through input frame
		// accumulate global sample count
		// if global sample < delayTime in samples
			// copy dry source
		// else
			// mix source w/ delay
		// if global sample count % frameSize == 0
			// send current buffer to to Sink
			// switch current buffer
	// for extra samples
		// write delay data
		// if global sample count % frameSize == 0
			// send current buffer to to Sink
			// switch current buffer
	// for frameSize % extra samples
		// pad with zeros

}

func makeBuffer(size int32, channels int8) *[][]float32 {
	b := make([][]float32, size)
	for i,_ := range(b) {
		b[i] = make([]float32, channels)
	}

	return &b
}

func (self *DelayFilter) initBuffers() {
	self.mixBuffer = make([][]float32, self.mixBufferSize * int64(self.header.FrameSize))
	for i,_ := range self.mixBuffer {
		self.mixBuffer[i] = make([]float32, self.header.Channels)
	}
}

func (self *DelayFilter) Stop() os.Error {
	return nil
}<commit_msg>Don't panic on bad input<commit_after>// Copyright (c) 2010 AFP Authors
// This source code is released under the terms of the
// MIT license. Please see the file LICENSE for license details.

package delay

import (
	"afp"
	"afp/flags"
	"os"
	"math"
)

type DelayFilter struct {
	context *afp.Context
	header afp.StreamHeader
	samplesPerSecond int
	samplesPerMillisecond int
	delayTimeInMs int
	extraSamples int
	mixBufferSize int64
	bufferSize int32
	channels int16
	bytesPerSample int16
	buffers [][][]float32
	mixBuffer [][]float32
}

func NewDelayFilter() afp.Filter {
	return &DelayFilter{}
}

func (self *DelayFilter) GetType() int {
	return afp.PIPE_LINK
}

func (self *DelayFilter) Init(ctx *afp.Context, args []string) os.Error {
	self.context = ctx

	parser := flags.FlagParser(args)
	var t *int = parser.Int("t", 125, "The delay time in milliseconds")
/*	var w *int = parser.Int("w", 40, "The wet (delayed) signal ratio: 0 (dry) to 100 (wet)")*/
	parser.Parse()

	self.delayTimeInMs = *t

	if self.delayTimeInMs <= 0 {
		return os.NewError("Delay time must be greater than zero")
	}

	return nil
}

func (self *DelayFilter) Start() {
	self.header = <-self.context.HeaderSource
	self.context.HeaderSink <- self.header

	self.samplesPerMillisecond = int(self.header.SampleRate / 1000)
	self.extraSamples = self.delayTimeInMs * self.samplesPerMillisecond;

	// the mixBuffer is a ring buffer, each subsection size is self.header.FrameSize, and has n+1 buffers
	// ie, if the delay size is <= frameSize, we have a ring buffer of size 2
	self.mixBufferSize = int64(math.Ceil(float64(self.extraSamples) / float64(self.header.FrameSize))) + 1

	self.initBuffers()
	self.process()
}

func (self *DelayFilter) process() {
	var (
		t int64 = 0
		d float32 = 0.75
		w float32 = 0.25
		mbStart int64 = 0
		mbOffset int64 = 0
	)

	d = w
	w = d

	for audio := range(self.context.Source) {
		// create a destination buffer
		destBuffer := makeBuffer(self.header.FrameSize, self.header.Channels)

		// set mixBuffer to current buffer in the ring to be filled & copy the source audio into that buffer
		mixBuffer := self.mixBuffer[mbStart * int64(self.header.FrameSize):((mbStart+1)*int64(self.header.FrameSize))-1]
		copy(mixBuffer, audio[:])

		println("t: ", t, " mbStart: ", mbStart, " mbOffset: ", mbOffset, " from: ", mbStart * int64(self.header.FrameSize), " to: ", ((mbStart+1)*int64(self.header.FrameSize))-1)

		for t1,sample := range(audio) {
/*			for c,_ := range(sample) {
				(*destBuffer)[t1][c] = self.mixBuffer[mbOffset][c]
			}
			if t > int64(self.extraSamples) {
				mbOffset++
				mbOffset %= (self.mixBufferSize * int64(self.header.FrameSize))
			}
			t++
*/
/*			(*destBuffer)[t1] = sample */

			if t < int64(self.extraSamples) {
				for c,_ := range(sample) {
					(*destBuffer)[t1][c] = 0 * w * d // amplitude * d
				}
			} else {
				for c,_ := range(sample) {
					(*destBuffer)[t1][c] = self.mixBuffer[mbOffset][c] // + amplitude * d
				}
				mbOffset++
			}

			if (t == int64(self.extraSamples)) {
				println("Starting delay at ", t, " mbOffset: ", mbOffset)
			}

			if (mbOffset >= (self.mixBufferSize * int64(self.header.FrameSize))) {
				mbOffset = 0
			}
			t++

		}

/*		self.context.Sink <- mixBuffer */
		self.context.Sink <- *destBuffer
/*		self.context.Sink <- self.mixBuffer[mbStart * int64(self.header.FrameSize):((mbStart+1)*int64(self.header.FrameSize))]*/

		mbStart++
		mbStart %= self.mixBufferSize

	}

	// while incoming audio available
		// read through input frame
		// accumulate global sample count
		// if global sample < delayTime in samples
			// copy dry source
		// else
			// mix source w/ delay
		// if global sample count % frameSize == 0
			// send current buffer to to Sink
			// switch current buffer
	// for extra samples
		// write delay data
		// if global sample count % frameSize == 0
			// send current buffer to to Sink
			// switch current buffer
	// for frameSize % extra samples
		// pad with zeros

}

func makeBuffer(size int32, channels int8) *[][]float32 {
	b := make([][]float32, size)
	for i,_ := range(b) {
		b[i] = make([]float32, channels)
	}

	return &b
}

func (self *DelayFilter) initBuffers() {
	self.mixBuffer = make([][]float32, self.mixBufferSize * int64(self.header.FrameSize))
	for i,_ := range self.mixBuffer {
		self.mixBuffer[i] = make([]float32, self.header.Channels)
	}
}

func (self *DelayFilter) Stop() os.Error {
	return nil
}<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"
	"strconv"
	//"time"

	"github.com/docopt/docopt-go"
	"github.com/platform9/fission/controller"
	"github.com/platform9/fission/poolmgr"
	"github.com/platform9/fission/router"
)

func runController(port int, etcdUrl string, filepath string) {
	_, err := os.Stat(filepath)
	if err != nil {
		if os.IsNotExist(err) {
			log.Fatalf("Error: path %v does not exist", filepath)
		} else {
			log.Fatalf("Error: can't access path %v", filepath)
		}
	}
	fileStore := controller.MakeFileStore(filepath)

	rs, err := controller.MakeResourceStore(fileStore, []string{etcdUrl})
	if err != nil {
		log.Fatalf("Error: %v", err)
	}

	api := &controller.API{
		FunctionStore:    controller.FunctionStore{ResourceStore: *rs},
		HTTPTriggerStore: controller.HTTPTriggerStore{ResourceStore: *rs},
		EnvironmentStore: controller.EnvironmentStore{ResourceStore: *rs},
	}
	api.Serve(port)
	log.Fatalf("Error: Controller exited.")
}

func runRouter(port int, controllerUrl string, poolmgrUrl string) {
	router.Start(port, controllerUrl, poolmgrUrl)
	log.Fatalf("Error: Router exited.")
}

func runPoolmgr(port int, controllerUrl string, namespace string) {
	err := poolmgr.StartPoolmgr(controllerUrl, namespace, port)
	if err != nil {
		log.Fatalf("Error starting poolmgr: %v", err)
	}
}

func getPort(portArg interface{}) int {
	portArgStr := portArg.(string)
	port, err := strconv.Atoi(portArgStr)
	if err != nil {
		log.Fatalf("Error: invalid port number '%v'", portArgStr)
	}
	return port
}

func getStringArgWithDefault(arg interface{}, defaultValue string) string {
	if arg != nil {
		return arg.(string)
	} else {
		return defaultValue
	}
}

func main() {
	usage := `fission-bundle: Package of all fission microservices: controller, router, poolmgr.

Use it to start one or more of the fission servers:

 Controller keeps track of functions, triggers, environments.

 Pool manager maintains a pool of generalized function containers, and specializes them on-demand. Poolmgr must be run from a pod in a Kubernetes cluster.

 Router implements HTTP triggers: it routes to running instances, working with the controller and poolmgr.

Usage:
  fission-bundle --controllerPort=<port> [--etcdUrl=<etcdUrl>] --filepath=<filepath>
  fission-bundle --routerPort=<port> [--controllerUrl=<url> --poolmgrUrl=<url>]
  fission-bundle --poolmgrPort=<port> [--controllerUrl=<url>]
Options:
  --controllerPort=<port>  Port that the controller should listen on.
  --routerPort=<port>      Port that the router should listen on.
  --poolmgrPort=<port>     Port that the poolmgr should listen on.
  --controllerUrl=<url>    Controller URL. Not required if --controllerPort is specified.
  --poolmgrUrl=<url>       Controller URL. Not required if --poolmgrPort is specified.
  --etcdUrl=<etcdUrl>      Etcd URL.
  --filepath=<filepath>    Directory to store functions in.
  --namespace=<namespace>  Kubernetes namespace in which to run function containers. Defaults to 'fission-function'.
`
	arguments, err := docopt.Parse(usage, nil, true, "fission-bundle", false)
	if err != nil {
		log.Fatalf("Error: %v", err)
	}

	namespace := getStringArgWithDefault(arguments["--namespace"], "fission-function")

	controllerUrl := getStringArgWithDefault(arguments["--controllerUrl"], "http://controller.fission")
	etcdUrl := getStringArgWithDefault(arguments["--etcdUrl"], "http://etcd:2379")
	poolmgrUrl := getStringArgWithDefault(arguments["--poolmgrUrl"], "http://poolmgr.fission")

	if arguments["--controllerPort"] != nil {
		port := getPort(arguments["--controllerPort"])
		runController(port, etcdUrl, arguments["--filepath"].(string))
	}

	if arguments["--routerPort"] != nil {
		port := getPort(arguments["--routerPort"])
		runRouter(port, controllerUrl, poolmgrUrl)
	}

	if arguments["--poolmgrPort"] != nil {
		port := getPort(arguments["--poolmgrPort"])
		runPoolmgr(port, controllerUrl, namespace)
	}

	select {}
}
<commit_msg>Don't check controller filestore path<commit_after>package main

import (
	"log"
	"os"
	"strconv"
	//"time"

	"github.com/docopt/docopt-go"
	"github.com/platform9/fission/controller"
	"github.com/platform9/fission/poolmgr"
	"github.com/platform9/fission/router"
)

func runController(port int, etcdUrl string, filepath string) {
	// filePath will be created if it doesn't exist.
	fileStore := controller.MakeFileStore(filepath)

	rs, err := controller.MakeResourceStore(fileStore, []string{etcdUrl})
	if err != nil {
		log.Fatalf("Error: %v", err)
	}

	api := &controller.API{
		FunctionStore:    controller.FunctionStore{ResourceStore: *rs},
		HTTPTriggerStore: controller.HTTPTriggerStore{ResourceStore: *rs},
		EnvironmentStore: controller.EnvironmentStore{ResourceStore: *rs},
	}
	api.Serve(port)
	log.Fatalf("Error: Controller exited.")
}

func runRouter(port int, controllerUrl string, poolmgrUrl string) {
	router.Start(port, controllerUrl, poolmgrUrl)
	log.Fatalf("Error: Router exited.")
}

func runPoolmgr(port int, controllerUrl string, namespace string) {
	err := poolmgr.StartPoolmgr(controllerUrl, namespace, port)
	if err != nil {
		log.Fatalf("Error starting poolmgr: %v", err)
	}
}

func getPort(portArg interface{}) int {
	portArgStr := portArg.(string)
	port, err := strconv.Atoi(portArgStr)
	if err != nil {
		log.Fatalf("Error: invalid port number '%v'", portArgStr)
	}
	return port
}

func getStringArgWithDefault(arg interface{}, defaultValue string) string {
	if arg != nil {
		return arg.(string)
	} else {
		return defaultValue
	}
}

func main() {
	usage := `fission-bundle: Package of all fission microservices: controller, router, poolmgr.

Use it to start one or more of the fission servers:

 Controller keeps track of functions, triggers, environments.

 Pool manager maintains a pool of generalized function containers, and specializes them on-demand. Poolmgr must be run from a pod in a Kubernetes cluster.

 Router implements HTTP triggers: it routes to running instances, working with the controller and poolmgr.

Usage:
  fission-bundle --controllerPort=<port> [--etcdUrl=<etcdUrl>] --filepath=<filepath>
  fission-bundle --routerPort=<port> [--controllerUrl=<url> --poolmgrUrl=<url>]
  fission-bundle --poolmgrPort=<port> [--controllerUrl=<url>]
Options:
  --controllerPort=<port>  Port that the controller should listen on.
  --routerPort=<port>      Port that the router should listen on.
  --poolmgrPort=<port>     Port that the poolmgr should listen on.
  --controllerUrl=<url>    Controller URL. Not required if --controllerPort is specified.
  --poolmgrUrl=<url>       Controller URL. Not required if --poolmgrPort is specified.
  --etcdUrl=<etcdUrl>      Etcd URL.
  --filepath=<filepath>    Directory to store functions in.
  --namespace=<namespace>  Kubernetes namespace in which to run function containers. Defaults to 'fission-function'.
`
	arguments, err := docopt.Parse(usage, nil, true, "fission-bundle", false)
	if err != nil {
		log.Fatalf("Error: %v", err)
	}

	namespace := getStringArgWithDefault(arguments["--namespace"], "fission-function")

	controllerUrl := getStringArgWithDefault(arguments["--controllerUrl"], "http://controller.fission")
	etcdUrl := getStringArgWithDefault(arguments["--etcdUrl"], "http://etcd:2379")
	poolmgrUrl := getStringArgWithDefault(arguments["--poolmgrUrl"], "http://poolmgr.fission")

	if arguments["--controllerPort"] != nil {
		port := getPort(arguments["--controllerPort"])
		runController(port, etcdUrl, arguments["--filepath"].(string))
	}

	if arguments["--routerPort"] != nil {
		port := getPort(arguments["--routerPort"])
		runRouter(port, controllerUrl, poolmgrUrl)
	}

	if arguments["--poolmgrPort"] != nil {
		port := getPort(arguments["--poolmgrPort"])
		runPoolmgr(port, controllerUrl, namespace)
	}

	select {}
}
<|endoftext|>


<commit_before>package flatmap

import (
	"reflect"
	"testing"
)

func TestExpand(t *testing.T) {
	cases := []struct {
		Map    map[string]string
		Key    string
		Output interface{}
	}{
		{
			Map: map[string]string{
				"foo": "bar",
				"bar": "baz",
			},
			Key:    "foo",
			Output: "bar",
		},

		{
			Map: map[string]string{
				"foo.#": "2",
				"foo.0": "one",
				"foo.1": "two",
			},
			Key: "foo",
			Output: []interface{}{
				"one",
				"two",
			},
		},

		{
			Map: map[string]string{
				"foo.#":         "1",
				"foo.0.name":    "bar",
				"foo.0.port":    "3000",
				"foo.0.enabled": "true",
			},
			Key: "foo",
			Output: []interface{}{
				map[string]interface{}{
					"name":    "bar",
					"port":    "3000",
					"enabled": true,
				},
			},
		},

		{
			Map: map[string]string{
				"foo.#":         "1",
				"foo.0.name":    "bar",
				"foo.0.ports.#": "2",
				"foo.0.ports.0": "1",
				"foo.0.ports.1": "2",
			},
			Key: "foo",
			Output: []interface{}{
				map[string]interface{}{
					"name": "bar",
					"ports": []interface{}{
						"1",
						"2",
					},
				},
			},
		},

		{
			Map: map[string]string{
				"list_of_map.#":   "2",
				"list_of_map.0.%": "1",
				"list_of_map.0.a": "1",
				"list_of_map.1.%": "2",
				"list_of_map.1.b": "2",
				"list_of_map.1.c": "3",
			},
			Key: "list_of_map",
			Output: []interface{}{
				map[string]interface{}{
					"a": "1",
				},
				map[string]interface{}{
					"b": "2",
					"c": "3",
				},
			},
		},

		{
			Map: map[string]string{
				"map_of_list.%":       "2",
				"map_of_list.list2.#": "1",
				"map_of_list.list2.0": "c",
				"map_of_list.list1.#": "2",
				"map_of_list.list1.0": "a",
				"map_of_list.list1.1": "b",
			},
			Key: "map_of_list",
			Output: map[string]interface{}{
				"list1": []interface{}{"a", "b"},
				"list2": []interface{}{"c"},
			},
		},
	}

	for _, tc := range cases {
		actual := Expand(tc.Map, tc.Key)
		if !reflect.DeepEqual(actual, tc.Output) {
			t.Errorf(
				"Key: %v\nMap:\n\n%#v\n\nOutput:\n\n%#v\n\nExpected:\n\n%#v\n",
				tc.Key,
				tc.Map,
				actual,
				tc.Output)
		}
	}
}
<commit_msg>Add test for set expansion in flatmap.Expand<commit_after>package flatmap

import (
	"reflect"
	"testing"
)

func TestExpand(t *testing.T) {
	cases := []struct {
		Map    map[string]string
		Key    string
		Output interface{}
	}{
		{
			Map: map[string]string{
				"foo": "bar",
				"bar": "baz",
			},
			Key:    "foo",
			Output: "bar",
		},

		{
			Map: map[string]string{
				"foo.#": "2",
				"foo.0": "one",
				"foo.1": "two",
			},
			Key: "foo",
			Output: []interface{}{
				"one",
				"two",
			},
		},

		{
			Map: map[string]string{
				"foo.#":         "1",
				"foo.0.name":    "bar",
				"foo.0.port":    "3000",
				"foo.0.enabled": "true",
			},
			Key: "foo",
			Output: []interface{}{
				map[string]interface{}{
					"name":    "bar",
					"port":    "3000",
					"enabled": true,
				},
			},
		},

		{
			Map: map[string]string{
				"foo.#":         "1",
				"foo.0.name":    "bar",
				"foo.0.ports.#": "2",
				"foo.0.ports.0": "1",
				"foo.0.ports.1": "2",
			},
			Key: "foo",
			Output: []interface{}{
				map[string]interface{}{
					"name": "bar",
					"ports": []interface{}{
						"1",
						"2",
					},
				},
			},
		},

		{
			Map: map[string]string{
				"list_of_map.#":   "2",
				"list_of_map.0.%": "1",
				"list_of_map.0.a": "1",
				"list_of_map.1.%": "2",
				"list_of_map.1.b": "2",
				"list_of_map.1.c": "3",
			},
			Key: "list_of_map",
			Output: []interface{}{
				map[string]interface{}{
					"a": "1",
				},
				map[string]interface{}{
					"b": "2",
					"c": "3",
				},
			},
		},

		{
			Map: map[string]string{
				"map_of_list.%":       "2",
				"map_of_list.list2.#": "1",
				"map_of_list.list2.0": "c",
				"map_of_list.list1.#": "2",
				"map_of_list.list1.0": "a",
				"map_of_list.list1.1": "b",
			},
			Key: "map_of_list",
			Output: map[string]interface{}{
				"list1": []interface{}{"a", "b"},
				"list2": []interface{}{"c"},
			},
		},

		{
			Map: map[string]string{
				"set.#":    "3",
				"set.1234": "a",
				"set.1235": "b",
				"set.1236": "c",
			},
			Key:    "set",
			Output: []interface{}{"a", "b", "c"},
		},
	}

	for _, tc := range cases {
		actual := Expand(tc.Map, tc.Key)
		if !reflect.DeepEqual(actual, tc.Output) {
			t.Errorf(
				"Key: %v\nMap:\n\n%#v\n\nOutput:\n\n%#v\n\nExpected:\n\n%#v\n",
				tc.Key,
				tc.Map,
				actual,
				tc.Output)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sfnt

import (
	"bytes"
	"io/ioutil"
	"path/filepath"
	"testing"

	"golang.org/x/image/font/gofont/goregular"
	"golang.org/x/image/math/fixed"
)

func moveTo(xa, ya int) Segment {
	return Segment{
		Op: SegmentOpMoveTo,
		Args: [6]fixed.Int26_6{
			0: fixed.I(xa),
			1: fixed.I(ya),
		},
	}
}

func lineTo(xa, ya int) Segment {
	return Segment{
		Op: SegmentOpLineTo,
		Args: [6]fixed.Int26_6{
			0: fixed.I(xa),
			1: fixed.I(ya),
		},
	}
}

func cubeTo(xa, ya, xb, yb, xc, yc int) Segment {
	return Segment{
		Op: SegmentOpCubeTo,
		Args: [6]fixed.Int26_6{
			0: fixed.I(xa),
			1: fixed.I(ya),
			2: fixed.I(xb),
			3: fixed.I(yb),
			4: fixed.I(xc),
			5: fixed.I(yc),
		},
	}
}

func TestTrueTypeParse(t *testing.T) {
	f, err := Parse(goregular.TTF)
	if err != nil {
		t.Fatalf("Parse: %v", err)
	}
	testTrueType(t, f)
}

func TestTrueTypeParseReaderAt(t *testing.T) {
	f, err := ParseReaderAt(bytes.NewReader(goregular.TTF))
	if err != nil {
		t.Fatalf("ParseReaderAt: %v", err)
	}
	testTrueType(t, f)
}

func testTrueType(t *testing.T, f *Font) {
	if got, want := f.UnitsPerEm(), Units(2048); got != want {
		t.Errorf("UnitsPerEm: got %d, want %d", got, want)
	}
	// The exact number of glyphs in goregular.TTF can vary, and future
	// versions may add more glyphs, but https://blog.golang.org/go-fonts says
	// that "The WGL4 character set... [has] more than 650 characters in all.
	if got, want := f.NumGlyphs(), 650; got <= want {
		t.Errorf("NumGlyphs: got %d, want > %d", got, want)
	}
}

func TestPostScript(t *testing.T) {
	data, err := ioutil.ReadFile(filepath.Join("..", "testdata", "CFFTest.otf"))
	if err != nil {
		t.Fatal(err)
	}
	f, err := Parse(data)
	if err != nil {
		t.Fatal(err)
	}

	// wants' vectors correspond 1-to-1 to what's in the CFFTest.sfd file,
	// although for some unknown reason, FontForge reverses the order somewhere
	// along the way when converting from SFD to OpenType/CFF.
	//
	// The .notdef glyph isn't explicitly in the SFD file, but for some unknown
	// reason, FontForge generates a .notdef glyph in the OpenType/CFF file.
	wants := [...][]Segment{{
		// .notdef
		// - contour #0
		moveTo(50, 0),
		lineTo(450, 0),
		lineTo(450, 533),
		lineTo(50, 533),
		// - contour #1
		moveTo(100, 50),
		lineTo(100, 483),
		lineTo(400, 483),
		lineTo(400, 50),
	}, {
		// zero
		// - contour #0
		moveTo(300, 700),
		cubeTo(380, 700, 420, 580, 420, 500),
		cubeTo(420, 350, 390, 100, 300, 100),
		cubeTo(220, 100, 180, 220, 180, 300),
		cubeTo(180, 450, 210, 700, 300, 700),
		// - contour #1
		moveTo(300, 800),
		cubeTo(200, 800, 100, 580, 100, 400),
		cubeTo(100, 220, 200, 0, 300, 0),
		cubeTo(400, 0, 500, 220, 500, 400),
		cubeTo(500, 580, 400, 800, 300, 800),
	}, {
		// one
		// - contour #0
		moveTo(100, 0),
		lineTo(300, 0),
		lineTo(300, 800),
		lineTo(100, 800),
	}, {
		// Q
		// - contour #0
		moveTo(657, 237),
		lineTo(289, 387),
		lineTo(519, 615),
		// - contour #1
		moveTo(792, 169),
		cubeTo(867, 263, 926, 502, 791, 665),
		cubeTo(645, 840, 380, 831, 228, 673),
		cubeTo(71, 509, 110, 231, 242, 93),
		cubeTo(369, -39, 641, 18, 722, 93),
		lineTo(802, 3),
		lineTo(864, 83),
	}, {
		// uni4E2D
		// - contour #0
		moveTo(141, 520),
		lineTo(137, 356),
		lineTo(245, 400),
		lineTo(331, 26),
		lineTo(355, 414),
		lineTo(463, 434),
		lineTo(453, 620),
		lineTo(341, 592),
		lineTo(331, 758),
		lineTo(243, 752),
		lineTo(235, 562),
	}}

	if ng := f.NumGlyphs(); ng != len(wants) {
		t.Fatalf("NumGlyphs: got %d, want %d", ng, len(wants))
	}
	var b Buffer
loop:
	for i, want := range wants {
		if err := f.LoadGlyph(&b, GlyphIndex(i), nil); err != nil {
			t.Errorf("i=%d: LoadGlyph: %v", i, err)
			continue
		}
		got := b.Segments
		if len(got) != len(want) {
			t.Errorf("i=%d: got %d elements, want %d\noverall:\ngot  %v\nwant %v",
				i, len(got), len(want), got, want)
			continue
		}
		for j, g := range got {
			if w := want[j]; g != w {
				t.Errorf("i=%d: element %d:\ngot  %v\nwant %v\noverall:\ngot  %v\nwant %v",
					i, j, g, w, got, want)
				continue loop
			}
		}
	}
}
<commit_msg>font/sfnt: add a comment about contour ordering.<commit_after>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sfnt

import (
	"bytes"
	"io/ioutil"
	"path/filepath"
	"testing"

	"golang.org/x/image/font/gofont/goregular"
	"golang.org/x/image/math/fixed"
)

func moveTo(xa, ya int) Segment {
	return Segment{
		Op: SegmentOpMoveTo,
		Args: [6]fixed.Int26_6{
			0: fixed.I(xa),
			1: fixed.I(ya),
		},
	}
}

func lineTo(xa, ya int) Segment {
	return Segment{
		Op: SegmentOpLineTo,
		Args: [6]fixed.Int26_6{
			0: fixed.I(xa),
			1: fixed.I(ya),
		},
	}
}

func cubeTo(xa, ya, xb, yb, xc, yc int) Segment {
	return Segment{
		Op: SegmentOpCubeTo,
		Args: [6]fixed.Int26_6{
			0: fixed.I(xa),
			1: fixed.I(ya),
			2: fixed.I(xb),
			3: fixed.I(yb),
			4: fixed.I(xc),
			5: fixed.I(yc),
		},
	}
}

func TestTrueTypeParse(t *testing.T) {
	f, err := Parse(goregular.TTF)
	if err != nil {
		t.Fatalf("Parse: %v", err)
	}
	testTrueType(t, f)
}

func TestTrueTypeParseReaderAt(t *testing.T) {
	f, err := ParseReaderAt(bytes.NewReader(goregular.TTF))
	if err != nil {
		t.Fatalf("ParseReaderAt: %v", err)
	}
	testTrueType(t, f)
}

func testTrueType(t *testing.T, f *Font) {
	if got, want := f.UnitsPerEm(), Units(2048); got != want {
		t.Errorf("UnitsPerEm: got %d, want %d", got, want)
	}
	// The exact number of glyphs in goregular.TTF can vary, and future
	// versions may add more glyphs, but https://blog.golang.org/go-fonts says
	// that "The WGL4 character set... [has] more than 650 characters in all.
	if got, want := f.NumGlyphs(), 650; got <= want {
		t.Errorf("NumGlyphs: got %d, want > %d", got, want)
	}
}

func TestPostScript(t *testing.T) {
	data, err := ioutil.ReadFile(filepath.Join("..", "testdata", "CFFTest.otf"))
	if err != nil {
		t.Fatal(err)
	}
	f, err := Parse(data)
	if err != nil {
		t.Fatal(err)
	}

	// wants' vectors correspond 1-to-1 to what's in the CFFTest.sfd file,
	// although OpenType/CFF and FontForge's SFD have reversed orders.
	// https://fontforge.github.io/validation.html says that "All paths must be
	// drawn in a consistent direction. Clockwise for external paths,
	// anti-clockwise for internal paths. (Actually PostScript requires the
	// exact opposite, but FontForge reverses PostScript contours when it loads
	// them so that everything is consistant internally -- and reverses them
	// again when it saves them, of course)."
	//
	// The .notdef glyph isn't explicitly in the SFD file, but for some unknown
	// reason, FontForge generates a .notdef glyph in the OpenType/CFF file.
	wants := [...][]Segment{{
		// .notdef
		// - contour #0
		moveTo(50, 0),
		lineTo(450, 0),
		lineTo(450, 533),
		lineTo(50, 533),
		// - contour #1
		moveTo(100, 50),
		lineTo(100, 483),
		lineTo(400, 483),
		lineTo(400, 50),
	}, {
		// zero
		// - contour #0
		moveTo(300, 700),
		cubeTo(380, 700, 420, 580, 420, 500),
		cubeTo(420, 350, 390, 100, 300, 100),
		cubeTo(220, 100, 180, 220, 180, 300),
		cubeTo(180, 450, 210, 700, 300, 700),
		// - contour #1
		moveTo(300, 800),
		cubeTo(200, 800, 100, 580, 100, 400),
		cubeTo(100, 220, 200, 0, 300, 0),
		cubeTo(400, 0, 500, 220, 500, 400),
		cubeTo(500, 580, 400, 800, 300, 800),
	}, {
		// one
		// - contour #0
		moveTo(100, 0),
		lineTo(300, 0),
		lineTo(300, 800),
		lineTo(100, 800),
	}, {
		// Q
		// - contour #0
		moveTo(657, 237),
		lineTo(289, 387),
		lineTo(519, 615),
		// - contour #1
		moveTo(792, 169),
		cubeTo(867, 263, 926, 502, 791, 665),
		cubeTo(645, 840, 380, 831, 228, 673),
		cubeTo(71, 509, 110, 231, 242, 93),
		cubeTo(369, -39, 641, 18, 722, 93),
		lineTo(802, 3),
		lineTo(864, 83),
	}, {
		// uni4E2D
		// - contour #0
		moveTo(141, 520),
		lineTo(137, 356),
		lineTo(245, 400),
		lineTo(331, 26),
		lineTo(355, 414),
		lineTo(463, 434),
		lineTo(453, 620),
		lineTo(341, 592),
		lineTo(331, 758),
		lineTo(243, 752),
		lineTo(235, 562),
	}}

	if ng := f.NumGlyphs(); ng != len(wants) {
		t.Fatalf("NumGlyphs: got %d, want %d", ng, len(wants))
	}
	var b Buffer
loop:
	for i, want := range wants {
		if err := f.LoadGlyph(&b, GlyphIndex(i), nil); err != nil {
			t.Errorf("i=%d: LoadGlyph: %v", i, err)
			continue
		}
		got := b.Segments
		if len(got) != len(want) {
			t.Errorf("i=%d: got %d elements, want %d\noverall:\ngot  %v\nwant %v",
				i, len(got), len(want), got, want)
			continue
		}
		for j, g := range got {
			if w := want[j]; g != w {
				t.Errorf("i=%d: element %d:\ngot  %v\nwant %v\noverall:\ngot  %v\nwant %v",
					i, j, g, w, got, want)
				continue loop
			}
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"fmt"
	"io"
	"io/ioutil"
	"path/filepath"
	"strconv"
	"text/template"

	"github.com/renstrom/dedent"
	"github.com/spf13/cobra"

	"k8s.io/apimachinery/pkg/runtime"
	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
	kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"
	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
	kubemaster "k8s.io/kubernetes/cmd/kubeadm/app/master"
	addonsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons"
	apiconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/apiconfig"
	certphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
	kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig"
	tokenphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/token"
	"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
	"k8s.io/kubernetes/pkg/api"
)

var (
	initDoneTempl = template.Must(template.New("init").Parse(dedent.Dedent(`
		Your Kubernetes master has initialized successfully!

		To start using your cluster, you need to run (as a regular user):

		  sudo cp {{.KubeConfigPath}} $HOME/
		  sudo chown $(id -u):$(id -g) $HOME/{{.KubeConfigName}}
		  export KUBECONFIG=$HOME/{{.KubeConfigName}}

		You should now deploy a pod network to the cluster.
		Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
		  http://kubernetes.io/docs/admin/addons/

		You can now join any number of machines by running the following on each node
		as root:

		  kubeadm join --token {{.Token}} {{.MasterIP}}:{{.MasterPort}}

		`)))
)

// NewCmdInit returns "kubeadm init" command.
func NewCmdInit(out io.Writer) *cobra.Command {
	cfg := &kubeadmapiext.MasterConfiguration{}
	api.Scheme.Default(cfg)

	var cfgPath string
	var skipPreFlight bool
	var skipTokenPrint bool
	cmd := &cobra.Command{
		Use:   "init",
		Short: "Run this in order to set up the Kubernetes master",
		Run: func(cmd *cobra.Command, args []string) {
			api.Scheme.Default(cfg)
			internalcfg := &kubeadmapi.MasterConfiguration{}
			api.Scheme.Convert(cfg, internalcfg, nil)

			i, err := NewInit(cfgPath, internalcfg, skipPreFlight, skipTokenPrint)
			kubeadmutil.CheckErr(err)
			kubeadmutil.CheckErr(i.Validate())
			kubeadmutil.CheckErr(i.Run(out))
		},
	}

	cmd.PersistentFlags().StringVar(
		&cfg.API.AdvertiseAddress, "apiserver-advertise-address", cfg.API.AdvertiseAddress,
		"The IP address the API Server will advertise it's listening on. 0.0.0.0 means the default network interface's address.",
	)
	cmd.PersistentFlags().Int32Var(
		&cfg.API.BindPort, "apiserver-bind-port", cfg.API.BindPort,
		"Port for the API Server to bind to",
	)
	cmd.PersistentFlags().StringVar(
		&cfg.Networking.ServiceSubnet, "service-cidr", cfg.Networking.ServiceSubnet,
		"Use alternative range of IP address for service VIPs",
	)
	cmd.PersistentFlags().StringVar(
		&cfg.Networking.PodSubnet, "pod-network-cidr", cfg.Networking.PodSubnet,
		"Specify range of IP addresses for the pod network; if set, the control plane will automatically allocate CIDRs for every node",
	)
	cmd.PersistentFlags().StringVar(
		&cfg.Networking.DNSDomain, "service-dns-domain", cfg.Networking.DNSDomain,
		`Use alternative domain for services, e.g. "myorg.internal"`,
	)
	cmd.PersistentFlags().StringVar(
		&cfg.KubernetesVersion, "kubernetes-version", cfg.KubernetesVersion,
		`Choose a specific Kubernetes version for the control plane`,
	)
	cmd.PersistentFlags().StringVar(
		&cfg.CertificatesDir, "cert-dir", cfg.CertificatesDir,
		`The path where to save and store the certificates`,
	)
	cmd.PersistentFlags().StringSliceVar(
		&cfg.APIServerCertSANs, "apiserver-cert-extra-sans", cfg.APIServerCertSANs,
		`Optional extra altnames to use for the API Server serving cert. Can be both IP addresses and dns names.`,
	)

	cmd.PersistentFlags().StringVar(&cfgPath, "config", cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)")

	cmd.PersistentFlags().BoolVar(
		&skipPreFlight, "skip-preflight-checks", skipPreFlight,
		"Skip preflight checks normally run before modifying the system",
	)
	cmd.PersistentFlags().BoolVar(
		&skipTokenPrint, "skip-token-print", skipTokenPrint,
		"Skip printing of the default bootstrap token generated by 'kubeadm init'",
	)

	cmd.PersistentFlags().StringVar(
		&cfg.Token, "token", cfg.Token,
		"The token to use for establishing bidirectional trust between nodes and masters.")

	cmd.PersistentFlags().DurationVar(
		&cfg.TokenTTL, "token-ttl", cfg.TokenTTL,
		"The duration before the bootstrap token is automatically deleted. 0 means 'never expires'.")

	return cmd
}

func NewInit(cfgPath string, cfg *kubeadmapi.MasterConfiguration, skipPreFlight, skipTokenPrint bool) (*Init, error) {

	fmt.Println("[kubeadm] WARNING: kubeadm is in beta, please do not use it for production clusters.")

	if cfgPath != "" {
		b, err := ioutil.ReadFile(cfgPath)
		if err != nil {
			return nil, fmt.Errorf("unable to read config from %q [%v]", cfgPath, err)
		}
		if err := runtime.DecodeInto(api.Codecs.UniversalDecoder(), b, cfg); err != nil {
			return nil, fmt.Errorf("unable to decode config from %q [%v]", cfgPath, err)
		}
	}

	// Set defaults dynamically that the API group defaulting can't (by fetching information from the internet, looking up network interfaces, etc.)
	err := setInitDynamicDefaults(cfg)
	if err != nil {
		return nil, err
	}

	if !skipPreFlight {
		fmt.Println("[preflight] Running pre-flight checks")

		// First, check if we're root separately from the other preflight checks and fail fast
		if err := preflight.RunRootCheckOnly(); err != nil {
			return nil, err
		}

		// Then continue with the others...
		if err := preflight.RunInitMasterChecks(cfg); err != nil {
			return nil, err
		}

		// Try to start the kubelet service in case it's inactive
		preflight.TryStartKubelet()
	} else {
		fmt.Println("[preflight] Skipping pre-flight checks")
	}

	return &Init{cfg: cfg, skipTokenPrint: skipTokenPrint}, nil
}

type Init struct {
	cfg            *kubeadmapi.MasterConfiguration
	skipTokenPrint bool
}

// Validate validates configuration passed to "kubeadm init"
func (i *Init) Validate() error {
	return validation.ValidateMasterConfiguration(i.cfg).ToAggregate()
}

// Run executes master node provisioning, including certificates, needed static pod manifests, etc.
func (i *Init) Run(out io.Writer) error {

	// PHASE 1: Generate certificates
	err := certphase.CreatePKIAssets(i.cfg)
	if err != nil {
		return err
	}

	// PHASE 2: Generate kubeconfig files for the admin and the kubelet

	masterEndpoint := fmt.Sprintf("https://%s:%d", i.cfg.API.AdvertiseAddress, i.cfg.API.BindPort)
	err = kubeconfigphase.CreateInitKubeConfigFiles(masterEndpoint, i.cfg.CertificatesDir, kubeadmapi.GlobalEnvParams.KubernetesDir)
	if err != nil {
		return err
	}

	// PHASE 3: Bootstrap the control plane
	if err := kubemaster.WriteStaticPodManifests(i.cfg); err != nil {
		return err
	}

	adminKubeConfigPath := filepath.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeadmconstants.AdminKubeConfigFileName)
	client, err := kubemaster.CreateClientAndWaitForAPI(adminKubeConfigPath)
	if err != nil {
		return err
	}

	if err := apiconfigphase.UpdateMasterRoleLabelsAndTaints(client); err != nil {
		return err
	}

	// Is deployment type self-hosted?
	if i.cfg.SelfHosted {
		// Temporary control plane is up, now we create our self hosted control
		// plane components and remove the static manifests:
		fmt.Println("[self-hosted] Creating self-hosted control plane...")
		if err := kubemaster.CreateSelfHostedControlPlane(i.cfg, client); err != nil {
			return err
		}
	}

	// PHASE 4: Set up the bootstrap tokens
	if !i.skipTokenPrint {
		fmt.Printf("[token] Using token: %s\n", i.cfg.Token)
	}

	tokenDescription := "The default bootstrap token generated by 'kubeadm init'."
	if err := tokenphase.UpdateOrCreateToken(client, i.cfg.Token, false, i.cfg.TokenTTL, kubeadmconstants.DefaultTokenUsages, tokenDescription); err != nil {
		return err
	}

	if err := tokenphase.CreateBootstrapConfigMapIfNotExists(client, adminKubeConfigPath); err != nil {
		return err
	}

	// PHASE 5: Install and deploy all addons, and configure things as necessary

	// Create the necessary ServiceAccounts
	err = apiconfigphase.CreateServiceAccounts(client)
	if err != nil {
		return err
	}

	err = apiconfigphase.CreateRBACRules(client)
	if err != nil {
		return err
	}

	if err := addonsphase.CreateEssentialAddons(i.cfg, client); err != nil {
		return err
	}

	ctx := map[string]string{
		"KubeConfigPath": filepath.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeadmconstants.AdminKubeConfigFileName),
		"KubeConfigName": kubeadmconstants.AdminKubeConfigFileName,
		"Token":          i.cfg.Token,
		"MasterIP":       i.cfg.API.AdvertiseAddress,
		"MasterPort":     strconv.Itoa(int(i.cfg.API.BindPort)),
	}
	if i.skipTokenPrint {
		ctx["Token"] = "<value withheld>"
	}

	return initDoneTempl.Execute(out, ctx)
}
<commit_msg>kubeadm: improve quickstart instructions<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"fmt"
	"io"
	"io/ioutil"
	"path/filepath"
	"strconv"
	"text/template"

	"github.com/renstrom/dedent"
	"github.com/spf13/cobra"

	"k8s.io/apimachinery/pkg/runtime"
	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
	kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"
	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
	kubemaster "k8s.io/kubernetes/cmd/kubeadm/app/master"
	addonsphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/addons"
	apiconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/apiconfig"
	certphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/certs"
	kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig"
	tokenphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/token"
	"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
	"k8s.io/kubernetes/pkg/api"
)

var (
	initDoneTempl = template.Must(template.New("init").Parse(dedent.Dedent(`
		Your Kubernetes master has initialized successfully!

		To start using your cluster, you need to run (as a regular user):

		  mkdir -p $HOME/.kube
		  sudo cp -i {{.KubeConfigPath}} $HOME/.kube/config
		  sudo chown $(id -u):$(id -g) $HOME/.kube/config

		You should now deploy a pod network to the cluster.
		Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
		  http://kubernetes.io/docs/admin/addons/

		You can now join any number of machines by running the following on each node
		as root:

		  kubeadm join --token {{.Token}} {{.MasterIP}}:{{.MasterPort}}

		`)))
)

// NewCmdInit returns "kubeadm init" command.
func NewCmdInit(out io.Writer) *cobra.Command {
	cfg := &kubeadmapiext.MasterConfiguration{}
	api.Scheme.Default(cfg)

	var cfgPath string
	var skipPreFlight bool
	var skipTokenPrint bool
	cmd := &cobra.Command{
		Use:   "init",
		Short: "Run this in order to set up the Kubernetes master",
		Run: func(cmd *cobra.Command, args []string) {
			api.Scheme.Default(cfg)
			internalcfg := &kubeadmapi.MasterConfiguration{}
			api.Scheme.Convert(cfg, internalcfg, nil)

			i, err := NewInit(cfgPath, internalcfg, skipPreFlight, skipTokenPrint)
			kubeadmutil.CheckErr(err)
			kubeadmutil.CheckErr(i.Validate())
			kubeadmutil.CheckErr(i.Run(out))
		},
	}

	cmd.PersistentFlags().StringVar(
		&cfg.API.AdvertiseAddress, "apiserver-advertise-address", cfg.API.AdvertiseAddress,
		"The IP address the API Server will advertise it's listening on. 0.0.0.0 means the default network interface's address.",
	)
	cmd.PersistentFlags().Int32Var(
		&cfg.API.BindPort, "apiserver-bind-port", cfg.API.BindPort,
		"Port for the API Server to bind to",
	)
	cmd.PersistentFlags().StringVar(
		&cfg.Networking.ServiceSubnet, "service-cidr", cfg.Networking.ServiceSubnet,
		"Use alternative range of IP address for service VIPs",
	)
	cmd.PersistentFlags().StringVar(
		&cfg.Networking.PodSubnet, "pod-network-cidr", cfg.Networking.PodSubnet,
		"Specify range of IP addresses for the pod network; if set, the control plane will automatically allocate CIDRs for every node",
	)
	cmd.PersistentFlags().StringVar(
		&cfg.Networking.DNSDomain, "service-dns-domain", cfg.Networking.DNSDomain,
		`Use alternative domain for services, e.g. "myorg.internal"`,
	)
	cmd.PersistentFlags().StringVar(
		&cfg.KubernetesVersion, "kubernetes-version", cfg.KubernetesVersion,
		`Choose a specific Kubernetes version for the control plane`,
	)
	cmd.PersistentFlags().StringVar(
		&cfg.CertificatesDir, "cert-dir", cfg.CertificatesDir,
		`The path where to save and store the certificates`,
	)
	cmd.PersistentFlags().StringSliceVar(
		&cfg.APIServerCertSANs, "apiserver-cert-extra-sans", cfg.APIServerCertSANs,
		`Optional extra altnames to use for the API Server serving cert. Can be both IP addresses and dns names.`,
	)

	cmd.PersistentFlags().StringVar(&cfgPath, "config", cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)")

	cmd.PersistentFlags().BoolVar(
		&skipPreFlight, "skip-preflight-checks", skipPreFlight,
		"Skip preflight checks normally run before modifying the system",
	)
	cmd.PersistentFlags().BoolVar(
		&skipTokenPrint, "skip-token-print", skipTokenPrint,
		"Skip printing of the default bootstrap token generated by 'kubeadm init'",
	)

	cmd.PersistentFlags().StringVar(
		&cfg.Token, "token", cfg.Token,
		"The token to use for establishing bidirectional trust between nodes and masters.")

	cmd.PersistentFlags().DurationVar(
		&cfg.TokenTTL, "token-ttl", cfg.TokenTTL,
		"The duration before the bootstrap token is automatically deleted. 0 means 'never expires'.")

	return cmd
}

func NewInit(cfgPath string, cfg *kubeadmapi.MasterConfiguration, skipPreFlight, skipTokenPrint bool) (*Init, error) {

	fmt.Println("[kubeadm] WARNING: kubeadm is in beta, please do not use it for production clusters.")

	if cfgPath != "" {
		b, err := ioutil.ReadFile(cfgPath)
		if err != nil {
			return nil, fmt.Errorf("unable to read config from %q [%v]", cfgPath, err)
		}
		if err := runtime.DecodeInto(api.Codecs.UniversalDecoder(), b, cfg); err != nil {
			return nil, fmt.Errorf("unable to decode config from %q [%v]", cfgPath, err)
		}
	}

	// Set defaults dynamically that the API group defaulting can't (by fetching information from the internet, looking up network interfaces, etc.)
	err := setInitDynamicDefaults(cfg)
	if err != nil {
		return nil, err
	}

	if !skipPreFlight {
		fmt.Println("[preflight] Running pre-flight checks")

		// First, check if we're root separately from the other preflight checks and fail fast
		if err := preflight.RunRootCheckOnly(); err != nil {
			return nil, err
		}

		// Then continue with the others...
		if err := preflight.RunInitMasterChecks(cfg); err != nil {
			return nil, err
		}

		// Try to start the kubelet service in case it's inactive
		preflight.TryStartKubelet()
	} else {
		fmt.Println("[preflight] Skipping pre-flight checks")
	}

	return &Init{cfg: cfg, skipTokenPrint: skipTokenPrint}, nil
}

type Init struct {
	cfg            *kubeadmapi.MasterConfiguration
	skipTokenPrint bool
}

// Validate validates configuration passed to "kubeadm init"
func (i *Init) Validate() error {
	return validation.ValidateMasterConfiguration(i.cfg).ToAggregate()
}

// Run executes master node provisioning, including certificates, needed static pod manifests, etc.
func (i *Init) Run(out io.Writer) error {

	// PHASE 1: Generate certificates
	err := certphase.CreatePKIAssets(i.cfg)
	if err != nil {
		return err
	}

	// PHASE 2: Generate kubeconfig files for the admin and the kubelet

	masterEndpoint := fmt.Sprintf("https://%s:%d", i.cfg.API.AdvertiseAddress, i.cfg.API.BindPort)
	err = kubeconfigphase.CreateInitKubeConfigFiles(masterEndpoint, i.cfg.CertificatesDir, kubeadmapi.GlobalEnvParams.KubernetesDir)
	if err != nil {
		return err
	}

	// PHASE 3: Bootstrap the control plane
	if err := kubemaster.WriteStaticPodManifests(i.cfg); err != nil {
		return err
	}

	adminKubeConfigPath := filepath.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeadmconstants.AdminKubeConfigFileName)
	client, err := kubemaster.CreateClientAndWaitForAPI(adminKubeConfigPath)
	if err != nil {
		return err
	}

	if err := apiconfigphase.UpdateMasterRoleLabelsAndTaints(client); err != nil {
		return err
	}

	// Is deployment type self-hosted?
	if i.cfg.SelfHosted {
		// Temporary control plane is up, now we create our self hosted control
		// plane components and remove the static manifests:
		fmt.Println("[self-hosted] Creating self-hosted control plane...")
		if err := kubemaster.CreateSelfHostedControlPlane(i.cfg, client); err != nil {
			return err
		}
	}

	// PHASE 4: Set up the bootstrap tokens
	if !i.skipTokenPrint {
		fmt.Printf("[token] Using token: %s\n", i.cfg.Token)
	}

	tokenDescription := "The default bootstrap token generated by 'kubeadm init'."
	if err := tokenphase.UpdateOrCreateToken(client, i.cfg.Token, false, i.cfg.TokenTTL, kubeadmconstants.DefaultTokenUsages, tokenDescription); err != nil {
		return err
	}

	if err := tokenphase.CreateBootstrapConfigMapIfNotExists(client, adminKubeConfigPath); err != nil {
		return err
	}

	// PHASE 5: Install and deploy all addons, and configure things as necessary

	// Create the necessary ServiceAccounts
	err = apiconfigphase.CreateServiceAccounts(client)
	if err != nil {
		return err
	}

	err = apiconfigphase.CreateRBACRules(client)
	if err != nil {
		return err
	}

	if err := addonsphase.CreateEssentialAddons(i.cfg, client); err != nil {
		return err
	}

	ctx := map[string]string{
		"KubeConfigPath": filepath.Join(kubeadmapi.GlobalEnvParams.KubernetesDir, kubeadmconstants.AdminKubeConfigFileName),
		"KubeConfigName": kubeadmconstants.AdminKubeConfigFileName,
		"Token":          i.cfg.Token,
		"MasterIP":       i.cfg.API.AdvertiseAddress,
		"MasterPort":     strconv.Itoa(int(i.cfg.API.BindPort)),
	}
	if i.skipTokenPrint {
		ctx["Token"] = "<value withheld>"
	}

	return initDoneTempl.Execute(out, ctx)
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"time"

	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apiserver/pkg/util/flag"
	clientgoclientset "k8s.io/client-go/kubernetes"
	clientv1 "k8s.io/client-go/pkg/api/v1"
	restclient "k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/client-go/tools/record"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
	"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
	_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
	cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
	"k8s.io/kubernetes/pkg/kubelet/cm"
	"k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker"
	"k8s.io/kubernetes/pkg/kubemark"
	fakeexec "k8s.io/kubernetes/pkg/util/exec"
	fakeiptables "k8s.io/kubernetes/pkg/util/iptables/testing"
	fakesysctl "k8s.io/kubernetes/pkg/util/sysctl/testing"
	_ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration

	"github.com/golang/glog"
	"github.com/spf13/pflag"
)

type HollowNodeConfig struct {
	KubeconfigPath      string
	KubeletPort         int
	KubeletReadOnlyPort int
	Morph               string
	NodeName            string
	ServerPort          int
	ContentType         string
	UseRealProxier      bool
}

const (
	maxPods            = 110
	podsPerCore        = 0
	configResyncPeriod = 15 * time.Minute
)

var knownMorphs = sets.NewString("kubelet", "proxy")

func (c *HollowNodeConfig) addFlags(fs *pflag.FlagSet) {
	fs.StringVar(&c.KubeconfigPath, "kubeconfig", "/kubeconfig/kubeconfig", "Path to kubeconfig file.")
	fs.IntVar(&c.KubeletPort, "kubelet-port", 10250, "Port on which HollowKubelet should be listening.")
	fs.IntVar(&c.KubeletReadOnlyPort, "kubelet-read-only-port", 10255, "Read-only port on which Kubelet is listening.")
	fs.StringVar(&c.NodeName, "name", "fake-node", "Name of this Hollow Node.")
	fs.IntVar(&c.ServerPort, "api-server-port", 443, "Port on which API server is listening.")
	fs.StringVar(&c.Morph, "morph", "", fmt.Sprintf("Specifies into which Hollow component this binary should morph. Allowed values: %v", knownMorphs.List()))
	fs.StringVar(&c.ContentType, "kube-api-content-type", "application/vnd.kubernetes.protobuf", "ContentType of requests sent to apiserver.")
	fs.BoolVar(&c.UseRealProxier, "use-real-proxier", true, "Set to true if you want to use real proxier inside hollow-proxy.")
}

func (c *HollowNodeConfig) createClientConfigFromFile() (*restclient.Config, error) {
	clientConfig, err := clientcmd.LoadFromFile(c.KubeconfigPath)
	if err != nil {
		return nil, fmt.Errorf("error while loading kubeconfig from file %v: %v", c.KubeconfigPath, err)
	}
	config, err := clientcmd.NewDefaultClientConfig(*clientConfig, &clientcmd.ConfigOverrides{}).ClientConfig()
	if err != nil {
		return nil, fmt.Errorf("error while creating kubeconfig: %v", err)
	}
	config.ContentType = c.ContentType
	config.QPS = 10
	config.Burst = 20
	return config, nil
}

func main() {
	config := HollowNodeConfig{}
	config.addFlags(pflag.CommandLine)
	flag.InitFlags()

	if !knownMorphs.Has(config.Morph) {
		glog.Fatalf("Unknown morph: %v. Allowed values: %v", config.Morph, knownMorphs.List())
	}

	// create a client to communicate with API server.
	clientConfig, err := config.createClientConfigFromFile()
	if err != nil {
		glog.Fatalf("Failed to create a ClientConfig: %v. Exiting.", err)
	}

	clientset, err := clientset.NewForConfig(clientConfig)
	if err != nil {
		glog.Fatalf("Failed to create a ClientSet: %v. Exiting.", err)
	}
	internalClientset, err := internalclientset.NewForConfig(clientConfig)
	if err != nil {
		glog.Fatalf("Failed to create an internal ClientSet: %v. Exiting.", err)
	}

	if config.Morph == "kubelet" {
		cadvisorInterface := new(cadvisortest.Fake)
		containerManager := cm.NewStubContainerManager()
		fakeDockerClient := libdocker.NewFakeDockerClient().WithTraceDisabled()
		fakeDockerClient.EnableSleep = true

		hollowKubelet := kubemark.NewHollowKubelet(
			config.NodeName,
			clientset,
			cadvisorInterface,
			fakeDockerClient,
			config.KubeletPort,
			config.KubeletReadOnlyPort,
			containerManager,
			maxPods,
			podsPerCore,
		)
		hollowKubelet.Run()
	}

	if config.Morph == "proxy" {
		eventClient, err := clientgoclientset.NewForConfig(clientConfig)
		if err != nil {
			glog.Fatalf("Failed to create API Server client: %v", err)
		}
		iptInterface := fakeiptables.NewFake()
		sysctl := fakesysctl.NewFake()
		execer := &fakeexec.FakeExec{}
		eventBroadcaster := record.NewBroadcaster()
		recorder := eventBroadcaster.NewRecorder(api.Scheme, clientv1.EventSource{Component: "kube-proxy", Host: config.NodeName})

		hollowProxy, err := kubemark.NewHollowProxyOrDie(
			config.NodeName,
			internalClientset,
			eventClient,
			iptInterface,
			sysctl,
			execer,
			eventBroadcaster,
			recorder,
			config.UseRealProxier,
		)
		if err != nil {
			glog.Fatalf("Failed to create hollowProxy instance: %v", err)
		}
		hollowProxy.Run()
	}
}
<commit_msg>hollow-node.go:delete useless para. and import<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"

	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apiserver/pkg/util/flag"
	clientgoclientset "k8s.io/client-go/kubernetes"
	clientv1 "k8s.io/client-go/pkg/api/v1"
	restclient "k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/client-go/tools/record"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/client/clientset_generated/clientset"
	"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
	_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
	cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
	"k8s.io/kubernetes/pkg/kubelet/cm"
	"k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker"
	"k8s.io/kubernetes/pkg/kubemark"
	fakeexec "k8s.io/kubernetes/pkg/util/exec"
	fakeiptables "k8s.io/kubernetes/pkg/util/iptables/testing"
	fakesysctl "k8s.io/kubernetes/pkg/util/sysctl/testing"
	_ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration

	"github.com/golang/glog"
	"github.com/spf13/pflag"
)

type HollowNodeConfig struct {
	KubeconfigPath      string
	KubeletPort         int
	KubeletReadOnlyPort int
	Morph               string
	NodeName            string
	ServerPort          int
	ContentType         string
	UseRealProxier      bool
}

const (
	maxPods     = 110
	podsPerCore = 0
)

var knownMorphs = sets.NewString("kubelet", "proxy")

func (c *HollowNodeConfig) addFlags(fs *pflag.FlagSet) {
	fs.StringVar(&c.KubeconfigPath, "kubeconfig", "/kubeconfig/kubeconfig", "Path to kubeconfig file.")
	fs.IntVar(&c.KubeletPort, "kubelet-port", 10250, "Port on which HollowKubelet should be listening.")
	fs.IntVar(&c.KubeletReadOnlyPort, "kubelet-read-only-port", 10255, "Read-only port on which Kubelet is listening.")
	fs.StringVar(&c.NodeName, "name", "fake-node", "Name of this Hollow Node.")
	fs.IntVar(&c.ServerPort, "api-server-port", 443, "Port on which API server is listening.")
	fs.StringVar(&c.Morph, "morph", "", fmt.Sprintf("Specifies into which Hollow component this binary should morph. Allowed values: %v", knownMorphs.List()))
	fs.StringVar(&c.ContentType, "kube-api-content-type", "application/vnd.kubernetes.protobuf", "ContentType of requests sent to apiserver.")
	fs.BoolVar(&c.UseRealProxier, "use-real-proxier", true, "Set to true if you want to use real proxier inside hollow-proxy.")
}

func (c *HollowNodeConfig) createClientConfigFromFile() (*restclient.Config, error) {
	clientConfig, err := clientcmd.LoadFromFile(c.KubeconfigPath)
	if err != nil {
		return nil, fmt.Errorf("error while loading kubeconfig from file %v: %v", c.KubeconfigPath, err)
	}
	config, err := clientcmd.NewDefaultClientConfig(*clientConfig, &clientcmd.ConfigOverrides{}).ClientConfig()
	if err != nil {
		return nil, fmt.Errorf("error while creating kubeconfig: %v", err)
	}
	config.ContentType = c.ContentType
	config.QPS = 10
	config.Burst = 20
	return config, nil
}

func main() {
	config := HollowNodeConfig{}
	config.addFlags(pflag.CommandLine)
	flag.InitFlags()

	if !knownMorphs.Has(config.Morph) {
		glog.Fatalf("Unknown morph: %v. Allowed values: %v", config.Morph, knownMorphs.List())
	}

	// create a client to communicate with API server.
	clientConfig, err := config.createClientConfigFromFile()
	if err != nil {
		glog.Fatalf("Failed to create a ClientConfig: %v. Exiting.", err)
	}

	clientset, err := clientset.NewForConfig(clientConfig)
	if err != nil {
		glog.Fatalf("Failed to create a ClientSet: %v. Exiting.", err)
	}
	internalClientset, err := internalclientset.NewForConfig(clientConfig)
	if err != nil {
		glog.Fatalf("Failed to create an internal ClientSet: %v. Exiting.", err)
	}

	if config.Morph == "kubelet" {
		cadvisorInterface := new(cadvisortest.Fake)
		containerManager := cm.NewStubContainerManager()
		fakeDockerClient := libdocker.NewFakeDockerClient().WithTraceDisabled()
		fakeDockerClient.EnableSleep = true

		hollowKubelet := kubemark.NewHollowKubelet(
			config.NodeName,
			clientset,
			cadvisorInterface,
			fakeDockerClient,
			config.KubeletPort,
			config.KubeletReadOnlyPort,
			containerManager,
			maxPods,
			podsPerCore,
		)
		hollowKubelet.Run()
	}

	if config.Morph == "proxy" {
		eventClient, err := clientgoclientset.NewForConfig(clientConfig)
		if err != nil {
			glog.Fatalf("Failed to create API Server client: %v", err)
		}
		iptInterface := fakeiptables.NewFake()
		sysctl := fakesysctl.NewFake()
		execer := &fakeexec.FakeExec{}
		eventBroadcaster := record.NewBroadcaster()
		recorder := eventBroadcaster.NewRecorder(api.Scheme, clientv1.EventSource{Component: "kube-proxy", Host: config.NodeName})

		hollowProxy, err := kubemark.NewHollowProxyOrDie(
			config.NodeName,
			internalClientset,
			eventClient,
			iptInterface,
			sysctl,
			execer,
			eventBroadcaster,
			recorder,
			config.UseRealProxier,
		)
		if err != nil {
			glog.Fatalf("Failed to create hollowProxy instance: %v", err)
		}
		hollowProxy.Run()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"path"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"

	"github.com/pborman/uuid"
	"github.intel.com/hpdd/logging/alert"
	"github.intel.com/hpdd/logging/debug"
	"github.intel.com/hpdd/policy/pdm/dmplugin"
	"github.intel.com/hpdd/policy/pkg/client"
)

// Mover is an S3 data mover
type Mover struct {
	name   string
	client *client.Client
	s3Svc  *s3.S3
	bucket string
	prefix string
}

// S3Mover returns a new *Mover
func S3Mover(c *client.Client, s3Svc *s3.S3, archiveID uint32, bucket string, prefix string) *Mover {
	return &Mover{
		name:   fmt.Sprintf("s3-%d", archiveID),
		client: c,
		s3Svc:  s3Svc,
		bucket: bucket,
		prefix: prefix,
	}
}

// Base returns the base path in which the mover is operating
func (m *Mover) Base() string {
	return m.client.Path()
}

func newFileID() string {
	return uuid.New()
}

func (m *Mover) destination(id string) string {
	return path.Join(m.prefix,
		"objects",
		fmt.Sprintf("%s", id[0:2]),
		fmt.Sprintf("%s", id[2:4]),
		id)
}

func (m *Mover) newUploader() *s3manager.Uploader {
	// can configure stuff here with custom setters, e.g.
	// var partSize10 = func(u *Uploader) {
	//     u.PartSize = 1024 * 1024 * 10
	// }
	// s3manager.NewUploaderWithClient(m.s3Svc, partSize10)
	return s3manager.NewUploaderWithClient(m.s3Svc)
}

func (m *Mover) newDownloader() *s3manager.Downloader {
	return s3manager.NewDownloaderWithClient(m.s3Svc)
}

// Archive fulfills an HSM Archive request
func (m *Mover) Archive(action *dmplugin.Action) error {
	debug.Printf("%s id:%d archive %s %s", m.name, action.ID(), action.PrimaryPath(), action.FileID())
	rate.Mark(1)
	start := time.Now()

	src, err := os.Open(path.Join(m.Base(), action.PrimaryPath()))
	if err != nil {
		return err
	}
	defer src.Close()

	fi, err := src.Stat()
	if err != nil {
		return err
	}

	fileID := newFileID()
	size := fi.Size()
	progressFunc := func(offset, length int64) error {
		return action.Update(offset, length, size)
	}
	progressReader := NewProgressReader(src, updateInterval, progressFunc)
	defer progressReader.StopUpdates()

	uploader := m.newUploader()
	out, err := uploader.Upload(&s3manager.UploadInput{
		Body:        progressReader,
		Bucket:      aws.String(m.bucket),
		Key:         aws.String(m.destination(fileID)),
		ContentType: aws.String("application/octet-stream"),
	})
	if err != nil {
		if multierr, ok := err.(s3manager.MultiUploadFailure); ok {
			alert.Warn("Upload error:", multierr.Code(), multierr.Message(), multierr.UploadID())
		}
		return err
	}

	debug.Printf("%s id:%d Archived %d bytes in %v from %s to %s", m.name, action.ID(), fi.Size(),
		time.Since(start),
		action.PrimaryPath(),
		out.Location)
	action.SetFileID([]byte(fileID))
	action.SetActualLength(uint64(fi.Size()))
	return nil
}

// Restore fulfills an HSM Restore request
func (m *Mover) Restore(action *dmplugin.Action) error {
	debug.Printf("%s id:%d restore %s %s", m.name, action.ID(), action.PrimaryPath(), action.FileID())
	rate.Mark(1)

	start := time.Now()
	if action.FileID() == "" {
		return fmt.Errorf("Missing file_id on action %d", action.ID())
	}

	srcObj := m.destination(action.FileID())
	out, err := m.s3Svc.HeadObject(&s3.HeadObjectInput{
		Bucket: aws.String(m.bucket),
		Key:    aws.String(srcObj),
	})
	if err != nil {
		return fmt.Errorf("s3.HeadObject() on %s failed: %s", srcObj, err)
	}
	debug.Printf("obj %s, size %d", srcObj, *out.ContentLength)

	dstSize := *out.ContentLength
	dstPath := path.Join(m.Base(), action.WritePath())
	dst, err := os.OpenFile(dstPath, os.O_WRONLY, 0644)
	if err != nil {
		return fmt.Errorf("Couldn't open %s for write: %s", dstPath, err)
	}
	defer dst.Close()

	progressFunc := func(offset, length int64) error {
		return action.Update(offset, length, dstSize)
	}
	progressWriter := NewProgressWriter(dst, updateInterval, progressFunc)
	defer progressWriter.StopUpdates()

	downloader := m.newDownloader()
	n, err := downloader.Download(progressWriter,
		&s3.GetObjectInput{
			Bucket: aws.String(m.bucket),
			Key:    aws.String(srcObj),
		})
	if err != nil {
		return fmt.Errorf("s3.Download() of %s failed: %s", srcObj, err)
	}

	debug.Printf("%s id:%d Restored %d bytes in %v from %s to %s", m.name, action.ID(), n,
		time.Since(start),
		srcObj,
		action.PrimaryPath())
	action.SetActualLength(uint64(n))
	return nil
}

// Remove fulfills an HSM Remove request
func (m *Mover) Remove(action *dmplugin.Action) error {
	debug.Printf("%s id:%d remove %s %s", m.name, action.ID(), action.PrimaryPath(), action.FileID())
	rate.Mark(1)

	_, err := m.s3Svc.DeleteObject(&s3.DeleteObjectInput{
		Bucket: aws.String(m.bucket),
		Key:    aws.String(m.destination(action.FileID())),
	})
	return err
}
<commit_msg>Fix compile pointer to interface errors<commit_after>package main

import (
	"fmt"
	"os"
	"path"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"

	"github.com/pborman/uuid"
	"github.intel.com/hpdd/logging/alert"
	"github.intel.com/hpdd/logging/debug"
	"github.intel.com/hpdd/policy/pdm/dmplugin"
	"github.intel.com/hpdd/policy/pkg/client"
)

// Mover is an S3 data mover
type Mover struct {
	name   string
	client client.Client
	s3Svc  *s3.S3
	bucket string
	prefix string
}

// S3Mover returns a new *Mover
func S3Mover(c client.Client, s3Svc *s3.S3, archiveID uint32, bucket string, prefix string) *Mover {
	return &Mover{
		name:   fmt.Sprintf("s3-%d", archiveID),
		client: c,
		s3Svc:  s3Svc,
		bucket: bucket,
		prefix: prefix,
	}
}

// Base returns the base path in which the mover is operating
func (m *Mover) Base() string {
	return m.client.Path()
}

func newFileID() string {
	return uuid.New()
}

func (m *Mover) destination(id string) string {
	return path.Join(m.prefix,
		"objects",
		fmt.Sprintf("%s", id[0:2]),
		fmt.Sprintf("%s", id[2:4]),
		id)
}

func (m *Mover) newUploader() *s3manager.Uploader {
	// can configure stuff here with custom setters, e.g.
	// var partSize10 = func(u *Uploader) {
	//     u.PartSize = 1024 * 1024 * 10
	// }
	// s3manager.NewUploaderWithClient(m.s3Svc, partSize10)
	return s3manager.NewUploaderWithClient(m.s3Svc)
}

func (m *Mover) newDownloader() *s3manager.Downloader {
	return s3manager.NewDownloaderWithClient(m.s3Svc)
}

// Archive fulfills an HSM Archive request
func (m *Mover) Archive(action *dmplugin.Action) error {
	debug.Printf("%s id:%d archive %s %s", m.name, action.ID(), action.PrimaryPath(), action.FileID())
	rate.Mark(1)
	start := time.Now()

	src, err := os.Open(path.Join(m.Base(), action.PrimaryPath()))
	if err != nil {
		return err
	}
	defer src.Close()

	fi, err := src.Stat()
	if err != nil {
		return err
	}

	fileID := newFileID()
	size := fi.Size()
	progressFunc := func(offset, length int64) error {
		return action.Update(offset, length, size)
	}
	progressReader := NewProgressReader(src, updateInterval, progressFunc)
	defer progressReader.StopUpdates()

	uploader := m.newUploader()
	out, err := uploader.Upload(&s3manager.UploadInput{
		Body:        progressReader,
		Bucket:      aws.String(m.bucket),
		Key:         aws.String(m.destination(fileID)),
		ContentType: aws.String("application/octet-stream"),
	})
	if err != nil {
		if multierr, ok := err.(s3manager.MultiUploadFailure); ok {
			alert.Warn("Upload error:", multierr.Code(), multierr.Message(), multierr.UploadID())
		}
		return err
	}

	debug.Printf("%s id:%d Archived %d bytes in %v from %s to %s", m.name, action.ID(), fi.Size(),
		time.Since(start),
		action.PrimaryPath(),
		out.Location)
	action.SetFileID([]byte(fileID))
	action.SetActualLength(uint64(fi.Size()))
	return nil
}

// Restore fulfills an HSM Restore request
func (m *Mover) Restore(action *dmplugin.Action) error {
	debug.Printf("%s id:%d restore %s %s", m.name, action.ID(), action.PrimaryPath(), action.FileID())
	rate.Mark(1)

	start := time.Now()
	if action.FileID() == "" {
		return fmt.Errorf("Missing file_id on action %d", action.ID())
	}

	srcObj := m.destination(action.FileID())
	out, err := m.s3Svc.HeadObject(&s3.HeadObjectInput{
		Bucket: aws.String(m.bucket),
		Key:    aws.String(srcObj),
	})
	if err != nil {
		return fmt.Errorf("s3.HeadObject() on %s failed: %s", srcObj, err)
	}
	debug.Printf("obj %s, size %d", srcObj, *out.ContentLength)

	dstSize := *out.ContentLength
	dstPath := path.Join(m.Base(), action.WritePath())
	dst, err := os.OpenFile(dstPath, os.O_WRONLY, 0644)
	if err != nil {
		return fmt.Errorf("Couldn't open %s for write: %s", dstPath, err)
	}
	defer dst.Close()

	progressFunc := func(offset, length int64) error {
		return action.Update(offset, length, dstSize)
	}
	progressWriter := NewProgressWriter(dst, updateInterval, progressFunc)
	defer progressWriter.StopUpdates()

	downloader := m.newDownloader()
	n, err := downloader.Download(progressWriter,
		&s3.GetObjectInput{
			Bucket: aws.String(m.bucket),
			Key:    aws.String(srcObj),
		})
	if err != nil {
		return fmt.Errorf("s3.Download() of %s failed: %s", srcObj, err)
	}

	debug.Printf("%s id:%d Restored %d bytes in %v from %s to %s", m.name, action.ID(), n,
		time.Since(start),
		srcObj,
		action.PrimaryPath())
	action.SetActualLength(uint64(n))
	return nil
}

// Remove fulfills an HSM Remove request
func (m *Mover) Remove(action *dmplugin.Action) error {
	debug.Printf("%s id:%d remove %s %s", m.name, action.ID(), action.PrimaryPath(), action.FileID())
	rate.Mark(1)

	_, err := m.s3Svc.DeleteObject(&s3.DeleteObjectInput{
		Bucket: aws.String(m.bucket),
		Key:    aws.String(m.destination(action.FileID())),
	})
	return err
}
<|endoftext|>