<commit_before>package mealplanner

import (
	"appengine/datastore"
	"appengine"
	"io"
	"os"
	"json"
	//"fmt"
)

type backup struct {
	Dishes              []Dish
	Ingredients         []Ingredient
	MeasuredIngredients map[string][]MeasuredIngredient
	Tags                map[string][]Word
	Pairings            map[string][]Pairing
	Menus               []Menu
}

type importer struct {
	context
	// decoded JSON data being imported
	jsonData backup
	// mapping from the string-id present in the jsonData
   //  to the actual datastore key of the entity after import
	//  only for the case that the string-id isn't a valid key for our
   //  library
	fixUpKeys map[string]*datastore.Key
	// an index of tags, (dish|ingredient)key -> tagstr -> dummy
   //  keyed based on the actual datastore key we will use, not
   //  the string from json
	allTags map[string]map[string] bool
	// slice of all new tags to be added
	newTags []interface{}
	// slice of the keys for the new tags to be added
	newTagKeys []*datastore.Key
}

func importFile(c *context, file io.Reader) {
	datastore.RunInTransaction(c.c, func(tc appengine.Context) os.Error {
		decoder := json.NewDecoder(file)
		data := backup{}
		err := decoder.Decode(&data)
		check(err)
		worker := &importer{
				context: *c,
				jsonData : data,
				fixUpKeys : make(map[string]*datastore.Key),
				allTags : make(map[string]map[string] bool),
				newTags : make([]interface{}, 0, 100),
				newTagKeys : make([]*datastore.Key, 0, 100),
		}
		worker.c = tc
		worker.doImport()
		return nil
	}, nil)
}

func (self *importer) doImport() {
	//fmt.Fprintf(self.w, "indexTags %v\n", self.allTags)
	self.indexCurrentTags()
	//self.debugPrintTags()
	//fmt.Fprintf(self.w, "ingredients\n")
	self.importIngredients()
	//fmt.Fprintf(self.w, "Dishes\n")
	self.importDishes()
	//self.debugPrintTags()
	//fmt.Fprintf(self.w, "MIs\n")
	self.importMeasuredIngredients()
	//fmt.Fprintf(self.w, "pairings\n")
	self.importPairings()
	//fmt.Fprintf(self.w, "menus\n")
	self.importMenus()
	//fmt.Fprintf(self.w, "tags\n")
	// add the tags we collected
	_, err := datastore.PutMulti(self.c, self.newTagKeys, self.newTags)
	check(err)
}

func (self *importer) indexCurrentTags() {
	// build an index of tags, (dish|ingredient)key -> tagstr -> tagkey
	query := self.NewQuery("Tags")
	iter := query.Run(self.c)
	word := &Word{}
	for key, err := iter.Next(word);
		err == nil;
		key, err = iter.Next(word) {
		parent := key.Parent().Encode()
		var m map[string]bool
		var found bool
		//fmt.Fprintf(self.w, "indexTags parent %v %v\n", parent, word.Word)
		if m, found = self.allTags[parent]; !found {
			//fmt.Fprintf(self.w, "indexTags make\n")
			m = make(map[string]bool)
			self.allTags[parent] = m
		}
		//fmt.Fprintf(self.w, "indexTags m %v\n", m)
		m[word.Word] = true
	}
}

func (self *importer) debugPrintTags() {
	j, _ := json.MarshalIndent(self.allTags, "", "\t")
	self.w.Write(j)
}

func (self *importer) restoreKey(encoded string, parent *datastore.Key) *datastore.Key {
	key, err := datastore.DecodeKey(encoded)
	check(err)
	if newKey, found := self.fixUpKeys[encoded]; found {
		return newKey
	}
	if !self.isInLibrary(key) {
		newKey := datastore.NewIncompleteKey(self.c, key.Kind(), parent)
		self.fixUpKeys[encoded] = newKey
		return newKey
	}
	return key
}

func (self *importer) importIngredients() {
	// get the previously listed ingredients
	// build an index by name
	prevIngredientsByName := self.indexItems(self.NewQuery("Ingredient"),
		&Ingredient{},
		func(key *datastore.Key, item interface{}) string {
			return item.(*Ingredient).Name
		})
	putItems := make([]interface{}, 0, len(self.jsonData.Ingredients))
	putKeys := make([]*datastore.Key, 0, len(self.jsonData.Ingredients))
	putIds := make([]string, 0, len(self.jsonData.Ingredients))

	// prepare all the ingredients
	for index, _ := range self.jsonData.Ingredients {
		i := &self.jsonData.Ingredients[index]
		id := i.Id
		key := self.restoreKey(id, self.lid)
		if key.Incomplete() {
			// check if we have an item of the same name already
			if ikey, ok := prevIngredientsByName[i.Name]; ok {
				self.fixUpKeys[id] = ikey
				key = ikey
			}
		}
		i.Id = ""
		putItems = append(putItems, i)
		putKeys = append(putKeys, key)
		putIds = append(putIds, id)
	}
	// put all the ingredients
	outKeys, err := datastore.PutMulti(self.c, putKeys, putItems)
	check(err)
   // update the fixUpKeys for any new items
	for index, putKey := range putKeys {
		if putKey.Incomplete() {
			self.fixUpKeys[putIds[index]] = outKeys[index]
		}
	}

	// add tags
	self.importTags(putIds, outKeys)
	// update keywords
	for index, _ := range putItems {
		ing := putItems[index].(*Ingredient)
		words := make(map[string]bool)
		addWords(ing.Name, words)
		addWords(ing.Category, words)
		for tag, _ := range self.allTags[outKeys[index].Encode()] {
			addWords(tag, words)
		}
		updateKeywords(self.c, outKeys[index], words)
	}
}

func (self *importer) importDishes() {
	// get the previously listed dishes
	// build an index by name
	prevDishesByImportId := self.indexItems(self.NewQuery("Dish"), &Dish{},
		func(key *datastore.Key, item interface{}) string {
			return item.(*Dish).Id
		})
	// lists for dishes being written
	count := len(self.jsonData.Dishes)
	putItems := make([]interface{}, 0, count)
	putKeys := make([]*datastore.Key, 0, count)
	putIds := make([]string, 0, count)

	// prepare all the dishes
	for index, _ := range self.jsonData.Dishes {
		i := &self.jsonData.Dishes[index]
		id := i.Id
		key := self.restoreKey(id, self.lid)
		if key.Incomplete() {
			// check if we have an item of the same name already
			if ikey, ok := prevDishesByImportId[id]; ok {
				self.fixUpKeys[id] = ikey
				key = ikey
			}
		}
		putItems = append(putItems, i)
		putKeys = append(putKeys, key)
		putIds = append(putIds, id)
	}
	// put all the dishes
	outKeys, err := datastore.PutMulti(self.c, putKeys, putItems)
	check(err)
   // update the fixUpKeys for any new items
	for index, putKey := range putKeys {
		if putKey.Incomplete() {
			self.fixUpKeys[putIds[index]] = outKeys[index]
		}
	}

	// add tags
	self.importTags(putIds, outKeys)
	// update keywords
	for index, _ := range putItems {
		dish := putItems[index].(*Dish)
		words := make(map[string]bool)
		addWords(dish.Name, words)
		addWords(dish.Source, words)
		for tag, _ := range self.allTags[outKeys[index].Encode()] {
			addWords(tag, words)
		}
		updateKeywords(self.c, outKeys[index], words)
	}
}

//jsonData.MeasuredIngredients map[string][]MeasuredIngredient
func (self *importer) importMeasuredIngredients() {
	miKeyFunc := func (key *datastore.Key, item interface{}) string {
				return key.Parent().Encode() + item.(*MeasuredIngredient).Ingredient.Encode();
			}
	// index existing items by their parent dish and the ingredient
	//  they reference
	prevMIs := self.indexItems(self.NewQuery("MeasuredIngredient"),
			 &MeasuredIngredient{}, miKeyFunc)
	count := len(self.jsonData.MeasuredIngredients)
	putItems := make([]interface{}, 0, count)
	putKeys := make([]*datastore.Key, 0, count)

	for dishId, jsonMis := range self.jsonData.MeasuredIngredients {
		dishKey := self.restoreKey(dishId, self.lid)
		dishKeyEncoded := dishKey.Encode()
		for index, _ := range jsonMis {
			jsonMi := &jsonMis[index]
			miKey := self.restoreKey(jsonMi.Id, dishKey)
			ingKey := self.restoreKey(jsonMi.Ingredient.Encode(), self.lid)
			if miKey.Incomplete() {
				miIndexKey := dishKeyEncoded + ingKey.Encode()
				if existingKey, found := prevMIs[miIndexKey]; found {
					miKey = existingKey
				}
			}
			jsonMi.Ingredient = ingKey
			jsonMi.Id = ""
			putItems = append(putItems, jsonMi)
			putKeys = append(putKeys, miKey)
		}
	}
	if len(putKeys) > 0 {
		_, err := datastore.PutMulti(self.c, putKeys, putItems)
		check(err)
	}
}

//jsonData.Pairings map[string][]Pairing
func (self *importer) importPairings() {
	pairingKeyFunc := func (key *datastore.Key, item interface{}) string {
					  return key.Parent().Encode() + item.(*Pairing).Other.Encode() + item.(*Pairing).Description;
				  }
	// index existing items by their parent dish and the ingredient
	//  they reference
	prevPairings := self.indexItems(self.NewQuery("Pairing"),
			 &Pairing{}, pairingKeyFunc)
	count := len(self.jsonData.Pairings)
	putItems := make([]interface{}, 0, count)
	putKeys := make([]*datastore.Key, 0, count)

	for dishId, jsonPairings := range self.jsonData.Pairings {
		dishKey := self.restoreKey(dishId, self.lid)
		dishKeyEncoded := dishKey.Encode()
		for index, _ := range jsonPairings {
			jsonPairing := &jsonPairings[index]
			pairingKey := self.restoreKey(jsonPairing.Id, dishKey)
			otherKey := self.restoreKey(jsonPairing.Other.Encode(), self.lid)
			pairingIndexKey := dishKeyEncoded + otherKey.Encode() + jsonPairing.Description
			// add the new pairing if it wasn't already present
			if _, found := prevPairings[pairingIndexKey]; !found {
				jsonPairing.Other = otherKey
				jsonPairing.Id = ""
				putItems = append(putItems, jsonPairing)
				putKeys = append(putKeys, pairingKey)
			}
		}
	}
	if len(putKeys) > 0 {
		_, err := datastore.PutMulti(self.c, putKeys, putItems)
		check(err)
	}
}

func (self *importer) importMenus() {
	menuKeyFunc := func (key *datastore.Key, item interface{}) string {
						 return item.(*Menu).Name
					 }
	// index existing items by their name
	prevMenus := self.indexItems(self.NewQuery("Menu"), &Menu{},
						  menuKeyFunc)
	count := len(self.jsonData.Menus)
	putItems := make([]interface{}, 0, count)
	putKeys := make([]*datastore.Key, 0, count)
	for index, _ := range self.jsonData.Menus {
		jsonMenu := &self.jsonData.Menus[index]
		key := self.restoreKey(jsonMenu.Id, self.lid)
		if key.Incomplete() {
			if existingKey, found := prevMenus[jsonMenu.Name]; found {
				key = existingKey
			}
		}
		for index, dishKey := range jsonMenu.Dishes {
			jsonMenu.Dishes[index] = self.restoreKey(dishKey.Encode(), self.lid)
		}
		jsonMenu.Id = ""
		putItems = append(putItems, jsonMenu)
		putKeys = append(putKeys, key)
	}
	if len(putKeys) > 0 {
		_, err := datastore.PutMulti(self.c, putKeys, putItems)
		check(err)
	}
}

// jsonData.Tags map[string][]Word
// take a slice of ids as appeared in json and slice of
//  keys as now stored for items with tags
// add any tags for these items that were in json but not yet stored
func (self *importer) importTags (ids []string, keys []*datastore.Key) {
	for index, parentKey := range keys {
		parentId := ids[index]
		destId := keys[index].Encode()
		if importTags, ok := self.jsonData.Tags[parentId]; ok {
			var myTags map[string]bool
			// if this item doesn't have a tags collection yet,
			// add it
			var found bool
			if myTags, found = self.allTags[destId]; !found {
				myTags = make(map[string]bool)
				self.allTags[destId] = myTags
			}
			// go through tags from json
			for _, tag := range importTags {
				if _, found := myTags[tag.Word]; !found {
					// this tag doesn't exist yet, add to our list
					self.newTags = append(self.newTags, &Word{"",tag.Word})
					newTagKey := datastore.NewIncompleteKey(self.c, "Tags", parentKey)
					self.newTagKeys = append(self.newTagKeys, newTagKey)
					myTags[tag.Word] = true
				}
			}
		}
	}
}

func (self *importer) indexItems(query *datastore.Query, item interface{},
	keyFunc func (*datastore.Key, interface{}) string ) map[string]*datastore.Key {
	index := make(map[string]*datastore.Key)
	iter := query.Run(self.c)
	for dataKey, err := iter.Next(item);
			err == nil;
			dataKey, err = iter.Next(item) {
		indexKey := keyFunc(dataKey, item)
		if len(indexKey) > 0 {
			index[indexKey] = dataKey
		}
	}
	return index
}

<commit_msg>fix import to skip invalid entries<commit_after>package mealplanner

import (
	"appengine/datastore"
	"appengine"
	"io"
	"os"
	"json"
	//"fmt"
)

type backup struct {
	Dishes              []Dish
	Ingredients         []Ingredient
	MeasuredIngredients map[string][]MeasuredIngredient
	Tags                map[string][]Word
	Pairings            map[string][]Pairing
	Menus               []Menu
}

type importer struct {
	context
	// decoded JSON data being imported
	jsonData backup
	// mapping from the string-id present in the jsonData
   //  to the actual datastore key of the entity after import
	//  only for the case that the string-id isn't a valid key for our
   //  library
	fixUpKeys map[string]*datastore.Key
	// an index of tags, (dish|ingredient)key -> tagstr -> dummy
   //  keyed based on the actual datastore key we will use, not
   //  the string from json
	allTags map[string]map[string] bool
	// slice of all new tags to be added
	newTags []interface{}
	// slice of the keys for the new tags to be added
	newTagKeys []*datastore.Key
}

func importFile(c *context, file io.Reader) {
	datastore.RunInTransaction(c.c, func(tc appengine.Context) os.Error {
		decoder := json.NewDecoder(file)
		data := backup{}
		err := decoder.Decode(&data)
		check(err)
		worker := &importer{
				context: *c,
				jsonData : data,
				fixUpKeys : make(map[string]*datastore.Key),
				allTags : make(map[string]map[string] bool),
				newTags : make([]interface{}, 0, 100),
				newTagKeys : make([]*datastore.Key, 0, 100),
		}
		worker.c = tc
		worker.doImport()
		return nil
	}, nil)
}

func (self *importer) doImport() {
	//fmt.Fprintf(self.w, "indexTags %v\n", self.allTags)
	self.indexCurrentTags()
	//self.debugPrintTags()
	//fmt.Fprintf(self.w, "ingredients\n")
	self.importIngredients()
	//fmt.Fprintf(self.w, "Dishes\n")
	self.importDishes()
	//self.debugPrintTags()
	//fmt.Fprintf(self.w, "MIs\n")
	self.importMeasuredIngredients()
	//fmt.Fprintf(self.w, "pairings\n")
	self.importPairings()
	//fmt.Fprintf(self.w, "menus\n")
	self.importMenus()
	//fmt.Fprintf(self.w, "tags\n")
	// add the tags we collected
	_, err := datastore.PutMulti(self.c, self.newTagKeys, self.newTags)
	check(err)
}

func (self *importer) indexCurrentTags() {
	// build an index of tags, (dish|ingredient)key -> tagstr -> tagkey
	query := self.NewQuery("Tags")
	iter := query.Run(self.c)
	word := &Word{}
	for key, err := iter.Next(word);
		err == nil;
		key, err = iter.Next(word) {
		parent := key.Parent().Encode()
		var m map[string]bool
		var found bool
		//fmt.Fprintf(self.w, "indexTags parent %v %v\n", parent, word.Word)
		if m, found = self.allTags[parent]; !found {
			//fmt.Fprintf(self.w, "indexTags make\n")
			m = make(map[string]bool)
			self.allTags[parent] = m
		}
		//fmt.Fprintf(self.w, "indexTags m %v\n", m)
		m[word.Word] = true
	}
}

func (self *importer) debugPrintTags() {
	j, _ := json.MarshalIndent(self.allTags, "", "\t")
	self.w.Write(j)
}

func (self *importer) restoreKey(encoded string, parent *datastore.Key) *datastore.Key {
	key, err := datastore.DecodeKey(encoded)
	check(err)
	if newKey, found := self.fixUpKeys[encoded]; found {
		return newKey
	}
	if !self.isInLibrary(key) {
		newKey := datastore.NewIncompleteKey(self.c, key.Kind(), parent)
		self.fixUpKeys[encoded] = newKey
		return newKey
	}
	return key
}

func (self *importer) importIngredients() {
	// get the previously listed ingredients
	// build an index by name
	prevIngredientsByName := self.indexItems(self.NewQuery("Ingredient"),
		&Ingredient{},
		func(key *datastore.Key, item interface{}) string {
			return item.(*Ingredient).Name
		})
	putItems := make([]interface{}, 0, len(self.jsonData.Ingredients))
	putKeys := make([]*datastore.Key, 0, len(self.jsonData.Ingredients))
	putIds := make([]string, 0, len(self.jsonData.Ingredients))

	// prepare all the ingredients
	for index, _ := range self.jsonData.Ingredients {
		i := &self.jsonData.Ingredients[index]
		id := i.Id
		key := self.restoreKey(id, self.lid)
		if key.Incomplete() {
			// check if we have an item of the same name already
			if ikey, ok := prevIngredientsByName[i.Name]; ok {
				self.fixUpKeys[id] = ikey
				key = ikey
			}
		}
		i.Id = ""
		putItems = append(putItems, i)
		putKeys = append(putKeys, key)
		putIds = append(putIds, id)
	}
	// put all the ingredients
	outKeys, err := datastore.PutMulti(self.c, putKeys, putItems)
	check(err)
   // update the fixUpKeys for any new items
	for index, putKey := range putKeys {
		if putKey.Incomplete() {
			self.fixUpKeys[putIds[index]] = outKeys[index]
		}
	}

	// add tags
	self.importTags(putIds, outKeys)
	// update keywords
	for index, _ := range putItems {
		ing := putItems[index].(*Ingredient)
		words := make(map[string]bool)
		addWords(ing.Name, words)
		addWords(ing.Category, words)
		for tag, _ := range self.allTags[outKeys[index].Encode()] {
			addWords(tag, words)
		}
		updateKeywords(self.c, outKeys[index], words)
	}
}

func (self *importer) importDishes() {
	// get the previously listed dishes
	// build an index by name
	prevDishesByImportId := self.indexItems(self.NewQuery("Dish"), &Dish{},
		func(key *datastore.Key, item interface{}) string {
			return item.(*Dish).Id
		})
	// lists for dishes being written
	count := len(self.jsonData.Dishes)
	putItems := make([]interface{}, 0, count)
	putKeys := make([]*datastore.Key, 0, count)
	putIds := make([]string, 0, count)

	// prepare all the dishes
	for index, _ := range self.jsonData.Dishes {
		i := &self.jsonData.Dishes[index]
		id := i.Id
		key := self.restoreKey(id, self.lid)
		if key.Incomplete() {
			// check if we have an item of the same name already
			if ikey, ok := prevDishesByImportId[id]; ok {
				self.fixUpKeys[id] = ikey
				key = ikey
			}
		}
		putItems = append(putItems, i)
		putKeys = append(putKeys, key)
		putIds = append(putIds, id)
	}
	// put all the dishes
	outKeys, err := datastore.PutMulti(self.c, putKeys, putItems)
	check(err)
   // update the fixUpKeys for any new items
	for index, putKey := range putKeys {
		if putKey.Incomplete() {
			self.fixUpKeys[putIds[index]] = outKeys[index]
		}
	}

	// add tags
	self.importTags(putIds, outKeys)
	// update keywords
	for index, _ := range putItems {
		dish := putItems[index].(*Dish)
		words := make(map[string]bool)
		addWords(dish.Name, words)
		addWords(dish.Source, words)
		for tag, _ := range self.allTags[outKeys[index].Encode()] {
			addWords(tag, words)
		}
		updateKeywords(self.c, outKeys[index], words)
	}
}

//jsonData.MeasuredIngredients map[string][]MeasuredIngredient
func (self *importer) importMeasuredIngredients() {
	miKeyFunc := func (key *datastore.Key, item interface{}) string {
				return key.Parent().Encode() + item.(*MeasuredIngredient).Ingredient.Encode();
			}
	// index existing items by their parent dish and the ingredient
	//  they reference
	prevMIs := self.indexItems(self.NewQuery("MeasuredIngredient"),
			 &MeasuredIngredient{}, miKeyFunc)
	count := len(self.jsonData.MeasuredIngredients)
	putItems := make([]interface{}, 0, count)
	putKeys := make([]*datastore.Key, 0, count)

	for dishId, jsonMis := range self.jsonData.MeasuredIngredients {
		dishKey := self.restoreKey(dishId, self.lid)
		dishKeyEncoded := dishKey.Encode()
		for index, _ := range jsonMis {
			jsonMi := &jsonMis[index]
			miKey := self.restoreKey(jsonMi.Id, dishKey)
			ingKey := self.restoreKey(jsonMi.Ingredient.Encode(), self.lid)
			// if we didn't import the ingredient, we need to skip this one
			if ingKey.Incomplete() {
				continue
			}
			if miKey.Incomplete() {
				miIndexKey := dishKeyEncoded + ingKey.Encode()
				if existingKey, found := prevMIs[miIndexKey]; found {
					miKey = existingKey
				}
			}
			jsonMi.Ingredient = ingKey
			jsonMi.Id = ""
			putItems = append(putItems, jsonMi)
			putKeys = append(putKeys, miKey)
		}
	}
	if len(putKeys) > 0 {
		_, err := datastore.PutMulti(self.c, putKeys, putItems)
		check(err)
	}
}

//jsonData.Pairings map[string][]Pairing
func (self *importer) importPairings() {
	pairingKeyFunc := func (key *datastore.Key, item interface{}) string {
					  return key.Parent().Encode() + item.(*Pairing).Other.Encode() + item.(*Pairing).Description;
				  }
	// index existing items by their parent dish and the ingredient
	//  they reference
	prevPairings := self.indexItems(self.NewQuery("Pairing"),
			 &Pairing{}, pairingKeyFunc)
	count := len(self.jsonData.Pairings)
	putItems := make([]interface{}, 0, count)
	putKeys := make([]*datastore.Key, 0, count)

	for dishId, jsonPairings := range self.jsonData.Pairings {
		dishKey := self.restoreKey(dishId, self.lid)
		dishKeyEncoded := dishKey.Encode()
		for index, _ := range jsonPairings {
			jsonPairing := &jsonPairings[index]
			pairingKey := self.restoreKey(jsonPairing.Id, dishKey)
			otherKey := self.restoreKey(jsonPairing.Other.Encode(), self.lid)
			if otherKey.Incomplete() {
				// if we didn't import the referenced item
				//  we have to skip this one
				continue
			}
			pairingIndexKey := dishKeyEncoded + otherKey.Encode() + jsonPairing.Description
			// add the new pairing if it wasn't already present
			if _, found := prevPairings[pairingIndexKey]; !found {
				jsonPairing.Other = otherKey
				jsonPairing.Id = ""
				putItems = append(putItems, jsonPairing)
				putKeys = append(putKeys, pairingKey)
			}
		}
	}
	if len(putKeys) > 0 {
		_, err := datastore.PutMulti(self.c, putKeys, putItems)
		check(err)
	}
}

func (self *importer) importMenus() {
	menuKeyFunc := func (key *datastore.Key, item interface{}) string {
						 return item.(*Menu).Name
					 }
	// index existing items by their name
	prevMenus := self.indexItems(self.NewQuery("Menu"), &Menu{},
						  menuKeyFunc)
	count := len(self.jsonData.Menus)
	putItems := make([]interface{}, 0, count)
	putKeys := make([]*datastore.Key, 0, count)
	for index, _ := range self.jsonData.Menus {
		jsonMenu := &self.jsonData.Menus[index]
		key := self.restoreKey(jsonMenu.Id, self.lid)
		if key.Incomplete() {
			if existingKey, found := prevMenus[jsonMenu.Name]; found {
				key = existingKey
			}
		}
		for index, dishKey := range jsonMenu.Dishes {
			jsonMenu.Dishes[index] = self.restoreKey(dishKey.Encode(), self.lid)
		}
		jsonMenu.Id = ""
		putItems = append(putItems, jsonMenu)
		putKeys = append(putKeys, key)
	}
	if len(putKeys) > 0 {
		_, err := datastore.PutMulti(self.c, putKeys, putItems)
		check(err)
	}
}

// jsonData.Tags map[string][]Word
// take a slice of ids as appeared in json and slice of
//  keys as now stored for items with tags
// add any tags for these items that were in json but not yet stored
func (self *importer) importTags (ids []string, keys []*datastore.Key) {
	for index, parentKey := range keys {
		parentId := ids[index]
		destId := keys[index].Encode()
		if importTags, ok := self.jsonData.Tags[parentId]; ok {
			var myTags map[string]bool
			// if this item doesn't have a tags collection yet,
			// add it
			var found bool
			if myTags, found = self.allTags[destId]; !found {
				myTags = make(map[string]bool)
				self.allTags[destId] = myTags
			}
			// go through tags from json
			for _, tag := range importTags {
				if _, found := myTags[tag.Word]; !found {
					// this tag doesn't exist yet, add to our list
					self.newTags = append(self.newTags, &Word{"",tag.Word})
					newTagKey := datastore.NewIncompleteKey(self.c, "Tags", parentKey)
					self.newTagKeys = append(self.newTagKeys, newTagKey)
					myTags[tag.Word] = true
				}
			}
		}
	}
}

func (self *importer) indexItems(query *datastore.Query, item interface{},
	keyFunc func (*datastore.Key, interface{}) string ) map[string]*datastore.Key {
	index := make(map[string]*datastore.Key)
	iter := query.Run(self.c)
	for dataKey, err := iter.Next(item);
			err == nil;
			dataKey, err = iter.Next(item) {
		indexKey := keyFunc(dataKey, item)
		if len(indexKey) > 0 {
			index[indexKey] = dataKey
		}
	}
	return index
}

<|endoftext|>


<commit_before>package goexcel

import (
	"bytes"
	"encoding/csv"
	"errors"
	"net/http"
	"reflect"
	"strings"

	"google.golang.org/appengine"
	"google.golang.org/appengine/log"

	"golang.org/x/net/context"

	"github.com/news-ai/tabulae/models"
)

func csvGetCustomFields(r *http.Request, c context.Context, numberOfColumns int, headers []string) map[string]bool {
	customFields := make(map[string]bool, len(headers))

	for x := 0; x < numberOfColumns; x++ {
		columnName := headers[x]
		if !customOrNative(columnName) {
			customFields[columnName] = true
		}
	}
	return customFields
}

func csvRowToContact(r *http.Request, c context.Context, singleRow []string, headers []string) (models.Contact, error) {
	var (
		contact       models.Contact
		employers     []int64
		pastEmployers []int64
		customFields  []models.CustomContactField
	)

	for x := 0; x < len(singleRow); x++ {
		columnName := headers[x]
		cellName := singleRow[x]
		rowToContact(r, c, columnName, cellName, &contact, &employers, &pastEmployers, &customFields)
	}

	contact.CustomFields = customFields
	contact.Employers = employers
	contact.PastEmployers = pastEmployers
	return contact, nil
}

func csvToContactList(r *http.Request, file []byte, headers []string) ([]models.Contact, map[string]bool, error) {
	c := appengine.NewContext(r)

	readerFile := bytes.NewReader(file)
	incomingRecords := csv.NewReader(readerFile)
	records, err := incomingRecords.ReadAll()
	if err != nil {
		log.Infof(c, "%v", err)
		return []models.Contact{}, map[string]bool{}, err
	}

	// Number of columns in sheet to compare
	numberOfColumns := len(records[0])
	if numberOfColumns != len(headers) {
		err := errors.New("Number of headers does not match the ones for the sheet")
		log.Infof(c, "%v", err)
		return []models.Contact{}, map[string]bool{}, err
	}

	// Loop through all the rows
	// Extract information
	emptyContact := models.Contact{}
	contacts := []models.Contact{}

	for i := 0; i < len(records); i++ {
		contact, err := csvRowToContact(r, c, records[i], headers)
		if err != nil {
			log.Infof(c, "%v", err)
			return []models.Contact{}, map[string]bool{}, err
		}

		// To get rid of empty contacts. We don't want to create empty contacts.
		if !reflect.DeepEqual(emptyContact, contact) {
			contacts = append(contacts, contact)
		}
	}

	// Get custom fields
	customFields := getCustomFields(r, c, numberOfColumns, headers)

	return contacts, customFields, nil
}

func csvFileToExcelHeader(r *http.Request, file []byte) ([]Column, error) {
	c := appengine.NewContext(r)

	readerFile := bytes.NewReader(file)
	incomingRecords := csv.NewReader(readerFile)
	records, err := incomingRecords.ReadAll()
	if err != nil {
		log.Errorf(c, "%v", err)
		return []Column{}, err
	}

	numberOfRows := 15
	if len(records) < numberOfRows+1 {
		numberOfRows = len(records)
	}

	numberOfColumns := len(records[0])
	columns := make([]Column, numberOfColumns)

	for _, row := range records[0:numberOfRows] {
		for currentColumn, cell := range row {
			columns[currentColumn].Rows = append(columns[currentColumn].Rows, strings.Trim(cell, " "))
		}
	}

	return columns, nil
}
<commit_msg>Adding lazy quotation marks<commit_after>package goexcel

import (
	"bytes"
	"encoding/csv"
	"errors"
	"net/http"
	"reflect"
	"strings"

	"google.golang.org/appengine"
	"google.golang.org/appengine/log"

	"golang.org/x/net/context"

	"github.com/news-ai/tabulae/models"
)

func csvGetCustomFields(r *http.Request, c context.Context, numberOfColumns int, headers []string) map[string]bool {
	customFields := make(map[string]bool, len(headers))

	for x := 0; x < numberOfColumns; x++ {
		columnName := headers[x]
		if !customOrNative(columnName) {
			customFields[columnName] = true
		}
	}
	return customFields
}

func csvRowToContact(r *http.Request, c context.Context, singleRow []string, headers []string) (models.Contact, error) {
	var (
		contact       models.Contact
		employers     []int64
		pastEmployers []int64
		customFields  []models.CustomContactField
	)

	for x := 0; x < len(singleRow); x++ {
		columnName := headers[x]
		cellName := singleRow[x]
		rowToContact(r, c, columnName, cellName, &contact, &employers, &pastEmployers, &customFields)
	}

	contact.CustomFields = customFields
	contact.Employers = employers
	contact.PastEmployers = pastEmployers
	return contact, nil
}

func csvToContactList(r *http.Request, file []byte, headers []string) ([]models.Contact, map[string]bool, error) {
	c := appengine.NewContext(r)

	readerFile := bytes.NewReader(file)
	incomingRecords := csv.NewReader(readerFile)
	records, err := incomingRecords.ReadAll()
	if err != nil {
		log.Infof(c, "%v", err)
		return []models.Contact{}, map[string]bool{}, err
	}

	// Number of columns in sheet to compare
	numberOfColumns := len(records[0])
	if numberOfColumns != len(headers) {
		err := errors.New("Number of headers does not match the ones for the sheet")
		log.Infof(c, "%v", err)
		return []models.Contact{}, map[string]bool{}, err
	}

	// Loop through all the rows
	// Extract information
	emptyContact := models.Contact{}
	contacts := []models.Contact{}

	for i := 0; i < len(records); i++ {
		contact, err := csvRowToContact(r, c, records[i], headers)
		if err != nil {
			log.Infof(c, "%v", err)
			return []models.Contact{}, map[string]bool{}, err
		}

		// To get rid of empty contacts. We don't want to create empty contacts.
		if !reflect.DeepEqual(emptyContact, contact) {
			contacts = append(contacts, contact)
		}
	}

	// Get custom fields
	customFields := getCustomFields(r, c, numberOfColumns, headers)

	return contacts, customFields, nil
}

func csvFileToExcelHeader(r *http.Request, file []byte) ([]Column, error) {
	c := appengine.NewContext(r)

	readerFile := bytes.NewReader(file)
	incomingRecords := csv.NewReader(readerFile)
	incomingRecords.LazyQuotes = true
	records, err := incomingRecords.ReadAll()
	if err != nil {
		log.Errorf(c, "%v", err)
		return []Column{}, err
	}

	numberOfRows := 15
	if len(records) < numberOfRows+1 {
		numberOfRows = len(records)
	}

	numberOfColumns := len(records[0])
	columns := make([]Column, numberOfColumns)

	for _, row := range records[0:numberOfRows] {
		for currentColumn, cell := range row {
			columns[currentColumn].Rows = append(columns[currentColumn].Rows, strings.Trim(cell, " "))
		}
	}

	return columns, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"os"
	"strings"

	"v.io/x/mojo/tests/expected"
	"v.io/x/mojo/tests/util"
)

var runBench *bool = flag.Bool("bench", false, "run benchmarks instead of tests")
var clientType *string = flag.String("client", "go", "run test with a different client")
var serverType *string = flag.String("server", "go", "run test with a different server")

var (
	// Maps the client type to client mojo files.
	clientMap = map[string]string{
		"go":   "test_client.mojo",
		"dart": "dart-tests/end_to_end_test/lib/client.dart",
	}

	// Maps the server type to server mojo files.
	serverMap = map[string]string{
		"go":   "test_server.mojo",
		"dart": "dart-tests/end_to_end_test/lib/server.dart",
	}
)

func main() {
	flag.Parse()
	wd, err := os.Getwd()
	if err != nil {
		panic(err)
	}
	v23proxy, err := util.StartV23ServerProxy(wd)
	if err != nil {
		panic(err)
	}
	endpoint, err := v23proxy.Endpoint()
	if err != nil {
		panic(err)
	}
	endpointFlag := fmt.Sprintf("-endpoint=%s/https://mojo.v.io/%s/mojo::v23proxy::tests::V23ProxyTest", endpoint, serverMap[*serverType])
	args := []string{endpointFlag, "--v23.tcp.address=127.0.0.1:0"}
	if *runBench {
		args = append(args, "-test.run=XXXX", "-test.bench=.")
	}
	if err := runTestClient(wd, args...); err != nil {
		fmt.Fprintf(os.Stderr, "%s", err)
		os.Exit(2)
	}
	if err := v23proxy.Stop(); err != nil {
		panic(err)
	}
}

func runTestClient(v23ProxyRoot string, args ...string) error {
	cmd := util.RunMojoShellForV23ProxyTests(clientMap[*clientType], v23ProxyRoot, args)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}
	var stdoutBuf bytes.Buffer
	go func() {
		io.Copy(os.Stdout, io.TeeReader(stdout, &stdoutBuf))
	}()
	stderr, err := cmd.StderrPipe()
	if err != nil {
		return err
	}
	go func() {
		io.Copy(os.Stderr, stderr)
	}()

	if err := cmd.Run(); err != nil {
		return err
	}

	scanner := bufio.NewScanner(bytes.NewReader(stdoutBuf.Bytes()))
	for scanner.Scan() {
		if strings.HasSuffix(scanner.Text(), expected.SuccessMessage) {
			return nil
		}
	}
	if err := scanner.Err(); err != nil {
		return err
	}
	return fmt.Errorf("TESTS FAILED")
}
<commit_msg>mojo/v23proxy: Add logs to isolate Flake area<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"os"
	"strings"
	"time"

	"v.io/x/mojo/tests/expected"
	"v.io/x/mojo/tests/util"
)

var runBench *bool = flag.Bool("bench", false, "run benchmarks instead of tests")
var clientType *string = flag.String("client", "go", "run test with a different client")
var serverType *string = flag.String("server", "go", "run test with a different server")

var (
	// Maps the client type to client mojo files.
	clientMap = map[string]string{
		"go":   "test_client.mojo",
		"dart": "dart-tests/end_to_end_test/lib/client.dart",
	}

	// Maps the server type to server mojo files.
	serverMap = map[string]string{
		"go":   "test_server.mojo",
		"dart": "dart-tests/end_to_end_test/lib/server.dart",
	}
)

func main() {
	flag.Parse()
	wd, err := os.Getwd()
	if err != nil {
		panic(err)
	}
	fmt.Println("DEBUG LOG: Starting server proxy...")
	v23proxy, err := util.StartV23ServerProxy(wd)
	if err != nil {
		panic(err)
	}

	fmt.Println("DEBUG LOG: Waiting for v23 endpoint...")
	timeout := make(chan bool, 1)
	endpoint_chan := make(chan string, 1)
	go func() {
		time.Sleep(15 * time.Second)
		timeout <- true
	}()
	go func() {
		endpoint, err := v23proxy.Endpoint()
		if err != nil {
			panic(err)
		}
		endpoint_chan <- endpoint
	}()

	var endpoint string
	select {
	case endpoint = <-endpoint_chan:
		// keep going!
		break
	case <-timeout:
		panic("Failed to read endpoint!")
	}
	fmt.Println("DEBUG LOG: Starting client proxy...")

	endpointFlag := fmt.Sprintf("-endpoint=%s/https://mojo.v.io/%s/mojo::v23proxy::tests::V23ProxyTest", endpoint, serverMap[*serverType])
	args := []string{endpointFlag, "--v23.tcp.address=127.0.0.1:0"}
	if *runBench {
		args = append(args, "-test.run=XXXX", "-test.bench=.")
	}
	if err := runTestClient(wd, args...); err != nil {
		fmt.Fprintf(os.Stderr, "%s", err)
		os.Exit(2)
	}
	if err := v23proxy.Stop(); err != nil {
		panic(err)
	}
}

func runTestClient(v23ProxyRoot string, args ...string) error {
	cmd := util.RunMojoShellForV23ProxyTests(clientMap[*clientType], v23ProxyRoot, args)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}
	var stdoutBuf bytes.Buffer
	go func() {
		io.Copy(os.Stdout, io.TeeReader(stdout, &stdoutBuf))
	}()
	stderr, err := cmd.StderrPipe()
	if err != nil {
		return err
	}
	go func() {
		io.Copy(os.Stderr, stderr)
	}()

	if err := cmd.Run(); err != nil {
		return err
	}

	scanner := bufio.NewScanner(bytes.NewReader(stdoutBuf.Bytes()))
	for scanner.Scan() {
		if strings.HasSuffix(scanner.Text(), expected.SuccessMessage) {
			return nil
		}
	}
	if err := scanner.Err(); err != nil {
		return err
	}
	return fmt.Errorf("TESTS FAILED")
}
<|endoftext|>


<commit_before>package purchase

// NeedsLicense determines whether a license is need to drive a type of vehicle. Only "car" and "truck" require a license.
func NeedsLicense(kind string) bool {
	panic("NeedsLicense not implemented")
}

// ChooseVehicle recommends a vehicle for selection. It always recommends the vehicle that comes first in dictionary order.
func ChooseVehicle(option1, option2 string) string {
	panic("ChooseVehicle not implemented")
}

// CalculateResellPrice calculates how much a vehicle can resell for at a certain age.
func CalculateResellPrice(originalPrice, age float64) float64 {
	panic("CalculateResellPrice not implemented")
}
<commit_msg>Solve vehicle purchase<commit_after>package purchase

import (
	"fmt"
	"strings"
)

// NeedsLicense determines whether a license is need to drive a type of vehicle. Only "car" and "truck" require a license.
func NeedsLicense(kind string) bool {
	return kind == "car" || kind == "truck"
}

// ChooseVehicle recommends a vehicle for selection. It always recommends the vehicle that comes first in dictionary order.
func ChooseVehicle(option1, option2 string) string {
	if strings.Compare(option1, option2) == 1 {
		return fmt.Sprintf("%s is clearly the better choice.", option2)
	}
	return fmt.Sprintf("%s is clearly the better choice.", option1)
}

// CalculateResellPrice calculates how much a vehicle can resell for at a certain age.
func CalculateResellPrice(originalPrice, age float64) float64 {
	if age < 3 {
		return originalPrice * .8
	} else if age < 10 {
		return originalPrice * .7
	} else {
		return originalPrice * .5
	}
}
<|endoftext|>


<commit_before>package knsq

import (
	"io"
	"log"
	"net/http"
	"net/url"
)

// TODO: Add socket implementation

type Client interface {
	// CreateTopic makes sure a topic exists on the given nsqd.
	CreateTopic(topic string) error
	// Send sends a message on a topic to the specified nsqd.
	Send(topic string, body io.Reader) error
}

// HttpClient implements the Client interface using HTTP requests.
type HttpClient struct {
	*url.URL
}

func (h HttpClient) CreateTopic(topic string) error {
	resp, err := http.Get(h.String() + "/create_topic?topic=" + topic)
	if resp != nil && resp.Body != nil {
		resp.Body.Close()
	}
	return err
}

func (h HttpClient) Send(topic string, body io.Reader) error {
	reqURL := h.String() + "/put?topic=" + topic
	log.Printf("Sending %s message to: %s", topic, reqURL)
	resp, err := http.DefaultClient.Post(reqURL, "application/json", body)
	if resp != nil && resp.Body != nil {
		defer resp.Body.Close()
	}
	return err
}
<commit_msg>Implement tcp<commit_after>package knsq

import (
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"

	"github.com/bitly/nsq/nsq"
)

type Client interface {
	// CreateTopic makes sure a topic exists on the given nsqd.
	CreateTopic(topic string) error
	// Send sends a message on a topic to the specified nsqd.
	Send(topic string, body io.Reader) error
}

// HttpClient implements the Client interface using HTTP requests.
type HttpClient struct {
	*url.URL
}

func (h HttpClient) CreateTopic(topic string) error {
	resp, err := http.Get(h.String() + "/create_topic?topic=" + topic)
	if resp != nil && resp.Body != nil {
		resp.Body.Close()
	}
	return err
}

func (h HttpClient) Send(topic string, body io.Reader) error {
	reqURL := h.String() + "/put?topic=" + topic
	log.Printf("Sending %s message to: %s", topic, reqURL)
	resp, err := http.DefaultClient.Post(reqURL, "application/json", body)
	if resp != nil && resp.Body != nil {
		defer resp.Body.Close()
	}
	return err
}

// TODO: Does this thing need to be thread-safe?
type rwcClient struct {
	io.ReadWriteCloser
}

// NewTCPClient creates an implementation of the Client interface
// using the TCP API of NSQd.
func NewTCPClient(addr string) (Client, error) {
	if err != nil {
		return nil, err
	}

	_, err = c.Write(nsq.MagicV2)
	return &rwcClient{c}, err
}

func (c *rwcClient) CreateTopic(topic string) error {
	// FIXME: This is awful. NSQd has no command to create a topic
	// except via the HTTP API. Needs patching.
	cmd := nsq.Publish(topic, []byte("dummymsg"))
	return cmd.Write(c)
}

func (c *rwcClient) Send(topic string, body io.Reader) error {
	buf, err := ioutil.ReadAll(body)
	if err != nil {
		return err
	}
	cmd := nsq.Publish(topic, buf)
	return cmd.Write(c)
}
<|endoftext|>


<commit_before>package labyrinth

import "github.com/hajimehoshi/ebiten"

type wizard struct {
	image       *ebiten.Image
	Center      *coord
	moveSpeed   int
	fireballs   []*fireball
	fireCreator *fireballCreator
}

// newWizard returns an initialized wizard
func newWizard(i *ebiten.Image, speed int, fbc *fireballCreator) *wizard {
	c := &coord{
		x: 100,
		y: 100,
	}

	return &wizard{
		image:       i,
		Center:      c,
		moveSpeed:   speed,
		fireCreator: fbc,
	}
}

func (w *wizard) Update(keys *KeyboardWrapper) error {
	if keys.IsKeyPressed(ebiten.KeyW) {
		w.Center.y -= w.moveSpeed
		_, height := w.image.Size()
		if w.Center.y-(height/2) <= 0 {
			w.Center.y = (height / 2)
		}
	}

	if keys.IsKeyPressed(ebiten.KeyS) {
		w.Center.y += w.moveSpeed
		_, height := w.image.Size()
		if w.Center.y+(height/2) >= ScreenHeight {
			w.Center.y = ScreenHeight - (height / 2)
		}
	}

	if keys.IsKeyPressed(ebiten.KeySpace) {
		if len(w.fireballs) == 0 {
			f := w.fireCreator.newFireball(*w.Center, normalFireball)
			w.fireballs = append(w.fireballs, f)
		}
	}

	activeFireballs := []*fireball{}
	for _, f := range w.fireballs {
		if err := f.Update(); err != nil {
			return err
		}
		if f.active {
			activeFireballs = append(activeFireballs, f)
		}
	}

	// Only keep active fireballs in the array
	w.fireballs = activeFireballs

	return nil
}

func (w *wizard) Draw(r *ebiten.Image) error {

	r.DrawImage(w.image, &ebiten.DrawImageOptions{
		ImageParts: w,
	})

	for _, f := range w.fireballs {
		if err := f.Draw(r); err != nil {
			return err
		}

	}

	return nil
}

func (w *wizard) Len() int {
	return 1
}

func (w *wizard) Dst(i int) (x0, y0, x1, y1 int) {
	width, height := w.image.Size()
	height = height / 2
	width = width / 2
	return w.Center.x - height,
		w.Center.y - width,
		w.Center.x + height,
		w.Center.y + width
}

func (w *wizard) Src(i int) (x0, y0, x1, y1 int) {
	width, height := w.image.Size()
	return 0, 0, width, height
}
<commit_msg>pull out wizard movement to functions<commit_after>package labyrinth

import "github.com/hajimehoshi/ebiten"

type wizard struct {
	image       *ebiten.Image
	Center      *coord
	moveSpeed   int
	fireballs   []*fireball
	fireCreator *fireballCreator
}

// newWizard returns an initialized wizard
func newWizard(i *ebiten.Image, speed int, fbc *fireballCreator) *wizard {
	c := &coord{
		x: 100,
		y: 100,
	}

	return &wizard{
		image:       i,
		Center:      c,
		moveSpeed:   speed,
		fireCreator: fbc,
	}
}

func (w *wizard) Update(keys *KeyboardWrapper) error {
	if keys.IsKeyPressed(ebiten.KeyW) {
		w.moveUp()
	}

	if keys.IsKeyPressed(ebiten.KeyS) {
		w.moveDown()
	}

	if keys.IsKeyPressed(ebiten.KeySpace) {
		if len(w.fireballs) == 0 {
			f := w.fireCreator.newFireball(*w.Center, normalFireball)
			w.fireballs = append(w.fireballs, f)
		}
	}

	activeFireballs := []*fireball{}
	for _, f := range w.fireballs {
		if err := f.Update(); err != nil {
			return err
		}
		if f.active {
			activeFireballs = append(activeFireballs, f)
		}
	}

	// Only keep active fireballs in the array
	w.fireballs = activeFireballs

	return nil
}

func (w *wizard) moveUp() {
	w.Center.y -= w.moveSpeed
	_, height := w.image.Size()
	if w.Center.y-(height/2) <= 0 {
		w.Center.y = (height / 2)
	}
}

func (w *wizard) moveDown() {
	w.Center.y += w.moveSpeed
	_, height := w.image.Size()
	if w.Center.y+(height/2) >= ScreenHeight {
		w.Center.y = ScreenHeight - (height / 2)
	}
}

func (w *wizard) Draw(r *ebiten.Image) error {
	r.DrawImage(w.image, &ebiten.DrawImageOptions{
		ImageParts: w,
	})

	for _, f := range w.fireballs {
		if err := f.Draw(r); err != nil {
			return err
		}
	}

	return nil
}

func (w *wizard) Len() int {
	return 1
}

func (w *wizard) Dst(i int) (x0, y0, x1, y1 int) {
	width, height := w.image.Size()
	height = height / 2
	width = width / 2
	return w.Center.x - height,
		w.Center.y - width,
		w.Center.x + height,
		w.Center.y + width
}

func (w *wizard) Src(i int) (x0, y0, x1, y1 int) {
	width, height := w.image.Size()
	return 0, 0, width, height
}
<|endoftext|>


<commit_before>package graph

import (
	"sort"
)

// A GonumGraph is a very generalized graph that can handle an arbitrary number of vertices and edges -- as well as act as either directed or undirected.
//
// Internally, it uses a map of successors AND predecessors, to speed up some operations (such as getting all successors/predecessors). It also speeds up thing like adding edges (assuming both edges exist).
//
// However, its generality is also its weakness (and partially a flaw in needing to satisfy MutableGraph). For most purposes, creating your own graph is probably better. For instance, see discrete.TileGraph for an example
// of an immutable 2D grid of tiles that also implements the Graph interface, but would be more suitable if all you needed was a simple undirected 2D grid.
type GonumGraph struct {
	successors   map[int]map[int]float64
	predecessors map[int]map[int]float64
	directed     bool
}

func NewGonumGraph(directed bool) *GonumGraph {
	return &GonumGraph{
		successors:   make(map[int]map[int]float64),
		predecessors: make(map[int]map[int]float64),
		directed:     directed,
	}
}

func NewPreAllocatedGonumGraph(directed bool, numVertices int) *GonumGraph {
	return &GonumGraph{
		successors:   make(map[int]map[int]float64, numVertices),
		predecessors: make(map[int]map[int]float64, numVertices),
		directed:     directed,
	}
}

/* Mutable Graph implementation */

func (graph *GonumGraph) NewNode(successors []int) (id int) {
	nodes := sort.IntSlice(graph.NodeList())
	sort.Sort(&nodes)
	for i, node := range nodes {
		if i != node {
			graph.AddNode(i, successors)
			return i
		}
	}

	newID := len(nodes)
	graph.AddNode(newID, successors)
	return newID
}

func (graph *GonumGraph) AddNode(id int, successors []int) {
	if _, ok := graph.successors[id]; ok {
		return
	}

	graph.successors[id] = make(map[int]float64, len(successors))
	if !graph.directed {
		graph.predecessors[id] = make(map[int]float64, len(successors))
	} else {
		graph.predecessors[id] = make(map[int]float64)
	}
	for _, succ := range successors {
		graph.successors[id][succ] = 1.0

		// Always add the reciprocal node to the graph
		if _, ok := graph.successors[succ]; !ok {
			graph.predecessors[succ] = make(map[int]float64)
			graph.successors[succ] = make(map[int]float64)
		}

		graph.predecessors[succ][id] = 1.0

		// But only add the reciprocal edge if we're undirected
		if !graph.directed {
			graph.successors[succ][id] = 1.0
			graph.predecessors[id][succ] = 1.0
		}
	}
}

func (graph *GonumGraph) AddEdge(id, successor int) {
	if _, ok := graph.successors[id]; !ok {
		return
	}

	if _, ok := graph.successors[successor]; !ok {
		graph.successors[successor] = make(map[int]float64)
		graph.predecessors[successor] = make(map[int]float64)
	}

	graph.successors[id][successor] = 1.0
	graph.predecessors[successor][id] = 1.0

	if !graph.directed {
		graph.successors[successor][id] = 1.0
		graph.predecessors[id][successor] = 1.0
	}
}

func (graph *GonumGraph) SetEdgeCost(id, successor int, cost float64) {
	// Normally I'd use graph.vertices.Contains(id) as above, but this is equivalent and a bit easier to read here
	if _, ok := graph.successors[id]; !ok {
		return
	} else if _, ok := graph.successors[id][successor]; !ok {
		return
	}
	graph.successors[id][successor] = cost
	graph.predecessors[successor][id] = cost

	// By the spec, only the empty graph will be toggled between directed and undirected. Therefore we can be sure the reciprocal edge exists
	if !graph.directed {
		graph.successors[successor][id] = cost
		graph.predecessors[id][successor] = cost
	}
}

func (graph *GonumGraph) RemoveNode(id int) {
	if _, ok := graph.successors[id]; ok {
		return
	}

	for succ, _ := range graph.successors[id] {
		delete(graph.predecessors[succ], id)
	}
	delete(graph.successors, id)

	for pred, _ := range graph.predecessors[id] {
		delete(graph.successors[pred], id)
	}
	delete(graph.predecessors, id)

}

func (graph *GonumGraph) RemoveEdge(id, succ int) {
	if _, ok := graph.successors[id]; !ok {
		return
	} else if _, ok := graph.successors[succ]; !ok {
		return
	}

	delete(graph.successors[id], succ)
	delete(graph.predecessors[succ], id)
	if !graph.directed {
		delete(graph.predecessors[id], succ)
		delete(graph.successors[succ], id)
	}
}

func (graph *GonumGraph) EmptyGraph() {
	if len(graph.successors) == 0 {
		return
	}
	graph.successors = make(map[int]map[int]float64)
	graph.predecessors = make(map[int]map[int]float64)
}

func (graph *GonumGraph) SetDirected(directed bool) {
	if len(graph.successors) > 0 {
		return
	}
	graph.directed = directed
}

/* Graph implementation */

func (graph *GonumGraph) Successors(id int) []int {
	if _, ok := graph.successors[id]; !ok {
		return nil
	}

	successors := make([]int, len(graph.successors[id]))
	for succ, _ := range graph.successors[id] {
		successors = append(successors, succ)
	}

	return successors
}

func (graph *GonumGraph) IsSuccessor(id, succ int) bool {
	if _, ok := graph.successors[id]; !ok {
		return false
	}

	_, ok := graph.successors[id][succ]

	return ok
}

func (graph *GonumGraph) Predecessors(id int) []int {
	if _, ok := graph.successors[id]; !ok {
		return nil
	}

	predecessors := make([]int, len(graph.predecessors[id]))
	for pred, _ := range graph.predecessors[id] {
		predecessors = append(predecessors, pred)
	}

	return predecessors
}

func (graph *GonumGraph) IsPredecessor(id, pred int) bool {
	if _, ok := graph.successors[id]; !ok {
		return false
	}

	_, ok := graph.predecessors[id][pred]

	return ok
}

func (graph *GonumGraph) IsAdjacent(id, neighbor int) bool {
	if _, ok := graph.successors[id]; !ok {
		return false
	}

	_, succ := graph.predecessors[id][neighbor]
	_, pred := graph.predecessors[id][neighbor]

	return succ || pred
}

func (graph *GonumGraph) NodeExists(id int) bool {
	_, ok := graph.successors[id]

	return ok
}

func (graph *GonumGraph) Degree(id int) int {
	if _, ok := graph.successors[id]; !ok {
		return 0
	}

	return len(graph.successors[id]) + len(graph.predecessors[id])
}

func (graph *GonumGraph) EdgeList() [][2]int {
	eList := make([][2]int, 0, len(graph.successors))
	for id, succMap := range graph.successors {
		for succ, _ := range succMap {
			eList = append(eList, [2]int{id, succ})
		}
	}

	return eList
}

func (graph *GonumGraph) NodeList() []int {
	nodes := make([]int, 0, len(graph.successors))
	for node, _ := range graph.successors {
		nodes = append(nodes, node)
	}

	return nodes
}

func (graph *GonumGraph) IsDirected() bool {
	return graph.directed
}

func (graph *GonumGraph) Cost(id, succ int) float64 {
	return graph.successors[id][succ]
}
<commit_msg>Brought gonum graph in-line with first-class edges and nodes<commit_after>package graph

import (
	"sort"
)

// A GonumGraph is a very generalized graph that can handle an arbitrary number of vertices and edges -- as well as act as either directed or undirected.
//
// Internally, it uses a map of successors AND predecessors, to speed up some operations (such as getting all successors/predecessors). It also speeds up thing like adding edges (assuming both edges exist).
//
// However, its generality is also its weakness (and partially a flaw in needing to satisfy MutableGraph). For most purposes, creating your own graph is probably better. For instance, see discrete.TileGraph for an example
// of an immutable 2D grid of tiles that also implements the Graph interface, but would be more suitable if all you needed was a simple undirected 2D grid.
type GonumGraph struct {
	successors   map[int]map[int]float64
	predecessors map[int]map[int]float64
	nodeMap      map[int]Node
	directed     bool
}

func NewGonumGraph(directed bool) *GonumGraph {
	return &GonumGraph{
		successors:   make(map[int]map[int]float64),
		predecessors: make(map[int]map[int]float64),
		directed:     directed,
	}
}

func NewPreAllocatedGonumGraph(directed bool, numVertices int) *GonumGraph {
	return &GonumGraph{
		successors:   make(map[int]map[int]float64, numVertices),
		predecessors: make(map[int]map[int]float64, numVertices),
		directed:     directed,
	}
}

/* Mutable Graph implementation */

func (graph *GonumGraph) NewNode(successors []Node) (node Node) {
	nodeList := graph.NodeList()
	ids := make([]int, len(nodeList))
	for i, node := range nodeList {
		ids[i] = node.ID()
	}

	nodes := sort.IntSlice(ids)
	sort.Sort(&nodes)
	for i, node := range nodes {
		if i != node {
			graph.AddNode(GonumNode(i), successors)
			return GonumNode(i)
		}
	}

	newID := len(nodes)
	graph.AddNode(GonumNode(newID), successors)
	return GonumNode(newID)
}

func (graph *GonumGraph) AddNode(node Node, successors []Node) {
	id := node.ID()
	if _, ok := graph.successors[id]; ok {
		return
	}

	graph.nodeMap[id] = node

	graph.successors[id] = make(map[int]float64, len(successors))
	if !graph.directed {
		graph.predecessors[id] = make(map[int]float64, len(successors))
	} else {
		graph.predecessors[id] = make(map[int]float64)
	}
	for _, successor := range successors {
		succ := successor.ID()
		graph.successors[id][succ] = 1.0

		// Always add the reciprocal node to the graph
		if _, ok := graph.successors[succ]; !ok {
			graph.nodeMap[succ] = successor
			graph.predecessors[succ] = make(map[int]float64)
			graph.successors[succ] = make(map[int]float64)
		}

		graph.predecessors[succ][id] = 1.0

		// But only add the reciprocal edge if we're undirected
		if !graph.directed {
			graph.successors[succ][id] = 1.0
			graph.predecessors[id][succ] = 1.0
		}
	}
}

func (graph *GonumGraph) AddEdge(e Edge) {
	id := e.Head().ID()
	successor := e.Tail().ID()
	if _, ok := graph.successors[id]; !ok {
		return
	}

	if _, ok := graph.successors[successor]; !ok {
		graph.nodeMap[successor] = e.Tail()
		graph.successors[successor] = make(map[int]float64)
		graph.predecessors[successor] = make(map[int]float64)
	}

	graph.successors[id][successor] = 1.0
	graph.predecessors[successor][id] = 1.0

	if !graph.directed {
		graph.successors[successor][id] = 1.0
		graph.predecessors[id][successor] = 1.0
	}
}

func (graph *GonumGraph) SetEdgeCost(e Edge, cost float64) {
	id := e.Head().ID()
	successor := e.Tail().ID()
	// Normally I'd use graph.vertices.Contains(id) as above, but this is equivalent and a bit easier to read here
	if _, ok := graph.successors[id]; !ok {
		return
	} else if _, ok := graph.successors[id][successor]; !ok {
		return
	}
	graph.successors[id][successor] = cost
	graph.predecessors[successor][id] = cost

	// By the spec, only the empty graph will be toggled between directed and undirected. Therefore we can be sure the reciprocal edge exists
	if !graph.directed {
		graph.successors[successor][id] = cost
		graph.predecessors[id][successor] = cost
	}
}

func (graph *GonumGraph) RemoveNode(node Node) {
	id := node.ID()
	if _, ok := graph.successors[id]; ok {
		return
	}
	delete(graph.nodeMap, id)

	for succ, _ := range graph.successors[id] {
		delete(graph.predecessors[succ], id)
	}
	delete(graph.successors, id)

	for pred, _ := range graph.predecessors[id] {
		delete(graph.successors[pred], id)
	}
	delete(graph.predecessors, id)

}

func (graph *GonumGraph) RemoveEdge(e Edge) {
	id := e.Head().ID()
	succ := e.Tail().ID()
	if _, ok := graph.successors[id]; !ok {
		return
	} else if _, ok := graph.successors[succ]; !ok {
		return
	}

	delete(graph.successors[id], succ)
	delete(graph.predecessors[succ], id)
	if !graph.directed {
		delete(graph.predecessors[id], succ)
		delete(graph.successors[succ], id)
	}
}

func (graph *GonumGraph) EmptyGraph() {
	if len(graph.successors) == 0 {
		return
	}
	graph.successors = make(map[int]map[int]float64)
	graph.predecessors = make(map[int]map[int]float64)
	graph.nodeMap = make(map[int]Node)
}

func (graph *GonumGraph) SetDirected(directed bool) {
	if len(graph.successors) > 0 {
		return
	}
	graph.directed = directed
}

/* Graph implementation */

func (graph *GonumGraph) Successors(node Node) []Node {
	id := node.ID()
	if _, ok := graph.successors[id]; !ok {
		return nil
	}

	successors := make([]Node, len(graph.successors[id]))
	for succ, _ := range graph.successors[id] {
		successors = append(successors, graph.nodeMap[succ])
	}

	return successors
}

func (graph *GonumGraph) IsSuccessor(node, successor Node) bool {
	succ := successor.ID()
	id := node.ID()
	if _, ok := graph.successors[id]; !ok {
		return false
	}

	_, ok := graph.successors[id][succ]

	return ok
}

func (graph *GonumGraph) Predecessors(node Node) []Node {
	id := node.ID()
	if _, ok := graph.successors[id]; !ok {
		return nil
	}

	predecessors := make([]Node, len(graph.predecessors[id]))
	for pred, _ := range graph.predecessors[id] {
		predecessors = append(predecessors, graph.nodeMap[pred])
	}

	return predecessors
}

func (graph *GonumGraph) IsPredecessor(node, predecessor Node) bool {
	id := node.ID()
	pred := predecessor.ID()
	if _, ok := graph.successors[id]; !ok {
		return false
	}

	_, ok := graph.predecessors[id][pred]

	return ok
}

func (graph *GonumGraph) IsAdjacent(node, neigh Node) bool {
	id := node.ID()
	neighbor := neigh.ID()
	if _, ok := graph.successors[id]; !ok {
		return false
	}

	_, succ := graph.predecessors[id][neighbor]
	_, pred := graph.predecessors[id][neighbor]

	return succ || pred
}

func (graph *GonumGraph) NodeExists(node Node) bool {
	_, ok := graph.successors[node.ID()]

	return ok
}

func (graph *GonumGraph) Degree(node Node) int {
	id := node.ID()
	if _, ok := graph.successors[id]; !ok {
		return 0
	}

	return len(graph.successors[id]) + len(graph.predecessors[id])
}

func (graph *GonumGraph) EdgeList() []Edge {
	eList := make([]Edge, 0, len(graph.successors))
	for id, succMap := range graph.successors {
		for succ, _ := range succMap {
			eList = append(eList, GonumEdge{graph.nodeMap[id], graph.nodeMap[succ]})
		}
	}

	return eList
}

func (graph *GonumGraph) NodeList() []Node {
	nodes := make([]Node, 0, len(graph.successors))
	for _, node := range graph.nodeMap {
		nodes = append(nodes, node)
	}

	return nodes
}

func (graph *GonumGraph) IsDirected() bool {
	return graph.directed
}

func (graph *GonumGraph) Cost(node, succ Node) float64 {
	return graph.successors[node.ID()][succ.ID()]
}
<|endoftext|>


<commit_before>package orm

import (
	"reflect"
	"testing"
)

func TestSave(t *testing.T) {
	//connect to Postgres
	orm, scream := ConnectToPostgres()
	if scream != nil {
		panic(scream)
	}

	ormTest, err := orm.NewHandler(DSLTest{})
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	//DropTable & CreateTable
	ormTest.DropTable()
	ormTest.CreateTable()

	//Check if we get an error when we try to insert a diferent object for this handler
	dslTestWithoutID := DSLTestWithoutID{FieldString: "teststring", FieldInt: 123}
	err = ormTest.Save(&dslTestWithoutID)
	if err.Error() != "Object table name (DSLTestWithoutID) is diferent from handler table name (DSLTest)" {
		t.Fatalf("Want: `Object table name (DSLTestWithoutID) is diferent from handler table name (DSLTest)`, got: `%v`", err)
	}

	//save a new object
	dslTest := DSLTest{FieldString: "teststring", FieldInt: 123}
	err = ormTest.Save(&dslTest)
	if err != nil {
		t.Fatalf("Err: %v", err)
	}

	//check if the id is 1
	id := dslTest.ID
	if id != 1 {
		t.Fatalf("want: 1; got: %v", id)
	}

	//check if the object was persisted
	dslTestFind, err := ormTest.Select().ByID(id)
	if err != nil {
		t.Fatalf("Err: %v", err)
	}
	obj := dslTestFind.(DSLTest)

	if dslTestFind == nil {
		t.Fatalf("want: a valida object, got nil")
	}

	//update the FieldInt atribute
	obj.FieldInt = 222
	err = ormTest.Save(&obj)
	if err != nil {
		t.Fatalf("Err: %v", err)
	}

	//check if the changes were persisted
	dslTestFindUptated, err := ormTest.Select().ByID(id)
	if err != nil {
		t.Fatalf("Err: %v", err)
	}
	obj = dslTestFindUptated.(DSLTest)

	if dslTestFindUptated == nil {
		t.Fatalf("want: a valida object, got nil")
	}

	if obj.FieldInt != 222 {
		t.Fatalf("want: 222, got: %v", obj.FieldInt)
	}

	//check if we got an error when trying to insert an object with negative ID
	dslTestNegativeID := DSLTest{FieldString: "teststring", FieldInt: 123, ID: -1}
	err = ormTest.Save(&dslTestNegativeID)
	if err.Error() != "Negative ID not allowed: {-1 teststring 123}" {
		t.Fatalf("want: `Negative ID not allowed: {-1 teststring 123}`, got `%v`", err)
	}
}

func TestInsertAndUpdate(t *testing.T) {
	//connect to Postgres
	orm, scream := ConnectToPostgres()
	if scream != nil {
		panic(scream)
	}

	//create a new handler for DSLTest structure
	ormTest, err := orm.NewHandler(DSLTest{})
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	//DropTable & CreateTable
	ormTest.DropTable()
	ormTest.CreateTable()

	//create a test object
	dslTest := DSLTest{FieldString: "teststring", FieldInt: 123}
	object := reflect.ValueOf(&dslTest).Elem()

	//insert on db
	err = ormTest.insert(object)
	if err != nil {
		t.Fatalf("Err: %v", err)
	}

	if dslTest.ID != 1 {
		t.Fatalf("\ndslTest.ID got:\t %v\nWant:\t\t\t 1\n", dslTest.ID)
	}

	//check if the object was persisted
	dslTestFind, err := ormTest.Select().ByID(1)
	if err != nil {
		t.Fatalf("Err: %v", err)
	}
	obj := dslTestFind.(DSLTest)

	if dslTestFind == nil {
		t.Fatalf("want: a valida object, got nil")
	}

	if obj.FieldInt != 123 {
		t.Fatalf("want: 123, got: %v", obj.FieldInt)
	}

	//change FieldInt atribute
	obj.FieldInt = 222

	//update on db
	object = reflect.ValueOf(&obj).Elem()
	err = ormTest.update(object)
	if err != nil {
		t.Fatalf("Err: %v", err)
	}

	//check if the object was correctly updated
	dslTestFind, err = ormTest.Select().ByID(1)
	if err != nil {
		t.Fatalf("Err: %v", err)
	}
	obj = dslTestFind.(DSLTest)

	if dslTestFind == nil {
		t.Fatalf("want: a valida object, got nil")
	}

	if obj.FieldInt != 222 {
		t.Fatalf("want: 222, got: %v", obj.FieldInt)
	}

	//force a error on insert
	oldInsertSQL := ormTest.insertSQL
	ormTest.insertSQL = "wrong-sql"
	err = ormTest.insert(object)
	if err.Error() != "pq: syntax error at or near \"wrong\"" {
		t.Fatalf("want: `pq: syntax error at or near \"wrong\"`, got: `%v`", err)
	}
	ormTest.insertSQL = oldInsertSQL

	//force a error on insert
	oldUpdateSQL := ormTest.updateSQL
	ormTest.updateSQL = "wrong-sql"
	err = ormTest.update(object)
	if err.Error() != "pq: syntax error at or near \"wrong\"" {
		t.Fatalf("want: `pq: syntax error at or near \"wrong\"`, got: `%v`", err)
	}
	ormTest.updateSQL = oldUpdateSQL
}

func TestAssembleValuesArray(t *testing.T) {

}

func TestAssembleSQLInsertStatement(t *testing.T) {
	var handler Handler
	handler.table = DSLTest{}

	//check if the sqlInsert string is assembled correctly
	handler.assembleSQLInsert()
	expected := `insert into DSLTest(FieldString, FieldInt) values ($1, $2) RETURNING id;`
	got := handler.insertSQL
	if got != expected {
		t.Fatalf("\nExpected:\t %v\nGot:\t\t %v\n", expected, got)
	}

	//expectedMap := []saveField{}
	mapInsert := handler.insertMap
	if len(mapInsert) != 2 {
		t.Fatalf("\nmapInsert lengh:\t %v\nWant:\t\t\t 2\n", len(mapInsert))
	}

	if mapInsert[0].name != "FieldString" || mapInsert[0].fieldType != "string" {
		t.Fatalf("\nmapInsert[0] got:\t %v\nWant:\t\t\t {FieldString string}\n", mapInsert[0])
	}

	if mapInsert[1].name != "FieldInt" || mapInsert[1].fieldType != "int" {
		t.Fatalf("\nmapInsert[0] got:\t %v\nWant:\t\t\t {FieldInt int}\n", mapInsert[1])
	}
}

func TestAssembleSQLUpdate(t *testing.T) {
	var handler Handler
	handler.table = DSLTest{}

	handler.assembleSQLUpdate()
	expected := `update DSLTest set FieldString = $1, FieldInt = $2 where id = $3`
	got := handler.updateSQL
	if got != expected {
		t.Fatalf("\nExpected:\t %v\nGot:\t\t %v\n", expected, got)
	}

	updateMap := handler.updateMap
	if len(updateMap) != 3 {
		t.Fatalf("updateMap lenght: %v, want: 3", len(updateMap))
	}

	if updateMap[0].name != "FieldString" || updateMap[0].fieldType != "string" {
		t.Fatalf("updateMap[0] got:\t %v\nWant:\t\t\t {FieldString string}\n", updateMap[0])
	}

	if updateMap[1].name != "FieldInt" || updateMap[1].fieldType != "int" {
		t.Fatalf("updateMap[1] got:\t %v\nWant:\t\t\t {FieldInt int}\n", updateMap[1])
	}

	if updateMap[2].name != "ID" || updateMap[2].fieldType != "int" {
		t.Fatalf("updateMap[2] got: %v, want: {ID int}\n", updateMap[2])
	}
}
<commit_msg>Add unit test for AssembleValuesArray<commit_after>package orm

import (
	"reflect"
	"testing"
)

func TestSave(t *testing.T) {
	//connect to Postgres
	orm, scream := ConnectToPostgres()
	if scream != nil {
		panic(scream)
	}

	ormTest, err := orm.NewHandler(DSLTest{})
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	//DropTable & CreateTable
	ormTest.DropTable()
	ormTest.CreateTable()

	//Check if we get an error when we try to insert a diferent object for this handler
	dslTestWithoutID := DSLTestWithoutID{FieldString: "teststring", FieldInt: 123}
	err = ormTest.Save(&dslTestWithoutID)
	if err.Error() != "Object table name (DSLTestWithoutID) is diferent from handler table name (DSLTest)" {
		t.Fatalf("Want: `Object table name (DSLTestWithoutID) is diferent from handler table name (DSLTest)`, got: `%v`", err)
	}

	//save a new object
	dslTest := DSLTest{FieldString: "teststring", FieldInt: 123}
	err = ormTest.Save(&dslTest)
	if err != nil {
		t.Fatalf("Err: %v", err)
	}

	//check if the id is 1
	id := dslTest.ID
	if id != 1 {
		t.Fatalf("want: 1; got: %v", id)
	}

	//check if the object was persisted
	dslTestFind, err := ormTest.Select().ByID(id)
	if err != nil {
		t.Fatalf("Err: %v", err)
	}
	obj := dslTestFind.(DSLTest)

	if dslTestFind == nil {
		t.Fatalf("want: a valida object, got nil")
	}

	//update the FieldInt atribute
	obj.FieldInt = 222
	err = ormTest.Save(&obj)
	if err != nil {
		t.Fatalf("Err: %v", err)
	}

	//check if the changes were persisted
	dslTestFindUptated, err := ormTest.Select().ByID(id)
	if err != nil {
		t.Fatalf("Err: %v", err)
	}
	obj = dslTestFindUptated.(DSLTest)

	if dslTestFindUptated == nil {
		t.Fatalf("want: a valida object, got nil")
	}

	if obj.FieldInt != 222 {
		t.Fatalf("want: 222, got: %v", obj.FieldInt)
	}

	//check if we got an error when trying to insert an object with negative ID
	dslTestNegativeID := DSLTest{FieldString: "teststring", FieldInt: 123, ID: -1}
	err = ormTest.Save(&dslTestNegativeID)
	if err.Error() != "Negative ID not allowed: {-1 teststring 123}" {
		t.Fatalf("want: `Negative ID not allowed: {-1 teststring 123}`, got `%v`", err)
	}
}

func TestInsertAndUpdate(t *testing.T) {
	//connect to Postgres
	orm, scream := ConnectToPostgres()
	if scream != nil {
		panic(scream)
	}

	//create a new handler for DSLTest structure
	ormTest, err := orm.NewHandler(DSLTest{})
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	//DropTable & CreateTable
	ormTest.DropTable()
	ormTest.CreateTable()

	//create a test object
	dslTest := DSLTest{FieldString: "teststring", FieldInt: 123}
	object := reflect.ValueOf(&dslTest).Elem()

	//insert on db
	err = ormTest.insert(object)
	if err != nil {
		t.Fatalf("Err: %v", err)
	}

	if dslTest.ID != 1 {
		t.Fatalf("\ndslTest.ID got:\t %v\nWant:\t\t\t 1\n", dslTest.ID)
	}

	//check if the object was persisted
	dslTestFind, err := ormTest.Select().ByID(1)
	if err != nil {
		t.Fatalf("Err: %v", err)
	}
	obj := dslTestFind.(DSLTest)

	if dslTestFind == nil {
		t.Fatalf("want: a valida object, got nil")
	}

	if obj.FieldInt != 123 {
		t.Fatalf("want: 123, got: %v", obj.FieldInt)
	}

	//change FieldInt atribute
	obj.FieldInt = 222

	//update on db
	object = reflect.ValueOf(&obj).Elem()
	err = ormTest.update(object)
	if err != nil {
		t.Fatalf("Err: %v", err)
	}

	//check if the object was correctly updated
	dslTestFind, err = ormTest.Select().ByID(1)
	if err != nil {
		t.Fatalf("Err: %v", err)
	}
	obj = dslTestFind.(DSLTest)

	if dslTestFind == nil {
		t.Fatalf("want: a valida object, got nil")
	}

	if obj.FieldInt != 222 {
		t.Fatalf("want: 222, got: %v", obj.FieldInt)
	}

	//force a error on insert
	oldInsertSQL := ormTest.insertSQL
	ormTest.insertSQL = "wrong-sql"
	err = ormTest.insert(object)
	if err.Error() != "pq: syntax error at or near \"wrong\"" {
		t.Fatalf("want: `pq: syntax error at or near \"wrong\"`, got: `%v`", err)
	}
	ormTest.insertSQL = oldInsertSQL

	//force a error on insert
	oldUpdateSQL := ormTest.updateSQL
	ormTest.updateSQL = "wrong-sql"
	err = ormTest.update(object)
	if err.Error() != "pq: syntax error at or near \"wrong\"" {
		t.Fatalf("want: `pq: syntax error at or near \"wrong\"`, got: `%v`", err)
	}
	ormTest.updateSQL = oldUpdateSQL
}

func TestAssembleValuesArray(t *testing.T) {
	//connect to Postgres
	orm, scream := ConnectToPostgres()
	if scream != nil {
		panic(scream)
	}
	//create a new handler for DSLTest structure
	ormTest, err := orm.NewHandler(DSLTest{})
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	argurmentsMap := []saveField{{name: "FieldString", fieldType: "string"}, {name: "FieldInt", fieldType: "int"}, {name: "ID", fieldType: "int"}}
	objectPtr := &DSLTest{ID: 1, FieldString: "teststring", FieldInt: 123}
	object := reflect.ValueOf(objectPtr).Elem()

	res := ormTest.assembleValuesArray(argurmentsMap, object)
	if len(res) != 3 {
		t.Fatalf("want: 3, got: %v", len(res))
	}

	switch v := res[0].(type) {
	default:
		t.Fatalf("want: string, got: %v", v)
	case string:
		if res[0].(string) != "teststring" {
			t.Fatalf("want: teststring, got: %v", res[0].(string))
		}
	}

	switch v := res[1].(type) {
	default:
		t.Fatalf("want: int, got: %v", v)
	case int:
		if res[1].(int) != 123 {
			t.Fatalf("want: teststring, got: %v", res[1].(int))
		}
	}

	switch v := res[2].(type) {
	default:
		t.Fatalf("want: int, got: %v", v)
	case int:
		if res[2].(int) != 1 {
			t.Fatalf("want: teststring, got: %v", res[2].(int))
		}
	}
}

func TestAssembleSQLInsertStatement(t *testing.T) {
	var handler Handler
	handler.table = DSLTest{}

	//check if the sqlInsert string is assembled correctly
	handler.assembleSQLInsert()
	expected := `insert into DSLTest(FieldString, FieldInt) values ($1, $2) RETURNING id;`
	got := handler.insertSQL
	if got != expected {
		t.Fatalf("\nExpected:\t %v\nGot:\t\t %v\n", expected, got)
	}

	//expectedMap := []saveField{}
	mapInsert := handler.insertMap
	if len(mapInsert) != 2 {
		t.Fatalf("\nmapInsert lengh:\t %v\nWant:\t\t\t 2\n", len(mapInsert))
	}

	if mapInsert[0].name != "FieldString" || mapInsert[0].fieldType != "string" {
		t.Fatalf("\nmapInsert[0] got:\t %v\nWant:\t\t\t {FieldString string}\n", mapInsert[0])
	}

	if mapInsert[1].name != "FieldInt" || mapInsert[1].fieldType != "int" {
		t.Fatalf("\nmapInsert[0] got:\t %v\nWant:\t\t\t {FieldInt int}\n", mapInsert[1])
	}
}

func TestAssembleSQLUpdate(t *testing.T) {
	var handler Handler
	handler.table = DSLTest{}

	handler.assembleSQLUpdate()
	expected := `update DSLTest set FieldString = $1, FieldInt = $2 where id = $3`
	got := handler.updateSQL
	if got != expected {
		t.Fatalf("\nExpected:\t %v\nGot:\t\t %v\n", expected, got)
	}

	updateMap := handler.updateMap
	if len(updateMap) != 3 {
		t.Fatalf("updateMap lenght: %v, want: 3", len(updateMap))
	}

	if updateMap[0].name != "FieldString" || updateMap[0].fieldType != "string" {
		t.Fatalf("updateMap[0] got:\t %v\nWant:\t\t\t {FieldString string}\n", updateMap[0])
	}

	if updateMap[1].name != "FieldInt" || updateMap[1].fieldType != "int" {
		t.Fatalf("updateMap[1] got:\t %v\nWant:\t\t\t {FieldInt int}\n", updateMap[1])
	}

	if updateMap[2].name != "ID" || updateMap[2].fieldType != "int" {
		t.Fatalf("updateMap[2] got: %v, want: {ID int}\n", updateMap[2])
	}
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/git-lfs/git-lfs/errors"
	"github.com/git-lfs/git-lfs/git/githistory"
	"github.com/git-lfs/git-lfs/tasklog"
	"github.com/git-lfs/git-lfs/tools"
	"github.com/git-lfs/git-lfs/tools/humanize"
	"github.com/git-lfs/gitobj/v2"
	"github.com/spf13/cobra"
)

var (
	// migrateInfoTopN is a flag given to the git-lfs-migrate(1) subcommand
	// 'info' which specifies how many info entries to show by default.
	migrateInfoTopN int

	// migrateInfoAboveFmt is a flag given to the git-lfs-migrate(1)
	// subcommand 'info' specifying a human-readable string threshold of
	// filesize before entries are counted.
	migrateInfoAboveFmt string
	// migrateInfoAbove is the number of bytes parsed from the above
	// migrateInfoAboveFmt flag.
	migrateInfoAbove uint64

	// migrateInfoUnitFmt is a flag given to the git-lfs-migrate(1)
	// subcommand 'info' specifying a human-readable string of units with
	// which to display the number of bytes.
	migrateInfoUnitFmt string
	// migrateInfoUnit is the number of bytes in the unit given as
	// migrateInfoUnitFmt.
	migrateInfoUnit uint64
)

func migrateInfoCommand(cmd *cobra.Command, args []string) {
	l := tasklog.NewLogger(os.Stderr,
		tasklog.ForceProgress(cfg.ForceProgress()),
	)

	db, err := getObjectDatabase()
	if err != nil {
		ExitWithError(err)
	}
	defer db.Close()

	rewriter := getHistoryRewriter(cmd, db, l)

	exts := make(map[string]*MigrateInfoEntry)

	above, err := humanize.ParseBytes(migrateInfoAboveFmt)
	if err != nil {
		ExitWithError(errors.Wrap(err, "cannot parse --above=<n>"))
	}

	if u := cmd.Flag("unit"); u.Changed {
		unit, err := humanize.ParseByteUnit(u.Value.String())
		if err != nil {
			ExitWithError(errors.Wrap(err, "cannot parse --unit=<unit>"))
		}

		migrateInfoUnit = unit
	}

	migrateInfoAbove = above

	migrate(args, rewriter, l, &githistory.RewriteOptions{
		BlobFn: func(path string, b *gitobj.Blob) (*gitobj.Blob, error) {
			ext := fmt.Sprintf("*%s", filepath.Ext(path))

			// If extension exists, group all items under extension,
			// else just use the file name.
			var groupName string
			if len(ext) > 1 {
				groupName = ext
			} else {
				groupName = filepath.Base(path)
			}

			entry := exts[groupName]
			if entry == nil {
				entry = &MigrateInfoEntry{Qualifier: groupName}
			}

			entry.Total++

			if b.Size > int64(migrateInfoAbove) {
				entry.TotalAbove++
				entry.BytesAbove += b.Size
			}

			exts[groupName] = entry

			return b, nil
		},
	})
	l.Close()

	entries := EntriesBySize(MapToEntries(exts))
	entries = removeEmptyEntries(entries)
	sort.Sort(sort.Reverse(entries))

	migrateInfoTopN = tools.ClampInt(migrateInfoTopN, 0, len(entries))

	entries = entries[:migrateInfoTopN]

	entries.Print(os.Stdout)
}

// MigrateInfoEntry represents a tuple of filetype to bytes and entry count
// above and below a threshold.
type MigrateInfoEntry struct {
	// Qualifier is the filepath's extension.
	Qualifier string

	// BytesAbove is total size of all files above a given threshold.
	BytesAbove int64
	// TotalAbove is the count of all files above a given size threshold.
	TotalAbove int64
	// Total is the count of all files.
	Total int64
}

// MapToEntries creates a set of `*MigrateInfoEntry`'s for a given map of
// filepath extensions to file size in bytes.
func MapToEntries(exts map[string]*MigrateInfoEntry) []*MigrateInfoEntry {
	entries := make([]*MigrateInfoEntry, 0, len(exts))
	for _, entry := range exts {
		entries = append(entries, entry)
	}

	return entries
}

// removeEmptyEntries removes `*MigrateInfoEntry`'s for which no matching file
// is above the given threshold "--above".
func removeEmptyEntries(entries []*MigrateInfoEntry) []*MigrateInfoEntry {
	nz := make([]*MigrateInfoEntry, 0, len(entries))
	for _, e := range entries {
		if e.TotalAbove > 0 {
			nz = append(nz, e)
		}
	}

	return nz
}

// EntriesBySize is an implementation of sort.Interface that sorts a set of
// `*MigrateInfoEntry`'s
type EntriesBySize []*MigrateInfoEntry

// Len returns the total length of the set of `*MigrateInfoEntry`'s.
func (e EntriesBySize) Len() int { return len(e) }

// Less returns the whether or not the MigrateInfoEntry given at `i` takes up
// less total size than the MigrateInfoEntry given at `j`.
func (e EntriesBySize) Less(i, j int) bool { return e[i].BytesAbove < e[j].BytesAbove }

// Swap swaps the entries given at i, j.
func (e EntriesBySize) Swap(i, j int) { e[i], e[j] = e[j], e[i] }

// Print formats the `*MigrateInfoEntry`'s in the set and prints them to the
// given io.Writer, "to", returning "n" the number of bytes written, and any
// error, if one occurred.
func (e EntriesBySize) Print(to io.Writer) (int, error) {
	if len(e) == 0 {
		return 0, nil
	}

	extensions := make([]string, 0, len(e))
	sizes := make([]string, 0, len(e))
	stats := make([]string, 0, len(e))
	percentages := make([]string, 0, len(e))

	for _, entry := range e {
		bytesAbove := uint64(entry.BytesAbove)
		above := entry.TotalAbove
		total := entry.Total
		percentAbove := 100 * (float64(above) / float64(total))

		var size string
		if migrateInfoUnit > 0 {
			size = humanize.FormatBytesUnit(bytesAbove, migrateInfoUnit)
		} else {
			size = humanize.FormatBytes(bytesAbove)
		}

		stat := fmt.Sprintf("%d/%d files(s)",
			above, total)

		percentage := fmt.Sprintf("%.0f%%", percentAbove)

		extensions = append(extensions, entry.Qualifier)
		sizes = append(sizes, size)
		stats = append(stats, stat)
		percentages = append(percentages, percentage)
	}

	extensions = tools.Ljust(extensions)
	sizes = tools.Ljust(sizes)
	stats = tools.Rjust(stats)
	percentages = tools.Rjust(percentages)

	output := make([]string, 0, len(e))
	for i := 0; i < len(e); i++ {
		extension := extensions[i]
		size := sizes[i]
		stat := stats[i]
		percentage := percentages[i]

		line := strings.Join([]string{extension, size, stat, percentage}, "\t")

		output = append(output, line)
	}

	return fmt.Fprintln(to, strings.Join(output, "\n"))
}
<commit_msg>commands: make migrate info sort deterministically<commit_after>package commands

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/git-lfs/git-lfs/errors"
	"github.com/git-lfs/git-lfs/git/githistory"
	"github.com/git-lfs/git-lfs/tasklog"
	"github.com/git-lfs/git-lfs/tools"
	"github.com/git-lfs/git-lfs/tools/humanize"
	"github.com/git-lfs/gitobj/v2"
	"github.com/spf13/cobra"
)

var (
	// migrateInfoTopN is a flag given to the git-lfs-migrate(1) subcommand
	// 'info' which specifies how many info entries to show by default.
	migrateInfoTopN int

	// migrateInfoAboveFmt is a flag given to the git-lfs-migrate(1)
	// subcommand 'info' specifying a human-readable string threshold of
	// filesize before entries are counted.
	migrateInfoAboveFmt string
	// migrateInfoAbove is the number of bytes parsed from the above
	// migrateInfoAboveFmt flag.
	migrateInfoAbove uint64

	// migrateInfoUnitFmt is a flag given to the git-lfs-migrate(1)
	// subcommand 'info' specifying a human-readable string of units with
	// which to display the number of bytes.
	migrateInfoUnitFmt string
	// migrateInfoUnit is the number of bytes in the unit given as
	// migrateInfoUnitFmt.
	migrateInfoUnit uint64
)

func migrateInfoCommand(cmd *cobra.Command, args []string) {
	l := tasklog.NewLogger(os.Stderr,
		tasklog.ForceProgress(cfg.ForceProgress()),
	)

	db, err := getObjectDatabase()
	if err != nil {
		ExitWithError(err)
	}
	defer db.Close()

	rewriter := getHistoryRewriter(cmd, db, l)

	exts := make(map[string]*MigrateInfoEntry)

	above, err := humanize.ParseBytes(migrateInfoAboveFmt)
	if err != nil {
		ExitWithError(errors.Wrap(err, "cannot parse --above=<n>"))
	}

	if u := cmd.Flag("unit"); u.Changed {
		unit, err := humanize.ParseByteUnit(u.Value.String())
		if err != nil {
			ExitWithError(errors.Wrap(err, "cannot parse --unit=<unit>"))
		}

		migrateInfoUnit = unit
	}

	migrateInfoAbove = above

	migrate(args, rewriter, l, &githistory.RewriteOptions{
		BlobFn: func(path string, b *gitobj.Blob) (*gitobj.Blob, error) {
			ext := fmt.Sprintf("*%s", filepath.Ext(path))

			// If extension exists, group all items under extension,
			// else just use the file name.
			var groupName string
			if len(ext) > 1 {
				groupName = ext
			} else {
				groupName = filepath.Base(path)
			}

			entry := exts[groupName]
			if entry == nil {
				entry = &MigrateInfoEntry{Qualifier: groupName}
			}

			entry.Total++

			if b.Size > int64(migrateInfoAbove) {
				entry.TotalAbove++
				entry.BytesAbove += b.Size
			}

			exts[groupName] = entry

			return b, nil
		},
	})
	l.Close()

	entries := EntriesBySize(MapToEntries(exts))
	entries = removeEmptyEntries(entries)
	sort.Sort(sort.Reverse(entries))

	migrateInfoTopN = tools.ClampInt(migrateInfoTopN, 0, len(entries))

	entries = entries[:migrateInfoTopN]

	entries.Print(os.Stdout)
}

// MigrateInfoEntry represents a tuple of filetype to bytes and entry count
// above and below a threshold.
type MigrateInfoEntry struct {
	// Qualifier is the filepath's extension.
	Qualifier string

	// BytesAbove is total size of all files above a given threshold.
	BytesAbove int64
	// TotalAbove is the count of all files above a given size threshold.
	TotalAbove int64
	// Total is the count of all files.
	Total int64
}

// MapToEntries creates a set of `*MigrateInfoEntry`'s for a given map of
// filepath extensions to file size in bytes.
func MapToEntries(exts map[string]*MigrateInfoEntry) []*MigrateInfoEntry {
	entries := make([]*MigrateInfoEntry, 0, len(exts))
	for _, entry := range exts {
		entries = append(entries, entry)
	}

	return entries
}

// removeEmptyEntries removes `*MigrateInfoEntry`'s for which no matching file
// is above the given threshold "--above".
func removeEmptyEntries(entries []*MigrateInfoEntry) []*MigrateInfoEntry {
	nz := make([]*MigrateInfoEntry, 0, len(entries))
	for _, e := range entries {
		if e.TotalAbove > 0 {
			nz = append(nz, e)
		}
	}

	return nz
}

// EntriesBySize is an implementation of sort.Interface that sorts a set of
// `*MigrateInfoEntry`'s
type EntriesBySize []*MigrateInfoEntry

// Len returns the total length of the set of `*MigrateInfoEntry`'s.
func (e EntriesBySize) Len() int { return len(e) }

// Less returns the whether or not the MigrateInfoEntry given at `i` takes up
// less total size than the MigrateInfoEntry given at `j`.
func (e EntriesBySize) Less(i, j int) bool {
	if e[i].BytesAbove == e[j].BytesAbove {
		return e[i].Qualifier > e[j].Qualifier
	} else {
		return e[i].BytesAbove < e[j].BytesAbove
	}
}

// Swap swaps the entries given at i, j.
func (e EntriesBySize) Swap(i, j int) { e[i], e[j] = e[j], e[i] }

// Print formats the `*MigrateInfoEntry`'s in the set and prints them to the
// given io.Writer, "to", returning "n" the number of bytes written, and any
// error, if one occurred.
func (e EntriesBySize) Print(to io.Writer) (int, error) {
	if len(e) == 0 {
		return 0, nil
	}

	extensions := make([]string, 0, len(e))
	sizes := make([]string, 0, len(e))
	stats := make([]string, 0, len(e))
	percentages := make([]string, 0, len(e))

	for _, entry := range e {
		bytesAbove := uint64(entry.BytesAbove)
		above := entry.TotalAbove
		total := entry.Total
		percentAbove := 100 * (float64(above) / float64(total))

		var size string
		if migrateInfoUnit > 0 {
			size = humanize.FormatBytesUnit(bytesAbove, migrateInfoUnit)
		} else {
			size = humanize.FormatBytes(bytesAbove)
		}

		stat := fmt.Sprintf("%d/%d files(s)",
			above, total)

		percentage := fmt.Sprintf("%.0f%%", percentAbove)

		extensions = append(extensions, entry.Qualifier)
		sizes = append(sizes, size)
		stats = append(stats, stat)
		percentages = append(percentages, percentage)
	}

	extensions = tools.Ljust(extensions)
	sizes = tools.Ljust(sizes)
	stats = tools.Rjust(stats)
	percentages = tools.Rjust(percentages)

	output := make([]string, 0, len(e))
	for i := 0; i < len(e); i++ {
		extension := extensions[i]
		size := sizes[i]
		stat := stats[i]
		percentage := percentages[i]

		line := strings.Join([]string{extension, size, stat, percentage}, "\t")

		output = append(output, line)
	}

	return fmt.Fprintln(to, strings.Join(output, "\n"))
}
<|endoftext|>


<commit_before>package builder

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"time"
)

func (b *Builder) WriteFeed(ctx *Context, r *Report) {
	baseUrl := "http://" + ctx.Meta.Domain
	var buf bytes.Buffer
	buf.WriteString(`<?xml version="1.0" encoding="UTF-8"?>`)
	buf.WriteString(`<rss version="2.0">`)
	buf.WriteString("<channel>")
	buf.WriteString(fmt.Sprintf("<title>%s</title>", ctx.Meta.Title+" - "+ctx.Meta.Subtitle))
	buf.WriteString(fmt.Sprintf("<link>%s</link>", baseUrl))
	buf.WriteString(fmt.Sprintf("<description>%s</description>", ctx.Meta.Desc))
	buf.WriteString(fmt.Sprintf("<lastBuildDate>%s</lastBuildDate>", time.Now().Format(time.RFC1123Z)))
	for _, p := range ctx.Posts {
		buf.WriteString("<item>")
		buf.WriteString(fmt.Sprintf("<title>%s</title>", p.Title))
		buf.WriteString(fmt.Sprintf("<link>%s</link>", baseUrl+p.Url))
		// buf.WriteString(fmt.Sprintf("<comments>%s</comments>", fi.Comments))
		buf.WriteString(fmt.Sprintf("<pubDate>%s</pubDate>", p.Created.Raw.Format(time.RFC1123Z)))
		for _, c := range p.Tags {
			buf.WriteString(fmt.Sprintf("<category>%s</category>", c.Name))
		}
		buf.WriteString(fmt.Sprintf("<description><![CDATA[ %s ]]></description>", p.ContentHTML()))
		buf.WriteString("</item>")
	}
	buf.WriteString("</channel>")
	buf.WriteString("</rss>")

	dstFile := path.Join(ctx.DstDir, "feed.xml")
	os.MkdirAll(path.Dir(dstFile), os.ModePerm)
	r.Error = ioutil.WriteFile(dstFile, buf.Bytes(), os.ModePerm)
	if r.Error != nil {
		return
	}

	// sitemap
	buf.Reset()
	buf.WriteString(`<?xml version="1.0" encoding="UTF-8"?>`)
	buf.WriteString(`<?xml-stylesheet type="text/xsl" href="/static/sitemap.xsl"?>`)
	buf.WriteString(`<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">`)
	buf.WriteString("<url>")
	buf.WriteString(fmt.Sprintf("<loc>%s</loc>", baseUrl))
	buf.WriteString(fmt.Sprintf("<lastmod>%s/</lastmod>", time.Now().Format(time.RFC3339)))
	buf.WriteString("<changefreq>daily</changefreq>")
	buf.WriteString("<priority>1.0</priority>")
	buf.WriteString("</url>")

	for _, p := range ctx.Posts {
		buf.WriteString("<url>")
		buf.WriteString(fmt.Sprintf("<loc>%s</loc>", baseUrl+p.Url))
		buf.WriteString(fmt.Sprintf("<lastmod>%s/</lastmod>", p.Created.Raw.Format(time.RFC3339)))
		buf.WriteString("<changefreq>daily</changefreq>")
		buf.WriteString("<priority>0.6</priority>")
		buf.WriteString("</url>")
	}
	buf.WriteString("<url>")
	if ctx.isSuffixed {
		buf.WriteString(fmt.Sprintf("<loc>%s</loc>", baseUrl+"/archive.html"))
	} else {
		buf.WriteString(fmt.Sprintf("<loc>%s</loc>", baseUrl+"/archive"))
	}
	buf.WriteString(fmt.Sprintf("<lastmod>%s/</lastmod>", time.Now().Format(time.RFC3339)))
	buf.WriteString("<changefreq>daily</changefreq>")
	buf.WriteString("<priority>0.6</priority>")
	buf.WriteString("</url>")

	for i := 1; i <= ctx.PostPageCount; i++ {
		buf.WriteString("<url>")
		if ctx.isSuffixed {
			buf.WriteString(fmt.Sprintf("<loc>%s/posts/%d.html</loc>", baseUrl, i))
		} else {
			buf.WriteString(fmt.Sprintf("<loc>%s/posts/%d</loc>", baseUrl, i))
		}
		buf.WriteString(fmt.Sprintf("<lastmod>%s/</lastmod>", time.Now().Format(time.RFC3339)))
		buf.WriteString("<changefreq>daily</changefreq>")
		buf.WriteString("<priority>0.6</priority>")
		buf.WriteString("</url>")
	}

	for _, p := range ctx.Pages {
		buf.WriteString("<url>")
		buf.WriteString(fmt.Sprintf("<loc>%s</loc>", baseUrl+p.Url))
		buf.WriteString(fmt.Sprintf("<lastmod>%s/</lastmod>", p.Created.Raw.Format(time.RFC3339)))
		buf.WriteString("<changefreq>weekly</changefreq>")
		buf.WriteString("<priority>0.5</priority>")
		buf.WriteString("</url>")
	}
	buf.WriteString("</urlset>")
	dstFile = path.Join(ctx.DstDir, "sitemap.xml")
	os.MkdirAll(path.Dir(dstFile), os.ModePerm)
	r.Error = ioutil.WriteFile(dstFile, buf.Bytes(), os.ModePerm)
}
<commit_msg>[builder] add tag links in sitemap<commit_after>package builder

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"time"
)

func (b *Builder) WriteFeed(ctx *Context, r *Report) {
	baseUrl := "http://" + ctx.Meta.Domain
	var buf bytes.Buffer
	buf.WriteString(`<?xml version="1.0" encoding="UTF-8"?>`)
	buf.WriteString(`<rss version="2.0">`)
	buf.WriteString("<channel>")
	buf.WriteString(fmt.Sprintf("<title>%s</title>", ctx.Meta.Title+" - "+ctx.Meta.Subtitle))
	buf.WriteString(fmt.Sprintf("<link>%s</link>", baseUrl))
	buf.WriteString(fmt.Sprintf("<description>%s</description>", ctx.Meta.Desc))
	buf.WriteString(fmt.Sprintf("<lastBuildDate>%s</lastBuildDate>", time.Now().Format(time.RFC1123Z)))
	for _, p := range ctx.Posts {
		buf.WriteString("<item>")
		buf.WriteString(fmt.Sprintf("<title>%s</title>", p.Title))
		buf.WriteString(fmt.Sprintf("<link>%s</link>", baseUrl+p.Url))
		// buf.WriteString(fmt.Sprintf("<comments>%s</comments>", fi.Comments))
		buf.WriteString(fmt.Sprintf("<pubDate>%s</pubDate>", p.Created.Raw.Format(time.RFC1123Z)))
		for _, c := range p.Tags {
			buf.WriteString(fmt.Sprintf("<category>%s</category>", c.Name))
		}
		buf.WriteString(fmt.Sprintf("<description><![CDATA[ %s ]]></description>", p.ContentHTML()))
		buf.WriteString("</item>")
	}
	buf.WriteString("</channel>")
	buf.WriteString("</rss>")

	dstFile := path.Join(ctx.DstDir, "feed.xml")
	os.MkdirAll(path.Dir(dstFile), os.ModePerm)
	r.Error = ioutil.WriteFile(dstFile, buf.Bytes(), os.ModePerm)
	if r.Error != nil {
		return
	}

	// sitemap
	buf.Reset()
	buf.WriteString(`<?xml version="1.0" encoding="UTF-8"?>`)
	buf.WriteString(`<?xml-stylesheet type="text/xsl" href="/static/sitemap.xsl"?>`)
	buf.WriteString(`<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">`)
	buf.WriteString("<url>")
	buf.WriteString(fmt.Sprintf("<loc>%s</loc>", baseUrl))
	buf.WriteString(fmt.Sprintf("<lastmod>%s/</lastmod>", time.Now().Format(time.RFC3339)))
	buf.WriteString("<changefreq>daily</changefreq>")
	buf.WriteString("<priority>1.0</priority>")
	buf.WriteString("</url>")

	for _, p := range ctx.Posts {
		buf.WriteString("<url>")
		buf.WriteString(fmt.Sprintf("<loc>%s</loc>", baseUrl+p.Url))
		buf.WriteString(fmt.Sprintf("<lastmod>%s/</lastmod>", p.Created.Raw.Format(time.RFC3339)))
		buf.WriteString("<changefreq>daily</changefreq>")
		buf.WriteString("<priority>0.6</priority>")
		buf.WriteString("</url>")
	}
	buf.WriteString("<url>")
	if ctx.isSuffixed {
		buf.WriteString(fmt.Sprintf("<loc>%s</loc>", baseUrl+"/archive.html"))
	} else {
		buf.WriteString(fmt.Sprintf("<loc>%s</loc>", baseUrl+"/archive"))
	}
	buf.WriteString(fmt.Sprintf("<lastmod>%s/</lastmod>", time.Now().Format(time.RFC3339)))
	buf.WriteString("<changefreq>daily</changefreq>")
	buf.WriteString("<priority>0.6</priority>")
	buf.WriteString("</url>")

	for i := 1; i <= ctx.PostPageCount; i++ {
		buf.WriteString("<url>")
		if ctx.isSuffixed {
			buf.WriteString(fmt.Sprintf("<loc>%s/posts/%d.html</loc>", baseUrl, i))
		} else {
			buf.WriteString(fmt.Sprintf("<loc>%s/posts/%d</loc>", baseUrl, i))
		}
		buf.WriteString(fmt.Sprintf("<lastmod>%s/</lastmod>", time.Now().Format(time.RFC3339)))
		buf.WriteString("<changefreq>daily</changefreq>")
		buf.WriteString("<priority>0.6</priority>")
		buf.WriteString("</url>")
	}

	for _, t := range ctx.Tags {
		buf.WriteString("<url>")
		buf.WriteString(fmt.Sprintf("<loc>%s</loc>", baseUrl+t.Url))
		buf.WriteString(fmt.Sprintf("<lastmod>%s/</lastmod>", time.Now().Format(time.RFC3339)))
		buf.WriteString("<changefreq>weekly</changefreq>")
		buf.WriteString("<priority>0.5</priority>")
		buf.WriteString("</url>")
	}

	for _, p := range ctx.Pages {
		buf.WriteString("<url>")
		buf.WriteString(fmt.Sprintf("<loc>%s</loc>", baseUrl+p.Url))
		buf.WriteString(fmt.Sprintf("<lastmod>%s/</lastmod>", p.Created.Raw.Format(time.RFC3339)))
		buf.WriteString("<changefreq>weekly</changefreq>")
		buf.WriteString("<priority>0.5</priority>")
		buf.WriteString("</url>")
	}

	buf.WriteString("</urlset>")
	dstFile = path.Join(ctx.DstDir, "sitemap.xml")
	os.MkdirAll(path.Dir(dstFile), os.ModePerm)
	r.Error = ioutil.WriteFile(dstFile, buf.Bytes(), os.ModePerm)
}
<|endoftext|>


<commit_before>package fuse

import (
	"fmt"
	"unsafe"
	"sync"
)

// HandleMap translates objects in Go space to 64-bit handles that can
// be given out to -say- the linux kernel.  It uses the free bits
// (16+3 on x64_64 and 32 on x86) to do an extra sanity check on the
// data.  (Thanks to Russ Cox for this suggestion).  In addition, it
// stores the object in a map, so the Go runtime will not garbage
// collect it.
//
// To use it, include Handled as first member of the structure
// you wish to export.
//
// This structure is thread-safe.
type HandleMap struct {
	mutex    sync.Mutex
	handles  map[uint64]*Handled
	nextFree uint32
}

func (me *HandleMap) verify() {
	if !paranoia {
		return
	}

	me.mutex.Lock()
	defer me.mutex.Unlock()
	for k, v := range me.handles {
		if DecodeHandle(k) != v {
			panic("handle map out of sync")
		}
	}
}

func NewHandleMap() *HandleMap {
	return &HandleMap{
		handles:  make(map[uint64]*Handled),
		nextFree: 1, // to make tests easier.
	}
}

type Handled struct {
	check uint32
}

func (me *HandleMap) Count() int {
	me.mutex.Lock()
	defer me.mutex.Unlock()
	return len(me.handles)
}

func (me *HandleMap) Register(obj *Handled) (handle uint64) {
	if obj.check != 0 {
		panic("Object already has a handle.")
	}
	me.mutex.Lock()
	defer me.mutex.Unlock()

	handle = uint64(uintptr(unsafe.Pointer(obj)))
	check := me.nextFree
	me.nextFree++
	if unsafe.Sizeof(obj) == 8 {
		me.nextFree = me.nextFree & (1<<(64-48+3) - 1)

		rest := (handle &^ (1<<48 - 1)) | (handle & (1<<3 - 1))
		if rest != 0 {
			panic("unaligned ptr or more than 48 bits in address")
		}
		handle >>= 3
		handle |= uint64(obj.check) << (64 - 48 + 3)
	}

	if unsafe.Sizeof(obj) == 4 {
		rest := (handle & 0x3)
		if rest != 0 {
			panic("unaligned ptr")
		}

		handle |= uint64(check) << 32
	}
	obj.check = check
	me.handles[handle] = obj
	return handle
}

func (me *HandleMap) Forget(handle uint64) (val *Handled) {
	val = DecodeHandle(handle)

	me.mutex.Lock()
	defer me.mutex.Unlock()
	val.check = 0
	me.handles[handle] = nil, false
	return val
}

func DecodeHandle(handle uint64) (val *Handled) {
	var check uint32
	if unsafe.Sizeof(val) == 8 {
		ptrBits := uintptr(handle & (1<<45 - 1))
		check = uint32(handle >> 45)
		val = (*Handled)(unsafe.Pointer(ptrBits << 3))
	}
	if unsafe.Sizeof(val) == 4 {
		check = uint32(handle >> 32)
		val = (*Handled)(unsafe.Pointer(uintptr(handle & ((1 << 32) - 1))))
	}
	if val.check != check {
		msg := fmt.Sprintf("handle check mismatch; handle has 0x%x, object has 0x%x",
			check, val.check)
		panic(msg)
	}
	return val
}
<commit_msg>Fix TestHandleMapPointerLayout test case of x86_64.<commit_after>package fuse

import (
	"fmt"
	"unsafe"
	"sync"
)

// HandleMap translates objects in Go space to 64-bit handles that can
// be given out to -say- the linux kernel.  It uses the free bits
// (16+3 on x64_64 and 32 on x86) to do an extra sanity check on the
// data.  (Thanks to Russ Cox for this suggestion).  In addition, it
// stores the object in a map, so the Go runtime will not garbage
// collect it.
//
// To use it, include Handled as first member of the structure
// you wish to export.
//
// This structure is thread-safe.
type HandleMap struct {
	mutex    sync.Mutex
	handles  map[uint64]*Handled
	nextFree uint32
}

func (me *HandleMap) verify() {
	if !paranoia {
		return
	}

	me.mutex.Lock()
	defer me.mutex.Unlock()
	for k, v := range me.handles {
		if DecodeHandle(k) != v {
			panic("handle map out of sync")
		}
	}
}

func NewHandleMap() *HandleMap {
	return &HandleMap{
		handles:  make(map[uint64]*Handled),
		nextFree: 1, // to make tests easier.
	}
}

type Handled struct {
	check uint32
}

func (me *HandleMap) Count() int {
	me.mutex.Lock()
	defer me.mutex.Unlock()
	return len(me.handles)
}

func (me *HandleMap) Register(obj *Handled) (handle uint64) {
	me.mutex.Lock()
	defer me.mutex.Unlock()

	handle = uint64(uintptr(unsafe.Pointer(obj)))
	check := me.nextFree
	me.nextFree++
	if unsafe.Sizeof(obj) == 8 {
		me.nextFree = me.nextFree & (1<<(64-48+3) - 1)

		rest := (handle &^ (1<<48 - 1)) | (handle & (1<<3 - 1))
		if rest != 0 {
			panic("unaligned ptr or more than 48 bits in address")
		}
		handle >>= 3
		handle |= uint64(obj.check) << (64 - 48 + 3)
	}

	if unsafe.Sizeof(obj) == 4 {
		rest := (handle & 0x3)
		if rest != 0 {
			panic("unaligned ptr")
		}

		handle |= uint64(check) << 32
	}
	if obj.check != 0 {
		panic("Object already has a handle.")
	}
	obj.check = check
	me.handles[handle] = obj
	return handle
}

func (me *HandleMap) Forget(handle uint64) (val *Handled) {
	val = DecodeHandle(handle)

	me.mutex.Lock()
	defer me.mutex.Unlock()
	val.check = 0
	me.handles[handle] = nil, false
	return val
}

func DecodeHandle(handle uint64) (val *Handled) {
	var check uint32
	if unsafe.Sizeof(val) == 8 {
		ptrBits := uintptr(handle & (1<<45 - 1))
		check = uint32(handle >> 45)
		val = (*Handled)(unsafe.Pointer(ptrBits << 3))
	}
	if unsafe.Sizeof(val) == 4 {
		check = uint32(handle >> 32)
		val = (*Handled)(unsafe.Pointer(uintptr(handle & ((1 << 32) - 1))))
	}
	if val.check != check {
		msg := fmt.Sprintf("handle check mismatch; handle has 0x%x, object has 0x%x",
			check, val.check)
		panic(msg)
	}
	return val
}
<|endoftext|>