<commit_before>package geo_skeleton

import (
	"fmt"
	"log"
	"math"
	"os"
	"time"
)

import (
	"./utils"
)

import (
	"github.com/paulmach/go.geojson"
	"github.com/sjsafranek/SkeletonDB"
)

// https://gist.github.com/DavidVaini/10308388
func Round(f float64) float64 {
	return math.Floor(f + .5)
}

func RoundToPrecision(f float64, places int) float64 {
	shift := math.Pow(10, float64(places))
	return Round(f*shift) / shift
}

// DB application Database
var (
	COMMIT_LOG_FILE string = "geo_skeleton_commit.log"
)

func NewGeoSkeletonDB(db_file string) Database {
	var geoDb = Database{
		File:  db_file,
		Table: "GeoJsonDatasources",
		DB:    skeleton.Database{File: db_file}}
	geoDb.Init()
	return geoDb
}

// Database strust for application.
type Database struct {
	Table            string
	File             string
	commit_log_queue chan string
	Precision        int
	DB               skeleton.Database
}

func (self Database) Init() {

	// Set initial data precision
	self.Precision = 8

	// start commit log
	go self.startCommitLog()

	// default table
	if "" == self.Table {
		self.Table = "GeoJSONLayers"
	}

	conn := self.DB.Connect()
	defer conn.Close()

	err := self.DB.CreateTable(conn, self.Table)
	if nil != err {
		panic(err)
	}
}

// Starts Database commit log
func (self *Database) startCommitLog() {
	self.commit_log_queue = make(chan string, 10000)
	// open file to write database commit log
	COMMIT_LOG, err := os.OpenFile(COMMIT_LOG_FILE, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
	if err != nil {
		log.Println(err)
	}
	defer COMMIT_LOG.Close()
	// read from chan and write to file
	for {
		if len(self.commit_log_queue) > 0 {
			line := <-self.commit_log_queue
			if _, err := COMMIT_LOG.WriteString(line + "\n"); err != nil {
				panic(err)
			}
		} else {
			time.Sleep(1000 * time.Millisecond)
		}
	}
}

// CommitQueueLength returns length of database commit_log_queue
// @returns int
func (self *Database) CommitQueueLength() int {
	return len(self.commit_log_queue)
}

// NewLayer creates new datasource layer
// @returns string - datasource id
// @returns Error
// TODO: RENAME TO NewDatasource
func (self *Database) NewLayer() (string, error) {
	// create geojson
	datasource_id, _ := utils.NewUUID()
	geojs := geojson.NewFeatureCollection()
	// convert to bytes
	value, err := geojs.MarshalJSON()
	if err != nil {
		return "", nil
	}
	self.commit_log_queue <- `{"method": "create_datasource", "data": { "datasource": "` + datasource_id + `", "layer": ` + string(value) + `}}`
	// Insert layer into database
	err = self.DB.Insert(self.Table, datasource_id, value)
	if err != nil {
		panic(err)
	}
	return datasource_id, err
}

// InsertLayer inserts layer into database
// @param datasource {string}
// @param geojs {Geojson}
// @returns Error
func (self *Database) InsertLayer(datasource_id string, geojs *geojson.FeatureCollection) error {
	// convert to bytes
	value, err := geojs.MarshalJSON()
	if err != nil {
		return err
	}
	err = self.DB.Insert(self.Table, datasource_id, value)
	if err != nil {
		panic(err)
	}

	return err
}

// GetLayer returns layer from database
// @param datasource {string}
// @returns Geojson
// @returns Error
func (self *Database) GetLayer(datasource_id string) (*geojson.FeatureCollection, error) {
	val, err := self.DB.Select(self.Table, datasource_id)
	if err != nil {
		return nil, err
	}
	if "" == string(val) {
		return nil, fmt.Errorf("Datasource not found")
	}
	// Read to struct
	geojs, err := geojson.UnmarshalFeatureCollection(val)
	if err != nil {
		return geojs, err
	}
	return geojs, nil
}

// DeleteLayer deletes layer from database
// @param datasource {string}
// @returns Error
func (self *Database) DeleteLayer(datasource_id string) error {
	self.commit_log_queue <- `{"method": "delete_layer", "data": { "datasource": "` + datasource_id + `"}}`
	err := self.DB.Remove(datasource_id, self.Table)
	return err
}

func (self *Database) normalizeGeometry(feat *geojson.Feature) (*geojson.Feature, error) {
	// FIT TO 7 - 8 DECIMAL PLACES OF PRECISION
	if nil == feat.Geometry {
		return nil, fmt.Errorf("Feature has no geometry!")
	}

	switch feat.Geometry.Type {

	case geojson.GeometryPoint:
		// []float64
		feat.Geometry.Point[0] = RoundToPrecision(feat.Geometry.Point[0], self.Precision)
		feat.Geometry.Point[1] = RoundToPrecision(feat.Geometry.Point[1], self.Precision)

	case geojson.GeometryMultiPoint:
		// [][]float64
		for i := range feat.Geometry.MultiPoint {
			for j := range feat.Geometry.MultiPoint[i] {
				feat.Geometry.MultiPoint[i][j] = RoundToPrecision(feat.Geometry.MultiPoint[i][j], self.Precision)
			}
		}

	case geojson.GeometryLineString:
		// [][]float64
		for i := range feat.Geometry.LineString {
			for j := range feat.Geometry.LineString[i] {
				feat.Geometry.LineString[i][j] = RoundToPrecision(feat.Geometry.LineString[i][j], self.Precision)
			}
		}

	case geojson.GeometryMultiLineString:
		// [][][]float64
		for i := range feat.Geometry.MultiLineString {
			for j := range feat.Geometry.MultiLineString[i] {
				for k := range feat.Geometry.MultiLineString[i][j] {
					feat.Geometry.MultiLineString[i][j][k] = RoundToPrecision(feat.Geometry.MultiLineString[i][j][k], self.Precision)
				}
			}
		}

	case geojson.GeometryPolygon:
		// [][][]float64
		for i := range feat.Geometry.Polygon {
			for j := range feat.Geometry.Polygon[i] {
				for k := range feat.Geometry.Polygon[i][j] {
					feat.Geometry.Polygon[i][j][k] = RoundToPrecision(feat.Geometry.Polygon[i][j][k], self.Precision)
				}
			}
		}

	case geojson.GeometryMultiPolygon:
		// [][][][]float64
		for i := range feat.Geometry.MultiPolygon {
			log.Printf("%v\n", feat.Geometry.MultiPolygon[i])
		}

	}

	/*
		//case GeometryCollection:
		//	geo.Geometries = g.Geometries
		//	// log.Printf("%v\n", feat.Geometry.Geometries)

	*/
	return feat, nil
}

func (self *Database) normalizeProperties(feat *geojson.Feature, featCollection *geojson.FeatureCollection) *geojson.Feature {

	// check if nil map
	if nil == feat.Properties {
		feat.Properties = make(map[string]interface{})
	}

	if 0 == len(featCollection.Features) {
		return feat
	}
	// Standardize properties for new feature
	for j := range featCollection.Features[0].Properties {
		if _, ok := feat.Properties[j]; !ok {
			feat.Properties[j] = ""
		}
	}

	// Standardize properties for existing features
	for i := range featCollection.Features {
		for j := range feat.Properties {
			if _, ok := featCollection.Features[i].Properties[j]; !ok {
				featCollection.Features[i].Properties[j] = ""
			}
		}
	}

	return feat
}

// InsertFeature adds feature to layer. Updates layer in Database
// @param datasource {string}
// @param feat {Geojson Feature}
// @returns Error
func (self *Database) InsertFeature(datasource_id string, feat *geojson.Feature) error {

	if nil == feat {
		return fmt.Errorf("feature value is <nil>!")
	}

	// Get layer from database
	featCollection, err := self.GetLayer(datasource_id)
	if err != nil {
		return err
	}

	// Apply required columns
	now := time.Now().Unix()

	// check if nil map
	if nil == feat.Properties {
		feat.Properties = make(map[string]interface{})
	}

	feat.Properties["is_active"] = true
	feat.Properties["is_deleted"] = false
	feat.Properties["date_created"] = now
	feat.Properties["date_modified"] = now
	feat.Properties["geo_id"] = fmt.Sprintf("%v", now)

	feat, err = self.normalizeGeometry(feat)
	if nil != err {
		return err
	}

	feat = self.normalizeProperties(feat, featCollection)

	// Write to commit log
	value, err := feat.MarshalJSON()
	if err != nil {
		return err
	}
	self.commit_log_queue <- `{"method": "insert_feature", "data": { "datasource": "` + datasource_id + `", "feature": ` + string(value) + `}}`

	// Add new feature to layer
	featCollection.AddFeature(feat)

	// insert layer
	err = self.InsertLayer(datasource_id, featCollection)
	if err != nil {
		panic(err)
	}
	return err
}

// EditFeature Edits feature in layer. Updates layer in Database
// @param datasource {string}
// @param geo_id {string}
// @param feat {Geojson Feature}
// @returns Error
func (self *Database) EditFeature(datasource_id string, geo_id string, feat *geojson.Feature) error {

	// Get layer from database
	featCollection, err := self.GetLayer(datasource_id)
	if err != nil {
		return err
	}

	feature_exists := false

	for i := range featCollection.Features {
		if geo_id == fmt.Sprintf("%v", featCollection.Features[i].Properties["geo_id"]) {

			now := time.Now().Unix()
			feat.Properties["date_modified"] = now

			feat, err = self.normalizeGeometry(feat)
			if nil != err {
				return err
			}

			feat = self.normalizeProperties(feat, featCollection)
			featCollection.Features[i] = feat
			// Write to commit log
			value, err := feat.MarshalJSON()
			if err != nil {
				return err
			}
			self.commit_log_queue <- `{"method": "edit_feature", "data": { "datasource": "` + datasource_id + `", "geo_id": "` + geo_id + `", "feature": ` + string(value) + `}}`
			feature_exists = true
		}
	}

	if !feature_exists {
		return fmt.Errorf("feature not found!")
	}

	// insert layer
	err = self.InsertLayer(datasource_id, featCollection)
	if err != nil {
		panic(err)
	}
	return err
}
<commit_msg>fixed utils<commit_after>package geo_skeleton

import (
	"fmt"
	"log"
	"math"
	"os"
	"time"
)

import (
	"github.com/paulmach/go.geojson"
	"github.com/sjsafranek/SkeletonDB"
)

// https://gist.github.com/DavidVaini/10308388
func Round(f float64) float64 {
	return math.Floor(f + .5)
}

func RoundToPrecision(f float64, places int) float64 {
	shift := math.Pow(10, float64(places))
	return Round(f*shift) / shift
}

// DB application Database
var (
	COMMIT_LOG_FILE string = "geo_skeleton_commit.log"
)

func NewGeoSkeletonDB(db_file string) Database {
	var geoDb = Database{
		File:  db_file,
		Table: "GeoJsonDatasources",
		DB:    skeleton.Database{File: db_file}}
	geoDb.Init()
	return geoDb
}

// Database strust for application.
type Database struct {
	Table            string
	File             string
	commit_log_queue chan string
	Precision        int
	DB               skeleton.Database
}

func (self Database) Init() {

	// Set initial data precision
	self.Precision = 8

	// start commit log
	go self.startCommitLog()

	// default table
	if "" == self.Table {
		self.Table = "GeoJSONLayers"
	}

	conn := self.DB.Connect()
	defer conn.Close()

	err := self.DB.CreateTable(conn, self.Table)
	if nil != err {
		panic(err)
	}
}

// Starts Database commit log
func (self *Database) startCommitLog() {
	self.commit_log_queue = make(chan string, 10000)
	// open file to write database commit log
	COMMIT_LOG, err := os.OpenFile(COMMIT_LOG_FILE, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
	if err != nil {
		log.Println(err)
	}
	defer COMMIT_LOG.Close()
	// read from chan and write to file
	for {
		if len(self.commit_log_queue) > 0 {
			line := <-self.commit_log_queue
			if _, err := COMMIT_LOG.WriteString(line + "\n"); err != nil {
				panic(err)
			}
		} else {
			time.Sleep(1000 * time.Millisecond)
		}
	}
}

// CommitQueueLength returns length of database commit_log_queue
// @returns int
func (self *Database) CommitQueueLength() int {
	return len(self.commit_log_queue)
}

// NewLayer creates new datasource layer
// @returns string - datasource id
// @returns Error
// TODO: RENAME TO NewDatasource
func (self *Database) NewLayer() (string, error) {
	// create geojson
	datasource_id, _ := NewUUID()
	geojs := geojson.NewFeatureCollection()
	// convert to bytes
	value, err := geojs.MarshalJSON()
	if err != nil {
		return "", nil
	}
	self.commit_log_queue <- `{"method": "create_datasource", "data": { "datasource": "` + datasource_id + `", "layer": ` + string(value) + `}}`
	// Insert layer into database
	err = self.DB.Insert(self.Table, datasource_id, value)
	if err != nil {
		panic(err)
	}
	return datasource_id, err
}

// InsertLayer inserts layer into database
// @param datasource {string}
// @param geojs {Geojson}
// @returns Error
func (self *Database) InsertLayer(datasource_id string, geojs *geojson.FeatureCollection) error {
	// convert to bytes
	value, err := geojs.MarshalJSON()
	if err != nil {
		return err
	}
	err = self.DB.Insert(self.Table, datasource_id, value)
	if err != nil {
		panic(err)
	}

	return err
}

// GetLayer returns layer from database
// @param datasource {string}
// @returns Geojson
// @returns Error
func (self *Database) GetLayer(datasource_id string) (*geojson.FeatureCollection, error) {
	val, err := self.DB.Select(self.Table, datasource_id)
	if err != nil {
		return nil, err
	}
	if "" == string(val) {
		return nil, fmt.Errorf("Datasource not found")
	}
	// Read to struct
	geojs, err := geojson.UnmarshalFeatureCollection(val)
	if err != nil {
		return geojs, err
	}
	return geojs, nil
}

// DeleteLayer deletes layer from database
// @param datasource {string}
// @returns Error
func (self *Database) DeleteLayer(datasource_id string) error {
	self.commit_log_queue <- `{"method": "delete_layer", "data": { "datasource": "` + datasource_id + `"}}`
	err := self.DB.Remove(datasource_id, self.Table)
	return err
}

func (self *Database) normalizeGeometry(feat *geojson.Feature) (*geojson.Feature, error) {
	// FIT TO 7 - 8 DECIMAL PLACES OF PRECISION
	if nil == feat.Geometry {
		return nil, fmt.Errorf("Feature has no geometry!")
	}

	switch feat.Geometry.Type {

	case geojson.GeometryPoint:
		// []float64
		feat.Geometry.Point[0] = RoundToPrecision(feat.Geometry.Point[0], self.Precision)
		feat.Geometry.Point[1] = RoundToPrecision(feat.Geometry.Point[1], self.Precision)

	case geojson.GeometryMultiPoint:
		// [][]float64
		for i := range feat.Geometry.MultiPoint {
			for j := range feat.Geometry.MultiPoint[i] {
				feat.Geometry.MultiPoint[i][j] = RoundToPrecision(feat.Geometry.MultiPoint[i][j], self.Precision)
			}
		}

	case geojson.GeometryLineString:
		// [][]float64
		for i := range feat.Geometry.LineString {
			for j := range feat.Geometry.LineString[i] {
				feat.Geometry.LineString[i][j] = RoundToPrecision(feat.Geometry.LineString[i][j], self.Precision)
			}
		}

	case geojson.GeometryMultiLineString:
		// [][][]float64
		for i := range feat.Geometry.MultiLineString {
			for j := range feat.Geometry.MultiLineString[i] {
				for k := range feat.Geometry.MultiLineString[i][j] {
					feat.Geometry.MultiLineString[i][j][k] = RoundToPrecision(feat.Geometry.MultiLineString[i][j][k], self.Precision)
				}
			}
		}

	case geojson.GeometryPolygon:
		// [][][]float64
		for i := range feat.Geometry.Polygon {
			for j := range feat.Geometry.Polygon[i] {
				for k := range feat.Geometry.Polygon[i][j] {
					feat.Geometry.Polygon[i][j][k] = RoundToPrecision(feat.Geometry.Polygon[i][j][k], self.Precision)
				}
			}
		}

	case geojson.GeometryMultiPolygon:
		// [][][][]float64
		for i := range feat.Geometry.MultiPolygon {
			log.Printf("%v\n", feat.Geometry.MultiPolygon[i])
		}

	}

	/*
		//case GeometryCollection:
		//	geo.Geometries = g.Geometries
		//	// log.Printf("%v\n", feat.Geometry.Geometries)

	*/
	return feat, nil
}

func (self *Database) normalizeProperties(feat *geojson.Feature, featCollection *geojson.FeatureCollection) *geojson.Feature {

	// check if nil map
	if nil == feat.Properties {
		feat.Properties = make(map[string]interface{})
	}

	if 0 == len(featCollection.Features) {
		return feat
	}
	// Standardize properties for new feature
	for j := range featCollection.Features[0].Properties {
		if _, ok := feat.Properties[j]; !ok {
			feat.Properties[j] = ""
		}
	}

	// Standardize properties for existing features
	for i := range featCollection.Features {
		for j := range feat.Properties {
			if _, ok := featCollection.Features[i].Properties[j]; !ok {
				featCollection.Features[i].Properties[j] = ""
			}
		}
	}

	return feat
}

// InsertFeature adds feature to layer. Updates layer in Database
// @param datasource {string}
// @param feat {Geojson Feature}
// @returns Error
func (self *Database) InsertFeature(datasource_id string, feat *geojson.Feature) error {

	if nil == feat {
		return fmt.Errorf("feature value is <nil>!")
	}

	// Get layer from database
	featCollection, err := self.GetLayer(datasource_id)
	if err != nil {
		return err
	}

	// Apply required columns
	now := time.Now().Unix()

	// check if nil map
	if nil == feat.Properties {
		feat.Properties = make(map[string]interface{})
	}

	feat.Properties["is_active"] = true
	feat.Properties["is_deleted"] = false
	feat.Properties["date_created"] = now
	feat.Properties["date_modified"] = now
	feat.Properties["geo_id"] = fmt.Sprintf("%v", now)

	feat, err = self.normalizeGeometry(feat)
	if nil != err {
		return err
	}

	feat = self.normalizeProperties(feat, featCollection)

	// Write to commit log
	value, err := feat.MarshalJSON()
	if err != nil {
		return err
	}
	self.commit_log_queue <- `{"method": "insert_feature", "data": { "datasource": "` + datasource_id + `", "feature": ` + string(value) + `}}`

	// Add new feature to layer
	featCollection.AddFeature(feat)

	// insert layer
	err = self.InsertLayer(datasource_id, featCollection)
	if err != nil {
		panic(err)
	}
	return err
}

// EditFeature Edits feature in layer. Updates layer in Database
// @param datasource {string}
// @param geo_id {string}
// @param feat {Geojson Feature}
// @returns Error
func (self *Database) EditFeature(datasource_id string, geo_id string, feat *geojson.Feature) error {

	// Get layer from database
	featCollection, err := self.GetLayer(datasource_id)
	if err != nil {
		return err
	}

	feature_exists := false

	for i := range featCollection.Features {
		if geo_id == fmt.Sprintf("%v", featCollection.Features[i].Properties["geo_id"]) {

			now := time.Now().Unix()
			feat.Properties["date_modified"] = now

			feat, err = self.normalizeGeometry(feat)
			if nil != err {
				return err
			}

			feat = self.normalizeProperties(feat, featCollection)
			featCollection.Features[i] = feat
			// Write to commit log
			value, err := feat.MarshalJSON()
			if err != nil {
				return err
			}
			self.commit_log_queue <- `{"method": "edit_feature", "data": { "datasource": "` + datasource_id + `", "geo_id": "` + geo_id + `", "feature": ` + string(value) + `}}`
			feature_exists = true
		}
	}

	if !feature_exists {
		return fmt.Errorf("feature not found!")
	}

	// insert layer
	err = self.InsertLayer(datasource_id, featCollection)
	if err != nil {
		panic(err)
	}
	return err
}
<|endoftext|>


<commit_before>package geomop

// Simplify performs Ramer-Douglas-Peucker simplification.

import (
	"github.com/foobaz/geom"
)

// there are two components that might be extremely slow
// the ring simplification algorithm searches for the
// maximally separated nodes, which is O(n^2), and the
// RDP simplification itself is typically O(n log n) but
// is O(n^2) in its worst case.
//
// If a poly with more than maxNodesToSimplify is used
// as input to the simplifier it will return the original.
const maxNodesToSimplify = 10000

func Simplify(g geom.T, eps float64) {
	switch (g).(type) {
	case geom.Polygon:
		p := g.(geom.Polygon)
		for n, r := range p {
			p[n] = SimplifyRing(r, eps)
		}
	case geom.MultiPolygon:
		for _, p := range g.(geom.MultiPolygon) {
			Simplify(p, eps)
		}
	case geom.GeometryCollection:
		for _, g := range g.(geom.GeometryCollection) {
			Simplify(g, eps)
		}
	}
}

// Simplify a single ring; return the new ring.
func SimplifyRing(r geom.Ring, eps float64) geom.Ring {
	// handle the ring case, break it down into two lines.
	// rule out degenerate case
	numPoints := len(r)
	if numPoints < 4 {
		return r
	}
	// to prevent this from bogging down on a ridiculous
	// input
	if numPoints > maxNodesToSimplify {
		return r
	}
	// find the two points with maximal separation,
	// and use those two points to create the polylines
	// which will be simplified
	//
	// this should result in stable simplification,
	// meaning that the resulting simplified polygon
	// will be the same when processed with a
	// different starting point.
	maxDistance := 0.0
	startIndex := 0
	endIndex := 0
	for i := 0; i < (numPoints - 1); i++ {
		for j := i + 1; j < numPoints; j++ {
			dist := d(r[i], r[j])
			if dist > maxDistance {
				maxDistance = dist
				startIndex = i
				endIndex = j
			}
		}
	}
	var s1, s2 []geom.Point
	// note: the nodes at startIndex and endIndex must be in both polylines
	s1 = append(s1, r[startIndex:endIndex+1]...)
	s2 = append(s2, r[endIndex:]...)
	s2 = append(s2, r[:startIndex+1]...)

	news1 := rdpSimplify(s1, eps)
	news2 := rdpSimplify(s2, eps)

	// seam these back together
	var newRing geom.Ring
	newRing = append(newRing, news1...)
	newRing = append(newRing, news2[1:len(news2)-1]...)

	return newRing
}

func rdpSimplify(points []geom.Point, epsilon float64) []geom.Point {
	return points[:rdpCompress(points, epsilon)]
}

func rdpCompress(points []geom.Point, epsilon float64) int {
	end := len(points)

	if end < 3 {
		// return points
		return end
	}

	// Find the point with the maximum distance
	var (
		first = points[0]
		last  = points[end-1]

		dmax  float64
		index int
	)

	for i := 1; i < end-1; i++ {
		d := distPointToSegment(points[i], first, last)
		if d > dmax {
			dmax, index = d, i
		}
	}

	// If max distance is lte to epsilon, return segment containing
	// the first and last points.
	if dmax <= epsilon {
		// return []point{first, last}
		points[1] = last
		return 2
	}

	// Recursive call
	r1 := rdpCompress(points[:index+1], epsilon)
	r2 := rdpCompress(points[index:], epsilon)

	// Build the result list
	// return append(r1[:len(r1)-1], r2...)
	x := r1 - 1
	n := copy(points[x:], points[index:index+r2])

	return x + n
}
<commit_msg>added grid-based poly simplifier<commit_after>package geomop

// Simplify performs Ramer-Douglas-Peucker simplification.

import (
	"github.com/foobaz/geom"
	"math"
)

type algorithm int

const (
	TypeRDP algorithm = iota
	TypeGrid
)

// there are two components that might be extremely slow
// the ring simplification algorithm searches for the
// maximally separated nodes, which is O(n^2), and the
// RDP simplification itself is typically O(n log n) but
// is O(n^2) in its worst case.
//
// If a poly with more than maxNodesToSimplify is used
// as input to the simplifier it will return the original.
const maxNodesToSimplify = 10000

func Simplify(g geom.T, eps float64, alg algorithm) {
	switch (g).(type) {
	case geom.Polygon:
		p := g.(geom.Polygon)
		for n, r := range p {
			// note if the first and last points are the same
			inputDupe := d(r[0], r[len(r)-1]) < tolerance
			newRing := simplifyRing(r, eps, alg)
			// if our first and last were the same before, make it so now
			if inputDupe {
				outputDupe := d(newRing[0], newRing[len(newRing)-1]) < tolerance
				if !outputDupe {
					newRing = append(newRing, newRing[0])
				}
			}
			p[n] = newRing

		}
	case geom.MultiPolygon:
		for _, p := range g.(geom.MultiPolygon) {
			Simplify(p, eps, alg)
		}
	case geom.GeometryCollection:
		for _, g := range g.(geom.GeometryCollection) {
			Simplify(g, eps, alg)
		}
	}
}

// Simplify a single ring; return the new ring.
func simplifyRing(r geom.Ring, eps float64, alg algorithm) geom.Ring {
	// rule out degenerate case
	if len(r) < 4 {
		return r
	}
	// which type of simplification?
	switch alg {
	case TypeRDP:
		return simplifyRingRDP(r, eps)
	case TypeGrid:
		return simplifyRingGrid(r, eps)
	default:
		panic("unknown poly simplification method invoked")
	}
}
func simplifyRingRDP(r geom.Ring, eps float64) geom.Ring {
	// to prevent this from bogging down on a ridiculous
	// input
	numPoints := len(r)
	if numPoints > maxNodesToSimplify {
		return r
	}
	// find the two points with maximal separation,
	// and use those two points to create the polylines
	// which will be simplified
	//
	// this should result in stable simplification,
	// meaning that the resulting simplified polygon
	// will be the same when processed with a
	// different starting point.
	maxDistance := 0.0
	startIndex := 0
	endIndex := 0
	for i := 0; i < (numPoints - 1); i++ {
		for j := i + 1; j < numPoints; j++ {
			dist := d(r[i], r[j])
			if dist > maxDistance {
				maxDistance = dist
				startIndex = i
				endIndex = j
			}
		}
	}
	var s1, s2 []geom.Point
	// note: the nodes at startIndex and endIndex must be in both polylines
	s1 = append(s1, r[startIndex:endIndex+1]...)
	s2 = append(s2, r[endIndex:]...)
	s2 = append(s2, r[:startIndex+1]...)

	news1 := rdpSimplify(s1, eps)
	news2 := rdpSimplify(s2, eps)

	// seam these back together
	var newRing geom.Ring
	newRing = append(newRing, news1...)
	newRing = append(newRing, news2[1:len(news2)-1]...)

	return newRing
}

func simplifyRingGrid(r geom.Ring, eps float64) geom.Ring {
	var newCoords []geom.Point
	for i := 0; i < len(r); i++ {
		newPoint := roundedPoint(r[i], eps)
		count := len(newCoords)
		// add if it's the first point or different from the most recent point
		if count == 0 || d(newPoint, newCoords[count-1]) > tolerance {
			newCoords = append(newCoords, newPoint)
		}
	}

	// check for collinearity
	// this check is a little more robust due to the removal of repeated points above
	cursor := 0
	polySize := len(newCoords)
	for cursor < polySize {
		// get indices for point before and after
		before := (cursor + polySize - 1) % polySize
		after := (cursor + 1) % polySize
		if distPointToSegment(newCoords[cursor], newCoords[before], newCoords[after]) < tolerance {
			// remove the point, reduce the size, leave the cursor alone
			newCoords = append(newCoords[:cursor], newCoords[cursor+1:]...)
			polySize--
		} else {
			// keep this point, bump the cursor
			cursor++
		}
	}
	return geom.Ring(newCoords)
}

func rdpSimplify(points []geom.Point, epsilon float64) []geom.Point {
	return points[:rdpCompress(points, epsilon)]
}

func rdpCompress(points []geom.Point, epsilon float64) int {
	end := len(points)

	if end < 3 {
		// return points
		return end
	}

	// Find the point with the maximum distance
	var (
		first = points[0]
		last  = points[end-1]

		dmax  float64
		index int
	)

	for i := 1; i < end-1; i++ {
		d := distPointToSegment(points[i], first, last)
		if d > dmax {
			dmax, index = d, i
		}
	}

	// If max distance is lte to epsilon, return segment containing
	// the first and last points.
	if dmax <= epsilon {
		// return []point{first, last}
		points[1] = last
		return 2
	}

	// Recursive call
	r1 := rdpCompress(points[:index+1], epsilon)
	r2 := rdpCompress(points[index:], epsilon)

	// Build the result list
	// return append(r1[:len(r1)-1], r2...)
	x := r1 - 1
	n := copy(points[x:], points[index:index+r2])

	return x + n
}

func roundedPoint(p geom.Point, eps float64) geom.Point {
	// protect against stupidity
	if eps == 0.0 {
		return p
	}
	var newPoint geom.Point
	for i := 0; i < len(p); i++ {
		coord := math.Floor(p[i]/eps+0.5) * eps
		newPoint = append(newPoint, coord)
	}
	return newPoint
}
<|endoftext|>


<commit_before>package github

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
)

// For test purpose
// May evolve
type IRepoInfo interface {
	GetCount() int
	URL() string
}

type RepoInfo struct {
	ID           int     `json:"id"`
	Name         string  `json:"name"`
	Count        int     `json:"stargazers_count"`
	CreationDate string  `json:"created_at"`
	LastStarDate string  `json:"last_star_date,omitempty"`
	LastUpdate   string  `json:"last_update,omitempty"`
	WorkedOn     bool    `json:"worked_on,omitempty"`
	Timestamps   []int64 `json:"timestamps,omitempty"`
	exist        bool
}

func (info RepoInfo) URL() string {
	return "https://api.github.com/repositories/" + strconv.Itoa(info.ID) + "/stargazers"
}

func (info RepoInfo) GetCount() int {
	return info.Count
}

func (info RepoInfo) Exist() bool {
	return info.exist
}

func (info *RepoInfo) SetExist(exist bool) {
	info.exist = exist
}

// Get the api url from a repo.
// The token is an API Github token to be able to lift off the 60 requests/hour limit
// The repo is a Github repo formated as follow `:username/:reponame`
func GetRepoInfo(token, repo string) (RepoInfo, error) {
	url := "https://api.github.com/repos/" + repo
	r, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return RepoInfo{}, err
	}
	if token != "" {
		r.Header.Add("Authorization", "token "+token)
	}

	client := http.Client{}
	resp, err := client.Do(r)
	if err != nil {
		return RepoInfo{}, err
	}

	// The repo doesn't exist so no error, just empty repo
	if resp.StatusCode == http.StatusNotFound {
		return RepoInfo{}, nil
	}

	if resp.StatusCode != http.StatusOK {
		return RepoInfo{}, fmt.Errorf("Unexpected status, expected %d, got %d", http.StatusOK, resp.StatusCode)
	}

	bodyBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return RepoInfo{}, err
	}

	var info RepoInfo
	if err := json.Unmarshal(bodyBytes, &info); err != nil {
		return RepoInfo{}, err
	}
	info.exist = true

	return info, nil
}

func BuildLinksFormat(url string) string {
	if strings.Contains(url, "?") {
		return "<" + url + "&page=%d>; rel=\"next\", <" + url + "&page=%d>; rel=\"last\""
	}
	return "<" + url + "?page=%d>; rel=\"next\", <" + url + "?page=%d>; rel=\"last\""
}
<commit_msg>Exported GH API URL and log the body on error<commit_after>package github

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
)

// For test purpose
// May evolve
type IRepoInfo interface {
	GetCount() int
	URL() string
}

const (
	GithubRepoURL = "https://api.github.com/repos/"
)

type RepoInfo struct {
	ID           int     `json:"id"`
	Name         string  `json:"name"`
	Count        int     `json:"stargazers_count"`
	CreationDate string  `json:"created_at"`
	LastStarDate string  `json:"last_star_date,omitempty"`
	LastUpdate   string  `json:"last_update,omitempty"`
	WorkedOn     bool    `json:"worked_on,omitempty"`
	Timestamps   []int64 `json:"timestamps,omitempty"`
	exist        bool
}

func (info RepoInfo) URL() string {
	return "https://api.github.com/repositories/" + strconv.Itoa(info.ID) + "/stargazers"
}

func (info RepoInfo) GetCount() int {
	return info.Count
}

func (info RepoInfo) Exist() bool {
	return info.exist
}

func (info *RepoInfo) SetExist(exist bool) {
	info.exist = exist
}

// Get the api url from a repo.
// The token is an API Github token to be able to lift off the 60 requests/hour limit
// The repo is a Github repo formated as follow `:username/:reponame`
func GetRepoInfo(token, repo string) (RepoInfo, error) {
	url := GithubRepoURL + repo
	r, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return RepoInfo{}, err
	}
	if token != "" {
		r.Header.Add("Authorization", "token "+token)
	}

	client := http.Client{}
	resp, err := client.Do(r)
	if err != nil {
		return RepoInfo{}, err
	}

	// The repo doesn't exist so no error, just empty repo
	if resp.StatusCode == http.StatusNotFound {
		return RepoInfo{}, nil
	}

	bodyBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return RepoInfo{}, err
	}

	if resp.StatusCode != http.StatusOK {
		return RepoInfo{}, fmt.Errorf("Unexpected status, expected %d, got %d\nThe body was: %s\n", http.StatusOK, resp.StatusCode, string(bodyBytes))
	}

	var info RepoInfo
	if err := json.Unmarshal(bodyBytes, &info); err != nil {
		return RepoInfo{}, err
	}
	info.exist = true

	return info, nil
}

func BuildLinksFormat(url string) string {
	if strings.Contains(url, "?") {
		return "<" + url + "&page=%d>; rel=\"next\", <" + url + "&page=%d>; rel=\"last\""
	}
	return "<" + url + "?page=%d>; rel=\"next\", <" + url + "?page=%d>; rel=\"last\""
}
<|endoftext|>


<commit_before>// Copyright 2012, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package bson

import (
	"io"
	"math"
	"reflect"
	"strconv"
	"time"

	"github.com/youtube/vitess/go/bytes2"
)

// LenWriter records the current write position on the buffer
// and can later be used to record the number of bytes written
// in conformance to BSON spec
type LenWriter struct {
	buf *bytes2.ChunkedWriter
	off int
	b   []byte
}

// NewLenWriter returns a LenWriter that reserves the
// bytes buf so they can store the length later.
func NewLenWriter(buf *bytes2.ChunkedWriter) LenWriter {
	off := buf.Len()
	b := buf.Reserve(WORD32)
	return LenWriter{buf, off, b}
}

// Close closes the current object being encoded by
// writing bson's EOO byte and recording the length.
func (lw LenWriter) Close() {
	lw.buf.WriteByte(EOO)
	Pack.PutUint32(lw.b, uint32(lw.buf.Len()-lw.off))
}

// Marshaler is the interface that needs to be
// satisfied by types that want to implement a custom
// marshaler.
// When being invoked as a top level object, key will
// be "". In such cases, MarshalBson must not encode
// any prefix.
type Marshaler interface {
	MarshalBson(buf *bytes2.ChunkedWriter, key string)
}

func canMarshal(val reflect.Value) Marshaler {
	// Check the Marshaler interface on T.
	if marshaler, ok := val.Interface().(Marshaler); ok {
		// Don't call custom marshaler for nil values.
		switch val.Kind() {
		case reflect.Ptr, reflect.Interface, reflect.Map, reflect.Slice:
			if val.IsNil() {
				return nil
			}
		}
		return marshaler
	}
	// Check the Marshaler interface on *T.
	if val.CanAddr() {
		if marshaler, ok := val.Addr().Interface().(Marshaler); ok {
			return marshaler
		}
	}
	return nil
}

// DefaultBufferSize is the default allocation size for ChunkedWriter.
const DefaultBufferSize = 1024 * 16

// MarshalToStream marshals val into writer.
func MarshalToStream(writer io.Writer, val interface{}) (err error) {
	buf := bytes2.NewChunkedWriter(DefaultBufferSize)
	if err = MarshalToBuffer(buf, val); err != nil {
		return err
	}
	_, err = buf.WriteTo(writer)
	return err
}

// Marshal marshals val into encoded.
func Marshal(val interface{}) (encoded []byte, err error) {
	buf := bytes2.NewChunkedWriter(DefaultBufferSize)
	err = MarshalToBuffer(buf, val)
	return buf.Bytes(), err
}

// MarshalToBuffer marshals val into buf. This is the most efficient
// function to use, especially when marshaling large nested objects.
func MarshalToBuffer(buf *bytes2.ChunkedWriter, val interface{}) (err error) {
	defer handleError(&err)
	if val == nil {
		return NewBsonError("cannot marshal nil")
	}

	v := reflect.Indirect(reflect.ValueOf(val))
	if marshaler := canMarshal(v); marshaler != nil {
		marshaler.MarshalBson(buf, "")
		return
	}

	switch v.Kind() {
	case reflect.String,
		reflect.Int64, reflect.Int32, reflect.Int,
		reflect.Uint64, reflect.Uint32, reflect.Uint,
		reflect.Float64, reflect.Bool:
		EncodeSimple(buf, v.Interface())
	case reflect.Struct:
		if v.Type() == timeType {
			EncodeSimple(buf, v.Interface())
		} else {
			encodeStructContent(buf, v)
		}
	case reflect.Map:
		encodeMapContent(buf, v)
	case reflect.Slice, reflect.Array:
		if v.Type() == bytesType {
			EncodeSimple(buf, v.Interface())
		} else {
			encodeSliceContent(buf, v)
		}
	default:
		return NewBsonError("unexpected type %v", v.Type())
	}
	return nil
}

// EncodeSimple marshals simple objects that cannot be
// encoded as a top level bson document.
func EncodeSimple(buf *bytes2.ChunkedWriter, val interface{}) {
	lenWriter := NewLenWriter(buf)
	EncodeField(buf, MAGICTAG, val)
	lenWriter.Close()
}

// EncodeField encodes val using the supplied key as embedded tag.
// Unlike EncodeInterface, EncodeField can handle complex objects
// like structs, pointers, etc. But it is slower.
func EncodeField(buf *bytes2.ChunkedWriter, key string, val interface{}) {
	encodeField(buf, key, reflect.ValueOf(val))
}

func encodeField(buf *bytes2.ChunkedWriter, key string, val reflect.Value) {
	// nil interfaces show up as invalid
	if !val.IsValid() {
		EncodePrefix(buf, Null, key)
		return
	}
	if marshaler := canMarshal(val); marshaler != nil {
		marshaler.MarshalBson(buf, key)
		return
	}

	switch val.Kind() {
	case reflect.String:
		EncodeString(buf, key, val.String())
	case reflect.Int64:
		EncodeInt64(buf, key, val.Int())
	case reflect.Int32:
		EncodeInt32(buf, key, int32(val.Int()))
	case reflect.Int:
		EncodeInt(buf, key, int(val.Int()))
	case reflect.Uint64:
		EncodeUint64(buf, key, uint64(val.Uint()))
	case reflect.Uint32:
		EncodeUint32(buf, key, uint32(val.Uint()))
	case reflect.Uint:
		EncodeUint(buf, key, uint(val.Uint()))
	case reflect.Float64:
		EncodeFloat64(buf, key, val.Float())
	case reflect.Bool:
		EncodeBool(buf, key, val.Bool())
	case reflect.Struct:
		if val.Type() == timeType {
			EncodeTime(buf, key, val.Interface().(time.Time))
		} else {
			encodeStruct(buf, key, val)
		}
	case reflect.Map:
		encodeMap(buf, key, val)
	case reflect.Slice:
		if val.Type() == bytesType {
			EncodeBinary(buf, key, val.Interface().([]byte))
		} else {
			encodeSlice(buf, key, val)
		}
	case reflect.Ptr, reflect.Interface:
		if val.IsNil() {
			EncodePrefix(buf, Null, key)
		} else {
			encodeField(buf, key, val.Elem())
		}
	default:
		panic(NewBsonError("don't know how to marshal %v", val.Type()))
	}
}

// EncodeOptionalPrefix encodes the key as prefix if it's not empty.
// If it is empty, then it's a no-op, with the assumption that
// it's a top level object.
func EncodeOptionalPrefix(buf *bytes2.ChunkedWriter, etype byte, key string) {
	if key == "" {
		return
	}
	EncodePrefix(buf, etype, key)
}

// EncodePrefix encodes key as prefix for the next object or value.
func EncodePrefix(buf *bytes2.ChunkedWriter, etype byte, key string) {
	b := buf.Reserve(len(key) + 2)
	b[0] = etype
	copy(b[1:], key)
	b[len(b)-1] = 0
}

// EncodeString encodes a string.
func EncodeString(buf *bytes2.ChunkedWriter, key string, val string) {
	// Encode strings as binary; go strings are not necessarily unicode
	EncodePrefix(buf, Binary, key)
	putUint32(buf, uint32(len(val)))
	buf.WriteByte(0)
	buf.WriteString(val)
}

// EncodeBinary encodes a []byte as binary.
func EncodeBinary(buf *bytes2.ChunkedWriter, key string, val []byte) {
	EncodePrefix(buf, Binary, key)
	putUint32(buf, uint32(len(val)))
	buf.WriteByte(0)
	buf.Write(val)
}

// EncodeInt64 encodes an int64.
func EncodeInt64(buf *bytes2.ChunkedWriter, key string, val int64) {
	EncodePrefix(buf, Long, key)
	putUint64(buf, uint64(val))
}

// EncodeInt32 encodes an int32.
func EncodeInt32(buf *bytes2.ChunkedWriter, key string, val int32) {
	EncodePrefix(buf, Int, key)
	putUint32(buf, uint32(val))
}

// EncodeInt encodes an int.
func EncodeInt(buf *bytes2.ChunkedWriter, key string, val int) {
	EncodeInt64(buf, key, int64(val))
}

// EncodeUint64 encodes an uint64.
func EncodeUint64(buf *bytes2.ChunkedWriter, key string, val uint64) {
	EncodePrefix(buf, Ulong, key)
	putUint64(buf, val)
}

// EncodeUint32 encodes an uint32.
func EncodeUint32(buf *bytes2.ChunkedWriter, key string, val uint32) {
	EncodeUint64(buf, key, uint64(val))
}

// EncodeUint encodes an uint.
func EncodeUint(buf *bytes2.ChunkedWriter, key string, val uint) {
	EncodeUint64(buf, key, uint64(val))
}

// EncodeFloat64 encodes a float64.
func EncodeFloat64(buf *bytes2.ChunkedWriter, key string, val float64) {
	EncodePrefix(buf, Number, key)
	bits := math.Float64bits(val)
	putUint64(buf, bits)
}

// EncodeBool encodes a bool.
func EncodeBool(buf *bytes2.ChunkedWriter, key string, val bool) {
	EncodePrefix(buf, Boolean, key)
	if val {
		buf.WriteByte(1)
	} else {
		buf.WriteByte(0)
	}
}

// EncodeTime encodes a time.Time.
func EncodeTime(buf *bytes2.ChunkedWriter, key string, val time.Time) {
	EncodePrefix(buf, Datetime, key)
	mtime := val.UnixNano() / 1e6
	putUint64(buf, uint64(mtime))
}

func encodeStruct(buf *bytes2.ChunkedWriter, key string, val reflect.Value) {
	EncodePrefix(buf, Object, key)
	encodeStructContent(buf, val)
}

func encodeStructContent(buf *bytes2.ChunkedWriter, val reflect.Value) {
	lenWriter := NewLenWriter(buf)
	t := val.Type()
	for i := 0; i < t.NumField(); i++ {
		key := t.Field(i).Name

		// NOTE(szopa): Ignore private fields (copied from
		// encoding/json). Yes, it feels like a hack.
		if t.Field(i).PkgPath != "" {
			continue
		}
		encodeField(buf, key, val.Field(i))
	}
	lenWriter.Close()
}

func encodeMap(buf *bytes2.ChunkedWriter, key string, val reflect.Value) {
	EncodePrefix(buf, Object, key)
	encodeMapContent(buf, val)
}

// a map seems to lose the 'CanAddr' property. So if we want
// to use a custom marshaler with a struct pointer receiver, like:
//   func (ps *PrivateStruct) MarshalBson(buf *bytes2.ChunkedWriter, key string) {
// the map has to be using pointers, i.e:
//   map[string]*PrivateStruct
// and not:
//   map[string]PrivateStruct
// (see unit test)
func encodeMapContent(buf *bytes2.ChunkedWriter, val reflect.Value) {
	lenWriter := NewLenWriter(buf)
	mt := val.Type()
	if mt.Key().Kind() != reflect.String {
		panic(NewBsonError("can't marshall maps with non-string key types"))
	}
	keys := val.MapKeys()
	for _, k := range keys {
		key := k.String()
		encodeField(buf, key, val.MapIndex(k))
	}
	lenWriter.Close()
}

func encodeSlice(buf *bytes2.ChunkedWriter, key string, val reflect.Value) {
	EncodePrefix(buf, Array, key)
	encodeSliceContent(buf, val)
}

func encodeSliceContent(buf *bytes2.ChunkedWriter, val reflect.Value) {
	lenWriter := NewLenWriter(buf)
	for i := 0; i < val.Len(); i++ {
		encodeField(buf, Itoa(i), val.Index(i))
	}
	lenWriter.Close()
}

func putUint32(buf *bytes2.ChunkedWriter, val uint32) {
	Pack.PutUint32(buf.Reserve(WORD32), val)
}

func putUint64(buf *bytes2.ChunkedWriter, val uint64) {
	Pack.PutUint64(buf.Reserve(WORD64), val)
}

var intStrMap [intAliasSize + 1]string

const (
	intAliasSize = 1024
)

func init() {
	for i := 0; i <= intAliasSize; i++ {
		intStrMap[i] = strconv.Itoa(i)
	}
}

func Itoa(i int) string {
	if i <= intAliasSize {
		return intStrMap[i]
	}
	return strconv.Itoa(i)
}
<commit_msg>bson: reduce DefaultBufferSize from 16K to 1K<commit_after>// Copyright 2012, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package bson

import (
	"io"
	"math"
	"reflect"
	"strconv"
	"time"

	"github.com/youtube/vitess/go/bytes2"
)

// LenWriter records the current write position on the buffer
// and can later be used to record the number of bytes written
// in conformance to BSON spec
type LenWriter struct {
	buf *bytes2.ChunkedWriter
	off int
	b   []byte
}

// NewLenWriter returns a LenWriter that reserves the
// bytes buf so they can store the length later.
func NewLenWriter(buf *bytes2.ChunkedWriter) LenWriter {
	off := buf.Len()
	b := buf.Reserve(WORD32)
	return LenWriter{buf, off, b}
}

// Close closes the current object being encoded by
// writing bson's EOO byte and recording the length.
func (lw LenWriter) Close() {
	lw.buf.WriteByte(EOO)
	Pack.PutUint32(lw.b, uint32(lw.buf.Len()-lw.off))
}

// Marshaler is the interface that needs to be
// satisfied by types that want to implement a custom
// marshaler.
// When being invoked as a top level object, key will
// be "". In such cases, MarshalBson must not encode
// any prefix.
type Marshaler interface {
	MarshalBson(buf *bytes2.ChunkedWriter, key string)
}

func canMarshal(val reflect.Value) Marshaler {
	// Check the Marshaler interface on T.
	if marshaler, ok := val.Interface().(Marshaler); ok {
		// Don't call custom marshaler for nil values.
		switch val.Kind() {
		case reflect.Ptr, reflect.Interface, reflect.Map, reflect.Slice:
			if val.IsNil() {
				return nil
			}
		}
		return marshaler
	}
	// Check the Marshaler interface on *T.
	if val.CanAddr() {
		if marshaler, ok := val.Addr().Interface().(Marshaler); ok {
			return marshaler
		}
	}
	return nil
}

// DefaultBufferSize is the default allocation size for ChunkedWriter.
const DefaultBufferSize = 1024

// MarshalToStream marshals val into writer.
func MarshalToStream(writer io.Writer, val interface{}) (err error) {
	buf := bytes2.NewChunkedWriter(DefaultBufferSize)
	if err = MarshalToBuffer(buf, val); err != nil {
		return err
	}
	_, err = buf.WriteTo(writer)
	return err
}

// Marshal marshals val into encoded.
func Marshal(val interface{}) (encoded []byte, err error) {
	buf := bytes2.NewChunkedWriter(DefaultBufferSize)
	err = MarshalToBuffer(buf, val)
	return buf.Bytes(), err
}

// MarshalToBuffer marshals val into buf. This is the most efficient
// function to use, especially when marshaling large nested objects.
func MarshalToBuffer(buf *bytes2.ChunkedWriter, val interface{}) (err error) {
	defer handleError(&err)
	if val == nil {
		return NewBsonError("cannot marshal nil")
	}

	v := reflect.Indirect(reflect.ValueOf(val))
	if marshaler := canMarshal(v); marshaler != nil {
		marshaler.MarshalBson(buf, "")
		return
	}

	switch v.Kind() {
	case reflect.String,
		reflect.Int64, reflect.Int32, reflect.Int,
		reflect.Uint64, reflect.Uint32, reflect.Uint,
		reflect.Float64, reflect.Bool:
		EncodeSimple(buf, v.Interface())
	case reflect.Struct:
		if v.Type() == timeType {
			EncodeSimple(buf, v.Interface())
		} else {
			encodeStructContent(buf, v)
		}
	case reflect.Map:
		encodeMapContent(buf, v)
	case reflect.Slice, reflect.Array:
		if v.Type() == bytesType {
			EncodeSimple(buf, v.Interface())
		} else {
			encodeSliceContent(buf, v)
		}
	default:
		return NewBsonError("unexpected type %v", v.Type())
	}
	return nil
}

// EncodeSimple marshals simple objects that cannot be
// encoded as a top level bson document.
func EncodeSimple(buf *bytes2.ChunkedWriter, val interface{}) {
	lenWriter := NewLenWriter(buf)
	EncodeField(buf, MAGICTAG, val)
	lenWriter.Close()
}

// EncodeField encodes val using the supplied key as embedded tag.
// Unlike EncodeInterface, EncodeField can handle complex objects
// like structs, pointers, etc. But it is slower.
func EncodeField(buf *bytes2.ChunkedWriter, key string, val interface{}) {
	encodeField(buf, key, reflect.ValueOf(val))
}

func encodeField(buf *bytes2.ChunkedWriter, key string, val reflect.Value) {
	// nil interfaces show up as invalid
	if !val.IsValid() {
		EncodePrefix(buf, Null, key)
		return
	}
	if marshaler := canMarshal(val); marshaler != nil {
		marshaler.MarshalBson(buf, key)
		return
	}

	switch val.Kind() {
	case reflect.String:
		EncodeString(buf, key, val.String())
	case reflect.Int64:
		EncodeInt64(buf, key, val.Int())
	case reflect.Int32:
		EncodeInt32(buf, key, int32(val.Int()))
	case reflect.Int:
		EncodeInt(buf, key, int(val.Int()))
	case reflect.Uint64:
		EncodeUint64(buf, key, uint64(val.Uint()))
	case reflect.Uint32:
		EncodeUint32(buf, key, uint32(val.Uint()))
	case reflect.Uint:
		EncodeUint(buf, key, uint(val.Uint()))
	case reflect.Float64:
		EncodeFloat64(buf, key, val.Float())
	case reflect.Bool:
		EncodeBool(buf, key, val.Bool())
	case reflect.Struct:
		if val.Type() == timeType {
			EncodeTime(buf, key, val.Interface().(time.Time))
		} else {
			encodeStruct(buf, key, val)
		}
	case reflect.Map:
		encodeMap(buf, key, val)
	case reflect.Slice:
		if val.Type() == bytesType {
			EncodeBinary(buf, key, val.Interface().([]byte))
		} else {
			encodeSlice(buf, key, val)
		}
	case reflect.Ptr, reflect.Interface:
		if val.IsNil() {
			EncodePrefix(buf, Null, key)
		} else {
			encodeField(buf, key, val.Elem())
		}
	default:
		panic(NewBsonError("don't know how to marshal %v", val.Type()))
	}
}

// EncodeOptionalPrefix encodes the key as prefix if it's not empty.
// If it is empty, then it's a no-op, with the assumption that
// it's a top level object.
func EncodeOptionalPrefix(buf *bytes2.ChunkedWriter, etype byte, key string) {
	if key == "" {
		return
	}
	EncodePrefix(buf, etype, key)
}

// EncodePrefix encodes key as prefix for the next object or value.
func EncodePrefix(buf *bytes2.ChunkedWriter, etype byte, key string) {
	b := buf.Reserve(len(key) + 2)
	b[0] = etype
	copy(b[1:], key)
	b[len(b)-1] = 0
}

// EncodeString encodes a string.
func EncodeString(buf *bytes2.ChunkedWriter, key string, val string) {
	// Encode strings as binary; go strings are not necessarily unicode
	EncodePrefix(buf, Binary, key)
	putUint32(buf, uint32(len(val)))
	buf.WriteByte(0)
	buf.WriteString(val)
}

// EncodeBinary encodes a []byte as binary.
func EncodeBinary(buf *bytes2.ChunkedWriter, key string, val []byte) {
	EncodePrefix(buf, Binary, key)
	putUint32(buf, uint32(len(val)))
	buf.WriteByte(0)
	buf.Write(val)
}

// EncodeInt64 encodes an int64.
func EncodeInt64(buf *bytes2.ChunkedWriter, key string, val int64) {
	EncodePrefix(buf, Long, key)
	putUint64(buf, uint64(val))
}

// EncodeInt32 encodes an int32.
func EncodeInt32(buf *bytes2.ChunkedWriter, key string, val int32) {
	EncodePrefix(buf, Int, key)
	putUint32(buf, uint32(val))
}

// EncodeInt encodes an int.
func EncodeInt(buf *bytes2.ChunkedWriter, key string, val int) {
	EncodeInt64(buf, key, int64(val))
}

// EncodeUint64 encodes an uint64.
func EncodeUint64(buf *bytes2.ChunkedWriter, key string, val uint64) {
	EncodePrefix(buf, Ulong, key)
	putUint64(buf, val)
}

// EncodeUint32 encodes an uint32.
func EncodeUint32(buf *bytes2.ChunkedWriter, key string, val uint32) {
	EncodeUint64(buf, key, uint64(val))
}

// EncodeUint encodes an uint.
func EncodeUint(buf *bytes2.ChunkedWriter, key string, val uint) {
	EncodeUint64(buf, key, uint64(val))
}

// EncodeFloat64 encodes a float64.
func EncodeFloat64(buf *bytes2.ChunkedWriter, key string, val float64) {
	EncodePrefix(buf, Number, key)
	bits := math.Float64bits(val)
	putUint64(buf, bits)
}

// EncodeBool encodes a bool.
func EncodeBool(buf *bytes2.ChunkedWriter, key string, val bool) {
	EncodePrefix(buf, Boolean, key)
	if val {
		buf.WriteByte(1)
	} else {
		buf.WriteByte(0)
	}
}

// EncodeTime encodes a time.Time.
func EncodeTime(buf *bytes2.ChunkedWriter, key string, val time.Time) {
	EncodePrefix(buf, Datetime, key)
	mtime := val.UnixNano() / 1e6
	putUint64(buf, uint64(mtime))
}

func encodeStruct(buf *bytes2.ChunkedWriter, key string, val reflect.Value) {
	EncodePrefix(buf, Object, key)
	encodeStructContent(buf, val)
}

func encodeStructContent(buf *bytes2.ChunkedWriter, val reflect.Value) {
	lenWriter := NewLenWriter(buf)
	t := val.Type()
	for i := 0; i < t.NumField(); i++ {
		key := t.Field(i).Name

		// NOTE(szopa): Ignore private fields (copied from
		// encoding/json). Yes, it feels like a hack.
		if t.Field(i).PkgPath != "" {
			continue
		}
		encodeField(buf, key, val.Field(i))
	}
	lenWriter.Close()
}

func encodeMap(buf *bytes2.ChunkedWriter, key string, val reflect.Value) {
	EncodePrefix(buf, Object, key)
	encodeMapContent(buf, val)
}

// a map seems to lose the 'CanAddr' property. So if we want
// to use a custom marshaler with a struct pointer receiver, like:
//   func (ps *PrivateStruct) MarshalBson(buf *bytes2.ChunkedWriter, key string) {
// the map has to be using pointers, i.e:
//   map[string]*PrivateStruct
// and not:
//   map[string]PrivateStruct
// (see unit test)
func encodeMapContent(buf *bytes2.ChunkedWriter, val reflect.Value) {
	lenWriter := NewLenWriter(buf)
	mt := val.Type()
	if mt.Key().Kind() != reflect.String {
		panic(NewBsonError("can't marshall maps with non-string key types"))
	}
	keys := val.MapKeys()
	for _, k := range keys {
		key := k.String()
		encodeField(buf, key, val.MapIndex(k))
	}
	lenWriter.Close()
}

func encodeSlice(buf *bytes2.ChunkedWriter, key string, val reflect.Value) {
	EncodePrefix(buf, Array, key)
	encodeSliceContent(buf, val)
}

func encodeSliceContent(buf *bytes2.ChunkedWriter, val reflect.Value) {
	lenWriter := NewLenWriter(buf)
	for i := 0; i < val.Len(); i++ {
		encodeField(buf, Itoa(i), val.Index(i))
	}
	lenWriter.Close()
}

func putUint32(buf *bytes2.ChunkedWriter, val uint32) {
	Pack.PutUint32(buf.Reserve(WORD32), val)
}

func putUint64(buf *bytes2.ChunkedWriter, val uint64) {
	Pack.PutUint64(buf.Reserve(WORD64), val)
}

var intStrMap [intAliasSize + 1]string

const (
	intAliasSize = 1024
)

func init() {
	for i := 0; i <= intAliasSize; i++ {
		intStrMap[i] = strconv.Itoa(i)
	}
}

func Itoa(i int) string {
	if i <= intAliasSize {
		return intStrMap[i]
	}
	return strconv.Itoa(i)
}
<|endoftext|>


<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package engine

import (
	libkb "github.com/keybase/client/go/libkb"
	keybase1 "github.com/keybase/client/go/protocol"
	jsonw "github.com/keybase/go-jsonw"
	"golang.org/x/net/context"
)

// Prove is an engine used for proving ownership of remote accounts,
// like Twitter, GitHub, etc.
type Prove struct {
	arg        *keybase1.StartProofArg
	me         *libkb.User
	st         libkb.ServiceType
	supersede  bool
	proof      *jsonw.Wrapper
	sig        string
	sigID      keybase1.SigID
	postRes    *libkb.PostProofRes
	signingKey libkb.GenericKey

	remoteNameNormalized string

	libkb.Contextified
}

// NewProve makes a new Prove Engine given an RPC-friendly ProveArg.
func NewProve(arg *keybase1.StartProofArg, g *libkb.GlobalContext) *Prove {
	return &Prove{
		arg:          arg,
		Contextified: libkb.NewContextified(g),
	}
}

// Name provides the name of this engine for the engine interface contract
func (p *Prove) Name() string {
	return "Prove"
}

// GetPrereqs returns the engine prereqs.
func (p *Prove) Prereqs() Prereqs {
	return Prereqs{Device: true}
}

// RequiredUIs returns the required UIs.
func (p *Prove) RequiredUIs() []libkb.UIKind {
	return []libkb.UIKind{
		libkb.LogUIKind,
		libkb.ProveUIKind,
		libkb.SecretUIKind,
	}
}

// SubConsumers returns the other UI consumers for this engine.
func (p *Prove) SubConsumers() []libkb.UIConsumer {
	return nil
}

func (p *Prove) loadMe() (err error) {
	p.me, err = libkb.LoadMe(libkb.NewLoadUserForceArg(p.G()))
	return
}

func (p *Prove) checkExists1(ctx *Context) (err error) {
	proofs := p.me.IDTable().GetActiveProofsFor(p.st)
	if len(proofs) != 0 && !p.arg.Force && p.st.LastWriterWins() {
		lst := proofs[len(proofs)-1]
		var redo bool
		redo, err = ctx.ProveUI.PromptOverwrite(context.TODO(), keybase1.PromptOverwriteArg{
			Account: lst.ToDisplayString(),
			Typ:     keybase1.PromptOverwriteType_SOCIAL,
		})
		if err != nil {
			return err
		}
		if !redo {
			return libkb.NotConfirmedError{}
		}
		p.supersede = true
	}
	return
}

func (p *Prove) promptRemoteName(ctx *Context) (err error) {
	remoteName := p.arg.Username
	if len(remoteName) == 0 {
		var prevErr error
		for len(remoteName) == 0 && err == nil {
			var un string
			un, err = ctx.ProveUI.PromptUsername(context.TODO(), keybase1.PromptUsernameArg{
				Prompt:    p.st.GetPrompt(),
				PrevError: libkb.ExportErrorAsStatus(prevErr),
			})
			if err == nil {
				remoteNameNormalized, prevErr := p.st.NormalizeRemoteName(un)
				if prevErr == nil {
					p.remoteNameNormalized = remoteNameNormalized
				}
			}
		}
	} else {
		remoteNameNormalized, err := p.st.NormalizeRemoteName(remoteName)
		if err == nil {
			p.remoteNameNormalized = remoteNameNormalized
		}
	}
	return
}

func (p *Prove) checkExists2(ctx *Context) (err error) {
	p.G().Log.Debug("+ CheckExists2")
	defer func() { p.G().Log.Debug("- CheckExists2 -> %s", libkb.ErrToOk(err)) }()
	if !p.st.LastWriterWins() {
		var found libkb.RemoteProofChainLink
		for _, proof := range p.me.IDTable().GetActiveProofsFor(p.st) {
			_, name := proof.ToKeyValuePair()
			if libkb.Cicmp(name, p.remoteNameNormalized) {
				found = proof
				break
			}
		}
		if found != nil {
			var redo bool
			redo, err = ctx.ProveUI.PromptOverwrite(context.TODO(), keybase1.PromptOverwriteArg{
				Account: found.ToDisplayString(),
				Typ:     keybase1.PromptOverwriteType_SITE,
			})
			if err != nil {
				return
			}
			if !redo {
				err = libkb.NotConfirmedError{}
				return
			}
			p.supersede = true
		}
	}
	return
}

func (p *Prove) doPrechecks(ctx *Context) (err error) {
	var w *libkb.Markup
	w, err = p.st.PreProofCheck(p.remoteNameNormalized)
	if w != nil {
		if uierr := ctx.ProveUI.OutputPrechecks(context.TODO(), keybase1.OutputPrechecksArg{Text: w.Export()}); uierr != nil {
			p.G().Log.Warning("prove ui OutputPrechecks call error: %s", uierr)
		}
	}
	return
}

func (p *Prove) doWarnings(ctx *Context) (err error) {
	if mu := p.st.PreProofWarning(p.remoteNameNormalized); mu != nil {
		var ok bool
		arg := keybase1.PreProofWarningArg{Text: mu.Export()}
		if ok, err = ctx.ProveUI.PreProofWarning(context.TODO(), arg); err == nil && !ok {
			err = libkb.NotConfirmedError{}
		}
	}
	return
}

func (p *Prove) generateProof(ctx *Context) (err error) {
	ska := libkb.SecretKeyArg{
		Me:      p.me,
		KeyType: libkb.DeviceSigningKeyType,
	}
	seckey, locked, err := p.G().Keyrings.GetSecretKeyAndSKBWithPrompt(ctx.SecretKeyPromptArg(ska, "proof signature"))
	if err != nil {
		return
	}

	if p.signingKey, err = locked.GetPubKey(); err != nil {
		return
	}
	if p.proof, err = p.me.ServiceProof(p.signingKey, p.st, p.remoteNameNormalized); err != nil {
		return
	}
	if p.sig, p.sigID, _, err = libkb.SignJSON(p.proof, seckey); err != nil {
		return
	}
	return
}

func (p *Prove) postProofToServer() (err error) {
	arg := libkb.PostProofArg{
		Sig:            p.sig,
		ProofType:      p.st.GetProofType(),
		ID:             p.sigID,
		Supersede:      p.supersede,
		RemoteUsername: p.remoteNameNormalized,
		RemoteKey:      p.st.GetAPIArgKey(),
		SigningKey:     p.signingKey,
	}
	p.postRes, err = libkb.PostProof(arg)
	return
}

func (p *Prove) instructAction(ctx *Context) (err error) {
	mkp := p.st.PostInstructions(p.remoteNameNormalized)
	var txt string
	if txt, err = p.st.FormatProofText(p.postRes); err != nil {
		return
	}
	err = ctx.ProveUI.OutputInstructions(context.TODO(), keybase1.OutputInstructionsArg{
		Instructions: mkp.Export(),
		Proof:        txt,
	})
	if err != nil {
		return err
	}

	return p.checkAutoPost(ctx, txt)
}

func (p *Prove) checkAutoPost(ctx *Context, txt string) error {
	if !p.arg.Auto {
		return nil
	}
	if libkb.RemoteServiceTypes[p.arg.Service] != keybase1.ProofType_ROOTER {
		return nil
	}
	p.G().Log.Debug("making automatic post of proof to rooter")
	apiArg := libkb.APIArg{
		Endpoint:    "rooter",
		NeedSession: true,
		Args: libkb.HTTPArgs{
			"post":     libkb.S{Val: txt},
			"username": libkb.S{Val: p.arg.Username},
		},
		Contextified: libkb.NewContextified(p.G()),
	}
	_, err := p.G().API.Post(apiArg)
	if err != nil {
		p.G().Log.Debug("error posting to rooter: %s", err)
		return err
	}
	return nil
}

func (p *Prove) promptPostedLoop(ctx *Context) (err error) {
	found := false
	for i := 0; ; i++ {
		var retry bool
		var status keybase1.ProofStatus
		var warn *libkb.Markup
		retry, err = ctx.ProveUI.OkToCheck(context.TODO(), keybase1.OkToCheckArg{
			Name:    p.st.DisplayName(p.remoteNameNormalized),
			Attempt: i,
		})
		if !retry || err != nil {
			break
		}
		found, status, err = libkb.CheckPosted(p.postRes.ID)
		if found || err != nil {
			break
		}
		warn, err = p.st.RecheckProofPosting(i, status, p.remoteNameNormalized)
		if warn != nil {
			uierr := ctx.ProveUI.DisplayRecheckWarning(context.TODO(), keybase1.DisplayRecheckWarningArg{
				Text: warn.Export(),
			})
			if uierr != nil {
				p.G().Log.Warning("prove ui DisplayRecheckWarning call error: %s", uierr)
			}
		}
		if err != nil {
			break
		}
	}
	if !found && err == nil {
		err = libkb.ProofNotYetAvailableError{}
	}

	return
}

func (p *Prove) checkProofText() error {
	p.G().Log.Debug("p.postRes.Text: %q", p.postRes.Text)
	p.G().Log.Debug("p.sigID: %q", p.sigID)
	return p.st.CheckProofText(p.postRes.Text, p.sigID, p.sig)
}

func (p *Prove) getServiceType() (err error) {
	if p.st = libkb.GetServiceType(p.arg.Service); p.st == nil {
		err = libkb.BadServiceError{Service: p.arg.Service}
	}
	return
}

// SigID returns the signature id of the proof posted to the
// server.
func (p *Prove) SigID() keybase1.SigID {
	return p.sigID
}

// Run runs the Prove engine, performing all steps of the proof process.
func (p *Prove) Run(ctx *Context) (err error) {
	p.G().Log.Debug("+ ProofEngine.Run")
	defer func() {
		p.G().Log.Debug("- ProofEngine.Run -> %s", libkb.ErrToOk(err))
	}()

	stage := func(s string) {
		p.G().Log.Debug("| ProofEngine.Run() %s", s)
	}

	stage("GetServiceType")
	if err = p.getServiceType(); err != nil {
		return
	}
	stage("LoadMe")
	if err = p.loadMe(); err != nil {
		return
	}
	stage("CheckExists1")
	if err = p.checkExists1(ctx); err != nil {
		return
	}
	stage("PromptRemoteName")
	if err = p.promptRemoteName(ctx); err != nil {
		return
	}
	stage("CheckExists2")
	if err = p.checkExists2(ctx); err != nil {
		return
	}
	stage("DoPrechecks")
	if err = p.doPrechecks(ctx); err != nil {
		return
	}
	stage("DoWarnings")
	if err = p.doWarnings(ctx); err != nil {
		return
	}
	stage("GenerateProof")
	if err = p.generateProof(ctx); err != nil {
		return
	}
	stage("PostProofToServer")
	if err = p.postProofToServer(); err != nil {
		return
	}
	stage("CheckProofText")
	if err = p.checkProofText(); err != nil {
		return
	}
	stage("InstructAction")
	if err = p.instructAction(ctx); err != nil {
		return
	}

	if !p.arg.PromptPosted {
		p.G().Log.Debug("PromptPosted not set, prove run finished")
		return
	}

	stage("PromptPostedLoop")
	if err = p.promptPostedLoop(ctx); err != nil {
		return
	}
	ctx.LogUI.Notice("Success!")
	return nil
}
<commit_msg>fix an infinite loop in promptRemoteName<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package engine

import (
	libkb "github.com/keybase/client/go/libkb"
	keybase1 "github.com/keybase/client/go/protocol"
	jsonw "github.com/keybase/go-jsonw"
	"golang.org/x/net/context"
)

// Prove is an engine used for proving ownership of remote accounts,
// like Twitter, GitHub, etc.
type Prove struct {
	arg        *keybase1.StartProofArg
	me         *libkb.User
	st         libkb.ServiceType
	supersede  bool
	proof      *jsonw.Wrapper
	sig        string
	sigID      keybase1.SigID
	postRes    *libkb.PostProofRes
	signingKey libkb.GenericKey

	remoteNameNormalized string

	libkb.Contextified
}

// NewProve makes a new Prove Engine given an RPC-friendly ProveArg.
func NewProve(arg *keybase1.StartProofArg, g *libkb.GlobalContext) *Prove {
	return &Prove{
		arg:          arg,
		Contextified: libkb.NewContextified(g),
	}
}

// Name provides the name of this engine for the engine interface contract
func (p *Prove) Name() string {
	return "Prove"
}

// GetPrereqs returns the engine prereqs.
func (p *Prove) Prereqs() Prereqs {
	return Prereqs{Device: true}
}

// RequiredUIs returns the required UIs.
func (p *Prove) RequiredUIs() []libkb.UIKind {
	return []libkb.UIKind{
		libkb.LogUIKind,
		libkb.ProveUIKind,
		libkb.SecretUIKind,
	}
}

// SubConsumers returns the other UI consumers for this engine.
func (p *Prove) SubConsumers() []libkb.UIConsumer {
	return nil
}

func (p *Prove) loadMe() (err error) {
	p.me, err = libkb.LoadMe(libkb.NewLoadUserForceArg(p.G()))
	return
}

func (p *Prove) checkExists1(ctx *Context) (err error) {
	proofs := p.me.IDTable().GetActiveProofsFor(p.st)
	if len(proofs) != 0 && !p.arg.Force && p.st.LastWriterWins() {
		lst := proofs[len(proofs)-1]
		var redo bool
		redo, err = ctx.ProveUI.PromptOverwrite(context.TODO(), keybase1.PromptOverwriteArg{
			Account: lst.ToDisplayString(),
			Typ:     keybase1.PromptOverwriteType_SOCIAL,
		})
		if err != nil {
			return err
		}
		if !redo {
			return libkb.NotConfirmedError{}
		}
		p.supersede = true
	}
	return
}

func (p *Prove) promptRemoteName(ctx *Context) error {
	// If the name is already supplied, there's no need to prompt.
	if len(p.arg.Username) > 0 {
		remoteNameNormalized, err := p.st.NormalizeRemoteName(p.arg.Username)
		if err == nil {
			p.remoteNameNormalized = remoteNameNormalized
		}
		return err
	}

	// Prompt for the name, retrying if it's invalid.
	var normalizationError error
	for {
		un, err := ctx.ProveUI.PromptUsername(context.TODO(), keybase1.PromptUsernameArg{
			Prompt:    p.st.GetPrompt(),
			PrevError: libkb.ExportErrorAsStatus(normalizationError),
		})
		if err != nil {
			// Errors here are conditions like EOF. Return them rather than retrying.
			return err
		}
		var remoteNameNormalized string
		remoteNameNormalized, normalizationError = p.st.NormalizeRemoteName(un)
		if normalizationError == nil {
			p.remoteNameNormalized = remoteNameNormalized
			return nil
		}
	}
}

func (p *Prove) checkExists2(ctx *Context) (err error) {
	p.G().Log.Debug("+ CheckExists2")
	defer func() { p.G().Log.Debug("- CheckExists2 -> %s", libkb.ErrToOk(err)) }()
	if !p.st.LastWriterWins() {
		var found libkb.RemoteProofChainLink
		for _, proof := range p.me.IDTable().GetActiveProofsFor(p.st) {
			_, name := proof.ToKeyValuePair()
			if libkb.Cicmp(name, p.remoteNameNormalized) {
				found = proof
				break
			}
		}
		if found != nil {
			var redo bool
			redo, err = ctx.ProveUI.PromptOverwrite(context.TODO(), keybase1.PromptOverwriteArg{
				Account: found.ToDisplayString(),
				Typ:     keybase1.PromptOverwriteType_SITE,
			})
			if err != nil {
				return
			}
			if !redo {
				err = libkb.NotConfirmedError{}
				return
			}
			p.supersede = true
		}
	}
	return
}

func (p *Prove) doPrechecks(ctx *Context) (err error) {
	var w *libkb.Markup
	w, err = p.st.PreProofCheck(p.remoteNameNormalized)
	if w != nil {
		if uierr := ctx.ProveUI.OutputPrechecks(context.TODO(), keybase1.OutputPrechecksArg{Text: w.Export()}); uierr != nil {
			p.G().Log.Warning("prove ui OutputPrechecks call error: %s", uierr)
		}
	}
	return
}

func (p *Prove) doWarnings(ctx *Context) (err error) {
	if mu := p.st.PreProofWarning(p.remoteNameNormalized); mu != nil {
		var ok bool
		arg := keybase1.PreProofWarningArg{Text: mu.Export()}
		if ok, err = ctx.ProveUI.PreProofWarning(context.TODO(), arg); err == nil && !ok {
			err = libkb.NotConfirmedError{}
		}
	}
	return
}

func (p *Prove) generateProof(ctx *Context) (err error) {
	ska := libkb.SecretKeyArg{
		Me:      p.me,
		KeyType: libkb.DeviceSigningKeyType,
	}
	seckey, locked, err := p.G().Keyrings.GetSecretKeyAndSKBWithPrompt(ctx.SecretKeyPromptArg(ska, "proof signature"))
	if err != nil {
		return
	}

	if p.signingKey, err = locked.GetPubKey(); err != nil {
		return
	}
	if p.proof, err = p.me.ServiceProof(p.signingKey, p.st, p.remoteNameNormalized); err != nil {
		return
	}
	if p.sig, p.sigID, _, err = libkb.SignJSON(p.proof, seckey); err != nil {
		return
	}
	return
}

func (p *Prove) postProofToServer() (err error) {
	arg := libkb.PostProofArg{
		Sig:            p.sig,
		ProofType:      p.st.GetProofType(),
		ID:             p.sigID,
		Supersede:      p.supersede,
		RemoteUsername: p.remoteNameNormalized,
		RemoteKey:      p.st.GetAPIArgKey(),
		SigningKey:     p.signingKey,
	}
	p.postRes, err = libkb.PostProof(arg)
	return
}

func (p *Prove) instructAction(ctx *Context) (err error) {
	mkp := p.st.PostInstructions(p.remoteNameNormalized)
	var txt string
	if txt, err = p.st.FormatProofText(p.postRes); err != nil {
		return
	}
	err = ctx.ProveUI.OutputInstructions(context.TODO(), keybase1.OutputInstructionsArg{
		Instructions: mkp.Export(),
		Proof:        txt,
	})
	if err != nil {
		return err
	}

	return p.checkAutoPost(ctx, txt)
}

func (p *Prove) checkAutoPost(ctx *Context, txt string) error {
	if !p.arg.Auto {
		return nil
	}
	if libkb.RemoteServiceTypes[p.arg.Service] != keybase1.ProofType_ROOTER {
		return nil
	}
	p.G().Log.Debug("making automatic post of proof to rooter")
	apiArg := libkb.APIArg{
		Endpoint:    "rooter",
		NeedSession: true,
		Args: libkb.HTTPArgs{
			"post":     libkb.S{Val: txt},
			"username": libkb.S{Val: p.arg.Username},
		},
		Contextified: libkb.NewContextified(p.G()),
	}
	_, err := p.G().API.Post(apiArg)
	if err != nil {
		p.G().Log.Debug("error posting to rooter: %s", err)
		return err
	}
	return nil
}

func (p *Prove) promptPostedLoop(ctx *Context) (err error) {
	found := false
	for i := 0; ; i++ {
		var retry bool
		var status keybase1.ProofStatus
		var warn *libkb.Markup
		retry, err = ctx.ProveUI.OkToCheck(context.TODO(), keybase1.OkToCheckArg{
			Name:    p.st.DisplayName(p.remoteNameNormalized),
			Attempt: i,
		})
		if !retry || err != nil {
			break
		}
		found, status, err = libkb.CheckPosted(p.postRes.ID)
		if found || err != nil {
			break
		}
		warn, err = p.st.RecheckProofPosting(i, status, p.remoteNameNormalized)
		if warn != nil {
			uierr := ctx.ProveUI.DisplayRecheckWarning(context.TODO(), keybase1.DisplayRecheckWarningArg{
				Text: warn.Export(),
			})
			if uierr != nil {
				p.G().Log.Warning("prove ui DisplayRecheckWarning call error: %s", uierr)
			}
		}
		if err != nil {
			break
		}
	}
	if !found && err == nil {
		err = libkb.ProofNotYetAvailableError{}
	}

	return
}

func (p *Prove) checkProofText() error {
	p.G().Log.Debug("p.postRes.Text: %q", p.postRes.Text)
	p.G().Log.Debug("p.sigID: %q", p.sigID)
	return p.st.CheckProofText(p.postRes.Text, p.sigID, p.sig)
}

func (p *Prove) getServiceType() (err error) {
	if p.st = libkb.GetServiceType(p.arg.Service); p.st == nil {
		err = libkb.BadServiceError{Service: p.arg.Service}
	}
	return
}

// SigID returns the signature id of the proof posted to the
// server.
func (p *Prove) SigID() keybase1.SigID {
	return p.sigID
}

// Run runs the Prove engine, performing all steps of the proof process.
func (p *Prove) Run(ctx *Context) (err error) {
	p.G().Log.Debug("+ ProofEngine.Run")
	defer func() {
		p.G().Log.Debug("- ProofEngine.Run -> %s", libkb.ErrToOk(err))
	}()

	stage := func(s string) {
		p.G().Log.Debug("| ProofEngine.Run() %s", s)
	}

	stage("GetServiceType")
	if err = p.getServiceType(); err != nil {
		return
	}
	stage("LoadMe")
	if err = p.loadMe(); err != nil {
		return
	}
	stage("CheckExists1")
	if err = p.checkExists1(ctx); err != nil {
		return
	}
	stage("PromptRemoteName")
	if err = p.promptRemoteName(ctx); err != nil {
		return
	}
	stage("CheckExists2")
	if err = p.checkExists2(ctx); err != nil {
		return
	}
	stage("DoPrechecks")
	if err = p.doPrechecks(ctx); err != nil {
		return
	}
	stage("DoWarnings")
	if err = p.doWarnings(ctx); err != nil {
		return
	}
	stage("GenerateProof")
	if err = p.generateProof(ctx); err != nil {
		return
	}
	stage("PostProofToServer")
	if err = p.postProofToServer(); err != nil {
		return
	}
	stage("CheckProofText")
	if err = p.checkProofText(); err != nil {
		return
	}
	stage("InstructAction")
	if err = p.instructAction(ctx); err != nil {
		return
	}

	if !p.arg.PromptPosted {
		p.G().Log.Debug("PromptPosted not set, prove run finished")
		return
	}

	stage("PromptPostedLoop")
	if err = p.promptPostedLoop(ctx); err != nil {
		return
	}
	ctx.LogUI.Notice("Success!")
	return nil
}
<|endoftext|>


<commit_before>// Utils for modifying IAM policies for resources across GCP
package google

import (
	"encoding/json"
	"fmt"
	"log"
	"reflect"
	"time"

	"github.com/hashicorp/errwrap"
	"github.com/hashicorp/terraform/helper/schema"
	"google.golang.org/api/cloudresourcemanager/v1"
)

const maxBackoffSeconds = 30

// These types are implemented per GCP resource type and specify how to do per-resource IAM operations.
// They are used in the generic Terraform IAM resource definitions
// (e.g. _member/_binding/_policy/_audit_config)
type (
	// The ResourceIamUpdater interface is implemented for each GCP resource supporting IAM policy.
	// Implementations should be created per resource and should keep track of the resource identifier.
	ResourceIamUpdater interface {
		// Fetch the existing IAM policy attached to a resource.
		GetResourceIamPolicy() (*cloudresourcemanager.Policy, error)

		// Replaces the existing IAM Policy attached to a resource.
		SetResourceIamPolicy(policy *cloudresourcemanager.Policy) error

		// A mutex guards against concurrent to call to the SetResourceIamPolicy method.
		// The mutex key should be made of the resource type and resource id.
		// For example: `iam-project-{id}`.
		GetMutexKey() string

		// Returns the unique resource identifier.
		GetResourceId() string

		// Textual description of this resource to be used in error message.
		// The description should include the unique resource identifier.
		DescribeResource() string
	}

	// Factory for generating ResourceIamUpdater for given ResourceData resource
	newResourceIamUpdaterFunc func(d *schema.ResourceData, config *Config) (ResourceIamUpdater, error)

	// Describes how to modify a policy for a given Terraform IAM (_policy/_member/_binding/_audit_config) resource
	iamPolicyModifyFunc func(p *cloudresourcemanager.Policy) error

	// Parser for Terraform resource identifier (d.Id) for resource whose IAM policy is being changed
	resourceIdParserFunc func(d *schema.ResourceData, config *Config) error
)

// Locking wrapper around read-only operation with retries.
func iamPolicyReadWithRetry(updater ResourceIamUpdater) (*cloudresourcemanager.Policy, error) {
	mutexKey := updater.GetMutexKey()
	mutexKV.Lock(mutexKey)
	defer mutexKV.Unlock(mutexKey)

	log.Printf("[DEBUG] Retrieving policy for %s\n", updater.DescribeResource())
	var policy *cloudresourcemanager.Policy
	err := retryTime(func() (perr error) {
		policy, perr = updater.GetResourceIamPolicy()
		return perr
	}, 10)
	if err != nil {
		return nil, err
	}
	log.Printf("[DEBUG] Retrieved policy for %s: %+v\n", updater.DescribeResource(), policy)
	return policy, nil
}

// Locking wrapper around read-modify-write cycle for IAM policy.
func iamPolicyReadModifyWrite(updater ResourceIamUpdater, modify iamPolicyModifyFunc) error {
	mutexKey := updater.GetMutexKey()
	mutexKV.Lock(mutexKey)
	defer mutexKV.Unlock(mutexKey)

	backoff := time.Second
	for {
		log.Printf("[DEBUG]: Retrieving policy for %s\n", updater.DescribeResource())
		p, err := updater.GetResourceIamPolicy()
		if isGoogleApiErrorWithCode(err, 429) {
			log.Printf("[DEBUG] 429 while attempting to read policy for %s, waiting %v before attempting again", updater.DescribeResource(), backoff)
			time.Sleep(backoff)
			continue
		} else if err != nil {
			return err
		}
		log.Printf("[DEBUG]: Retrieved policy for %s: %+v\n", updater.DescribeResource(), p)

		err = modify(p)
		if err != nil {
			return err
		}

		log.Printf("[DEBUG]: Setting policy for %s to %+v\n", updater.DescribeResource(), p)
		err = updater.SetResourceIamPolicy(p)
		if err == nil {
			fetchBackoff := 1 * time.Second
			for successfulFetches := 0; successfulFetches < 3; {
				if fetchBackoff > maxBackoffSeconds*time.Second {
					return fmt.Errorf("Error applying IAM policy to %s: Waited too long for propagation.\n", updater.DescribeResource())
				}
				time.Sleep(fetchBackoff)
				log.Printf("[DEBUG]: Retrieving policy for %s\n", updater.DescribeResource())
				new_p, err := updater.GetResourceIamPolicy()
				if err != nil {
					// Quota for Read is pretty limited, so watch out for running out of quota.
					if isGoogleApiErrorWithCode(err, 429) {
						fetchBackoff = fetchBackoff * 2
					} else {
						return err
					}
				}
				log.Printf("[DEBUG]: Retrieved policy for %s: %+v\n", updater.DescribeResource(), p)
				if new_p == nil {
					// https://github.com/terraform-providers/terraform-provider-google/issues/2625
					fetchBackoff = fetchBackoff * 2
					continue
				}
				modified_p := new_p
				// This relies on the fact that `modify` is idempotent: since other changes might have
				// happened between the call to set the policy and now, we just need to make sure that
				// our change has been made.  'modify(p) == p' is our check for whether this has been
				// correctly applied.
				err = modify(modified_p)
				if err != nil {
					return err
				}
				if modified_p == new_p {
					successfulFetches += 1
				} else {
					fetchBackoff = fetchBackoff * 2
				}
			}
			break
		}
		if isConflictError(err) {
			log.Printf("[DEBUG]: Concurrent policy changes, restarting read-modify-write after %s\n", backoff)
			time.Sleep(backoff)
			backoff = backoff * 2
			if backoff > 30*time.Second {
				return errwrap.Wrapf(fmt.Sprintf("Error applying IAM policy to %s: Too many conflicts.  Latest error: {{err}}", updater.DescribeResource()), err)
			}
			continue
		}
		return errwrap.Wrapf(fmt.Sprintf("Error applying IAM policy for %s: {{err}}", updater.DescribeResource()), err)
	}
	log.Printf("[DEBUG]: Set policy for %s", updater.DescribeResource())
	return nil
}

// Flattens AuditConfigs so each role has a single Binding with combined members
func mergeBindings(bindings []*cloudresourcemanager.Binding) []*cloudresourcemanager.Binding {
	bm := createIamBindingsMap(bindings)
	return listFromIamBindingMap(bm)
}

// Flattens Bindings so each role has a single Binding with combined members
func removeAllBindingsWithRole(b []*cloudresourcemanager.Binding, role string) []*cloudresourcemanager.Binding {
	bMap := createIamBindingsMap(b)
	delete(bMap, role)
	return listFromIamBindingMap(bMap)
}

// Removes given role/bound-member pairs from the given Bindings (i.e subtraction).
func subtractFromBindings(bindings []*cloudresourcemanager.Binding, toRemove ...*cloudresourcemanager.Binding) []*cloudresourcemanager.Binding {
	currMap := createIamBindingsMap(bindings)
	toRemoveMap := createIamBindingsMap(toRemove)

	for role, removeSet := range toRemoveMap {
		members, ok := currMap[role]
		if !ok {
			continue
		}
		// Remove all removed members
		for m := range removeSet {
			delete(members, m)
		}
		// Remove role from bindings
		if len(members) == 0 {
			delete(currMap, role)
		}
	}

	return listFromIamBindingMap(currMap)
}

// Construct map of role to set of members from list of bindings.
func createIamBindingsMap(bindings []*cloudresourcemanager.Binding) map[string]map[string]struct{} {
	bm := make(map[string]map[string]struct{})
	// Get each binding
	for _, b := range bindings {
		// Initialize members map
		if _, ok := bm[b.Role]; !ok {
			bm[b.Role] = make(map[string]struct{})
		}
		// Get each member (user/principal) for the binding
		for _, m := range b.Members {
			// Add the member
			bm[b.Role][m] = struct{}{}
		}
	}
	return bm
}

// Return list of Bindings for a map of role to member sets
func listFromIamBindingMap(bm map[string]map[string]struct{}) []*cloudresourcemanager.Binding {
	rb := make([]*cloudresourcemanager.Binding, 0, len(bm))
	for role, members := range bm {
		if len(members) == 0 {
			continue
		}
		rb = append(rb, &cloudresourcemanager.Binding{
			Role:    role,
			Members: stringSliceFromGolangSet(members),
		})
	}
	return rb
}

// Flatten AuditConfigs so each service has a single exemption list of log type to members
func mergeAuditConfigs(auditConfigs []*cloudresourcemanager.AuditConfig) []*cloudresourcemanager.AuditConfig {
	am := createIamAuditConfigsMap(auditConfigs)
	return listFromIamAuditConfigMap(am)
}

// Flattens AuditConfigs so each role has a single Binding with combined members\
func removeAllAuditConfigsWithService(ac []*cloudresourcemanager.AuditConfig, service string) []*cloudresourcemanager.AuditConfig {
	acMap := createIamAuditConfigsMap(ac)
	delete(acMap, service)
	return listFromIamAuditConfigMap(acMap)
}

// Build a AuditConfig service to audit log config map
func createIamAuditConfigsMap(auditConfigs []*cloudresourcemanager.AuditConfig) map[string]map[string]map[string]struct{} {
	acMap := make(map[string]map[string]map[string]struct{})

	for _, ac := range auditConfigs {
		if _, ok := acMap[ac.Service]; !ok {
			acMap[ac.Service] = make(map[string]map[string]struct{})
		}
		alcMap := acMap[ac.Service]
		for _, alc := range ac.AuditLogConfigs {
			if _, ok := alcMap[alc.LogType]; !ok {
				alcMap[alc.LogType] = make(map[string]struct{})
			}
			memberMap := alcMap[alc.LogType]
			// Add members to map for log type.
			for _, m := range alc.ExemptedMembers {
				memberMap[m] = struct{}{}
			}
		}
	}

	return acMap
}

// Construct list of AuditConfigs from audit config maps.
func listFromIamAuditConfigMap(acMap map[string]map[string]map[string]struct{}) []*cloudresourcemanager.AuditConfig {
	ac := make([]*cloudresourcemanager.AuditConfig, 0, len(acMap))

	for service, logConfigMap := range acMap {
		if len(logConfigMap) == 0 {
			continue
		}

		logConfigs := make([]*cloudresourcemanager.AuditLogConfig, 0, len(logConfigMap))
		for logType, memberSet := range logConfigMap {
			alc := &cloudresourcemanager.AuditLogConfig{
				LogType:         logType,
				ForceSendFields: []string{"exemptedMembers"},
			}
			if len(memberSet) > 0 {
				alc.ExemptedMembers = stringSliceFromGolangSet(memberSet)
			}
			logConfigs = append(logConfigs, alc)
		}

		ac = append(ac, &cloudresourcemanager.AuditConfig{
			Service:         service,
			AuditLogConfigs: logConfigs,
		})
	}
	return ac
}

func jsonPolicyDiffSuppress(k, old, new string, d *schema.ResourceData) bool {
	var oldPolicy, newPolicy cloudresourcemanager.Policy
	if err := json.Unmarshal([]byte(old), &oldPolicy); err != nil {
		log.Printf("[ERROR] Could not unmarshal old policy %s: %v", old, err)
		return false
	}
	if err := json.Unmarshal([]byte(new), &newPolicy); err != nil {
		log.Printf("[ERROR] Could not unmarshal new policy %s: %v", new, err)
		return false
	}
	return compareIamPolicies(&newPolicy, &oldPolicy)
}

func compareIamPolicies(a, b *cloudresourcemanager.Policy) bool {
	if a.Etag != b.Etag {
		log.Printf("[DEBUG] policies etag differ: %q vs %q", a.Etag, b.Etag)
		return false
	}
	if a.Version != b.Version {
		log.Printf("[DEBUG] policies version differ: %q vs %q", a.Version, b.Version)
		return false
	}
	if !compareBindings(a.Bindings, b.Bindings) {
		log.Printf("[DEBUG] policies bindings differ: %#v vs %#v", a.Bindings, b.Bindings)
		return false
	}
	if !compareAuditConfigs(a.AuditConfigs, b.AuditConfigs) {
		log.Printf("[DEBUG] policies audit configs differ: %#v vs %#v", a.AuditConfigs, b.AuditConfigs)
		return false
	}
	return true
}

func compareBindings(a, b []*cloudresourcemanager.Binding) bool {
	aMap := createIamBindingsMap(a)
	bMap := createIamBindingsMap(b)
	return reflect.DeepEqual(aMap, bMap)
}

func compareAuditConfigs(a, b []*cloudresourcemanager.AuditConfig) bool {
	aMap := createIamAuditConfigsMap(a)
	bMap := createIamAuditConfigsMap(b)
	return reflect.DeepEqual(aMap, bMap)
}
<commit_msg>Fix IAM member sensitivity. (#1034)<commit_after>// Utils for modifying IAM policies for resources across GCP
package google

import (
	"encoding/json"
	"fmt"
	"log"
	"reflect"
	"strings"
	"time"

	"github.com/hashicorp/errwrap"
	"github.com/hashicorp/terraform/helper/schema"
	"google.golang.org/api/cloudresourcemanager/v1"
)

const maxBackoffSeconds = 30

// These types are implemented per GCP resource type and specify how to do per-resource IAM operations.
// They are used in the generic Terraform IAM resource definitions
// (e.g. _member/_binding/_policy/_audit_config)
type (
	// The ResourceIamUpdater interface is implemented for each GCP resource supporting IAM policy.
	// Implementations should be created per resource and should keep track of the resource identifier.
	ResourceIamUpdater interface {
		// Fetch the existing IAM policy attached to a resource.
		GetResourceIamPolicy() (*cloudresourcemanager.Policy, error)

		// Replaces the existing IAM Policy attached to a resource.
		SetResourceIamPolicy(policy *cloudresourcemanager.Policy) error

		// A mutex guards against concurrent to call to the SetResourceIamPolicy method.
		// The mutex key should be made of the resource type and resource id.
		// For example: `iam-project-{id}`.
		GetMutexKey() string

		// Returns the unique resource identifier.
		GetResourceId() string

		// Textual description of this resource to be used in error message.
		// The description should include the unique resource identifier.
		DescribeResource() string
	}

	// Factory for generating ResourceIamUpdater for given ResourceData resource
	newResourceIamUpdaterFunc func(d *schema.ResourceData, config *Config) (ResourceIamUpdater, error)

	// Describes how to modify a policy for a given Terraform IAM (_policy/_member/_binding/_audit_config) resource
	iamPolicyModifyFunc func(p *cloudresourcemanager.Policy) error

	// Parser for Terraform resource identifier (d.Id) for resource whose IAM policy is being changed
	resourceIdParserFunc func(d *schema.ResourceData, config *Config) error
)

// Locking wrapper around read-only operation with retries.
func iamPolicyReadWithRetry(updater ResourceIamUpdater) (*cloudresourcemanager.Policy, error) {
	mutexKey := updater.GetMutexKey()
	mutexKV.Lock(mutexKey)
	defer mutexKV.Unlock(mutexKey)

	log.Printf("[DEBUG] Retrieving policy for %s\n", updater.DescribeResource())
	var policy *cloudresourcemanager.Policy
	err := retryTime(func() (perr error) {
		policy, perr = updater.GetResourceIamPolicy()
		return perr
	}, 10)
	if err != nil {
		return nil, err
	}
	log.Printf("[DEBUG] Retrieved policy for %s: %+v\n", updater.DescribeResource(), policy)
	return policy, nil
}

// Locking wrapper around read-modify-write cycle for IAM policy.
func iamPolicyReadModifyWrite(updater ResourceIamUpdater, modify iamPolicyModifyFunc) error {
	mutexKey := updater.GetMutexKey()
	mutexKV.Lock(mutexKey)
	defer mutexKV.Unlock(mutexKey)

	backoff := time.Second
	for {
		log.Printf("[DEBUG]: Retrieving policy for %s\n", updater.DescribeResource())
		p, err := updater.GetResourceIamPolicy()
		if isGoogleApiErrorWithCode(err, 429) {
			log.Printf("[DEBUG] 429 while attempting to read policy for %s, waiting %v before attempting again", updater.DescribeResource(), backoff)
			time.Sleep(backoff)
			continue
		} else if err != nil {
			return err
		}
		log.Printf("[DEBUG]: Retrieved policy for %s: %+v\n", updater.DescribeResource(), p)

		err = modify(p)
		if err != nil {
			return err
		}

		log.Printf("[DEBUG]: Setting policy for %s to %+v\n", updater.DescribeResource(), p)
		err = updater.SetResourceIamPolicy(p)
		if err == nil {
			fetchBackoff := 1 * time.Second
			for successfulFetches := 0; successfulFetches < 3; {
				if fetchBackoff > maxBackoffSeconds*time.Second {
					return fmt.Errorf("Error applying IAM policy to %s: Waited too long for propagation.\n", updater.DescribeResource())
				}
				time.Sleep(fetchBackoff)
				log.Printf("[DEBUG]: Retrieving policy for %s\n", updater.DescribeResource())
				new_p, err := updater.GetResourceIamPolicy()
				if err != nil {
					// Quota for Read is pretty limited, so watch out for running out of quota.
					if isGoogleApiErrorWithCode(err, 429) {
						fetchBackoff = fetchBackoff * 2
					} else {
						return err
					}
				}
				log.Printf("[DEBUG]: Retrieved policy for %s: %+v\n", updater.DescribeResource(), p)
				if new_p == nil {
					// https://github.com/terraform-providers/terraform-provider-google/issues/2625
					fetchBackoff = fetchBackoff * 2
					continue
				}
				modified_p := new_p
				// This relies on the fact that `modify` is idempotent: since other changes might have
				// happened between the call to set the policy and now, we just need to make sure that
				// our change has been made.  'modify(p) == p' is our check for whether this has been
				// correctly applied.
				err = modify(modified_p)
				if err != nil {
					return err
				}
				if modified_p == new_p {
					successfulFetches += 1
				} else {
					fetchBackoff = fetchBackoff * 2
				}
			}
			break
		}
		if isConflictError(err) {
			log.Printf("[DEBUG]: Concurrent policy changes, restarting read-modify-write after %s\n", backoff)
			time.Sleep(backoff)
			backoff = backoff * 2
			if backoff > 30*time.Second {
				return errwrap.Wrapf(fmt.Sprintf("Error applying IAM policy to %s: Too many conflicts.  Latest error: {{err}}", updater.DescribeResource()), err)
			}
			continue
		}
		return errwrap.Wrapf(fmt.Sprintf("Error applying IAM policy for %s: {{err}}", updater.DescribeResource()), err)
	}
	log.Printf("[DEBUG]: Set policy for %s", updater.DescribeResource())
	return nil
}

// Flattens AuditConfigs so each role has a single Binding with combined members
func mergeBindings(bindings []*cloudresourcemanager.Binding) []*cloudresourcemanager.Binding {
	bm := createIamBindingsMap(bindings)
	return listFromIamBindingMap(bm)
}

// Flattens Bindings so each role has a single Binding with combined members
func removeAllBindingsWithRole(b []*cloudresourcemanager.Binding, role string) []*cloudresourcemanager.Binding {
	bMap := createIamBindingsMap(b)
	delete(bMap, role)
	return listFromIamBindingMap(bMap)
}

// Removes given role/bound-member pairs from the given Bindings (i.e subtraction).
func subtractFromBindings(bindings []*cloudresourcemanager.Binding, toRemove ...*cloudresourcemanager.Binding) []*cloudresourcemanager.Binding {
	currMap := createIamBindingsMap(bindings)
	toRemoveMap := createIamBindingsMap(toRemove)

	for role, removeSet := range toRemoveMap {
		members, ok := currMap[role]
		if !ok {
			continue
		}
		// Remove all removed members
		for m := range removeSet {
			delete(members, m)
		}
		// Remove role from bindings
		if len(members) == 0 {
			delete(currMap, role)
		}
	}

	return listFromIamBindingMap(currMap)
}

// Construct map of role to set of members from list of bindings.
func createIamBindingsMap(bindings []*cloudresourcemanager.Binding) map[string]map[string]struct{} {
	bm := make(map[string]map[string]struct{})
	// Get each binding
	for _, b := range bindings {
		// Initialize members map
		if _, ok := bm[b.Role]; !ok {
			bm[b.Role] = make(map[string]struct{})
		}
		// Get each member (user/principal) for the binding
		for _, m := range b.Members {
			// Add the member
			bm[b.Role][strings.ToLower(m)] = struct{}{}
		}
	}
	return bm
}

// Return list of Bindings for a map of role to member sets
func listFromIamBindingMap(bm map[string]map[string]struct{}) []*cloudresourcemanager.Binding {
	rb := make([]*cloudresourcemanager.Binding, 0, len(bm))
	for role, members := range bm {
		if len(members) == 0 {
			continue
		}
		rb = append(rb, &cloudresourcemanager.Binding{
			Role:    role,
			Members: stringSliceFromGolangSet(members),
		})
	}
	return rb
}

// Flatten AuditConfigs so each service has a single exemption list of log type to members
func mergeAuditConfigs(auditConfigs []*cloudresourcemanager.AuditConfig) []*cloudresourcemanager.AuditConfig {
	am := createIamAuditConfigsMap(auditConfigs)
	return listFromIamAuditConfigMap(am)
}

// Flattens AuditConfigs so each role has a single Binding with combined members\
func removeAllAuditConfigsWithService(ac []*cloudresourcemanager.AuditConfig, service string) []*cloudresourcemanager.AuditConfig {
	acMap := createIamAuditConfigsMap(ac)
	delete(acMap, service)
	return listFromIamAuditConfigMap(acMap)
}

// Build a AuditConfig service to audit log config map
func createIamAuditConfigsMap(auditConfigs []*cloudresourcemanager.AuditConfig) map[string]map[string]map[string]struct{} {
	acMap := make(map[string]map[string]map[string]struct{})

	for _, ac := range auditConfigs {
		if _, ok := acMap[ac.Service]; !ok {
			acMap[ac.Service] = make(map[string]map[string]struct{})
		}
		alcMap := acMap[ac.Service]
		for _, alc := range ac.AuditLogConfigs {
			if _, ok := alcMap[alc.LogType]; !ok {
				alcMap[alc.LogType] = make(map[string]struct{})
			}
			memberMap := alcMap[alc.LogType]
			// Add members to map for log type.
			for _, m := range alc.ExemptedMembers {
				memberMap[m] = struct{}{}
			}
		}
	}

	return acMap
}

// Construct list of AuditConfigs from audit config maps.
func listFromIamAuditConfigMap(acMap map[string]map[string]map[string]struct{}) []*cloudresourcemanager.AuditConfig {
	ac := make([]*cloudresourcemanager.AuditConfig, 0, len(acMap))

	for service, logConfigMap := range acMap {
		if len(logConfigMap) == 0 {
			continue
		}

		logConfigs := make([]*cloudresourcemanager.AuditLogConfig, 0, len(logConfigMap))
		for logType, memberSet := range logConfigMap {
			alc := &cloudresourcemanager.AuditLogConfig{
				LogType:         logType,
				ForceSendFields: []string{"exemptedMembers"},
			}
			if len(memberSet) > 0 {
				alc.ExemptedMembers = stringSliceFromGolangSet(memberSet)
			}
			logConfigs = append(logConfigs, alc)
		}

		ac = append(ac, &cloudresourcemanager.AuditConfig{
			Service:         service,
			AuditLogConfigs: logConfigs,
		})
	}
	return ac
}

func jsonPolicyDiffSuppress(k, old, new string, d *schema.ResourceData) bool {
	var oldPolicy, newPolicy cloudresourcemanager.Policy
	if err := json.Unmarshal([]byte(old), &oldPolicy); err != nil {
		log.Printf("[ERROR] Could not unmarshal old policy %s: %v", old, err)
		return false
	}
	if err := json.Unmarshal([]byte(new), &newPolicy); err != nil {
		log.Printf("[ERROR] Could not unmarshal new policy %s: %v", new, err)
		return false
	}
	return compareIamPolicies(&newPolicy, &oldPolicy)
}

func compareIamPolicies(a, b *cloudresourcemanager.Policy) bool {
	if a.Etag != b.Etag {
		log.Printf("[DEBUG] policies etag differ: %q vs %q", a.Etag, b.Etag)
		return false
	}
	if a.Version != b.Version {
		log.Printf("[DEBUG] policies version differ: %q vs %q", a.Version, b.Version)
		return false
	}
	if !compareBindings(a.Bindings, b.Bindings) {
		log.Printf("[DEBUG] policies bindings differ: %#v vs %#v", a.Bindings, b.Bindings)
		return false
	}
	if !compareAuditConfigs(a.AuditConfigs, b.AuditConfigs) {
		log.Printf("[DEBUG] policies audit configs differ: %#v vs %#v", a.AuditConfigs, b.AuditConfigs)
		return false
	}
	return true
}

func compareBindings(a, b []*cloudresourcemanager.Binding) bool {
	aMap := createIamBindingsMap(a)
	bMap := createIamBindingsMap(b)
	return reflect.DeepEqual(aMap, bMap)
}

func compareAuditConfigs(a, b []*cloudresourcemanager.AuditConfig) bool {
	aMap := createIamAuditConfigsMap(a)
	bMap := createIamAuditConfigsMap(b)
	return reflect.DeepEqual(aMap, bMap)
}
<|endoftext|>


<commit_before>package embd

import "testing"

type fakeDigitalPin struct {
	n int
}

func (p *fakeDigitalPin) N() int {
	return p.n
}

func (*fakeDigitalPin) SetDirection(dir Direction) error {
	return nil
}

func (*fakeDigitalPin) Read() (int, error) {
	return 0, nil
}

func (*fakeDigitalPin) Write(val int) error {
	return nil
}

func (*fakeDigitalPin) ActiveLow(b bool) error {
	return nil
}

func (*fakeDigitalPin) PullUp() error {
	return nil
}

func (*fakeDigitalPin) PullDown() error {
	return nil
}

func (*fakeDigitalPin) Close() error {
	return nil
}

func newFakeDigitalPin(n int) DigitalPin {
	return &fakeDigitalPin{n}
}

func TestGpioDriverDigitalPin(t *testing.T) {
	tests := []struct {
		key interface{}
		n   int
	}{
		{1, 1},
	}
	pinMap := PinMap{
		&PinDesc{ID: "P1_1", Aliases: []string{"1"}, Caps: CapDigital, DigitalLogical: 1},
	}
	driver := newGPIODriver(pinMap, newFakeDigitalPin, nil, nil)
	for _, test := range tests {
		pin, err := driver.DigitalPin(test.key)
		if err != nil {
			t.Errorf("Looking up %v: unexpected error: %v", test.key, err)
			continue
		}
		if pin.N() != test.n {
			t.Errorf("Looking up %v: got %v, want %v", test.key, pin.N(), test.n)
		}
	}
}

type fakeAnalogPin struct {
	n int
}

func (p *fakeAnalogPin) N() int {
	return p.n
}

func (*fakeAnalogPin) Read() (int, error) {
	return 0, nil
}

func (*fakeAnalogPin) Write(val int) error {
	return nil
}

func (*fakeAnalogPin) Close() error {
	return nil
}

func newFakeAnalogPin(n int) AnalogPin {
	return &fakeAnalogPin{n}
}

func TestGpioDriverAnalogPin(t *testing.T) {
	tests := []struct {
		key interface{}
		n   int
	}{
		{1, 1},
	}
	pinMap := PinMap{
		&PinDesc{ID: "P1_1", Aliases: []string{"1"}, Caps: CapAnalog, AnalogLogical: 1},
	}
	driver := newGPIODriver(pinMap, nil, newFakeAnalogPin, nil)
	for _, test := range tests {
		pin, err := driver.AnalogPin(test.key)
		if err != nil {
			t.Errorf("Looking up %v: unexpected error: %v", test.key, err)
			continue
		}
		if pin.N() != test.n {
			t.Errorf("Looking up %v: got %v, want %v", test.key, pin.N(), test.n)
		}
	}
}

func TestGpioDriverUnavailablePinType(t *testing.T) {
	pinMap := PinMap{
		&PinDesc{ID: "P1_1", Aliases: []string{"1"}, Caps: CapDigital, DigitalLogical: 1},
		&PinDesc{ID: "P1_2", Aliases: []string{"1"}, Caps: CapAnalog, AnalogLogical: 1},
	}
	driver := newGPIODriver(pinMap, nil, nil, nil)
	_, err := driver.DigitalPin(1)
	if err == nil {
		t.Fatal("Looking up digital pin 1: did not get error")
	}
	expected := "gpio: digital io not supported on this host"
	if err.Error() != expected {
		t.Fatalf("Looking up digital pin 1: got error %q, expected %q", err, expected)
	}
	_, err = driver.AnalogPin(1)
	if err == nil {
		t.Fatal("Looking up analog pin 1: did not get error")
	}
	expected = "gpio: analog io not supported on this host"
	if err.Error() != expected {
		t.Fatalf("Looking up analog pin 1: got error %q, expected %q", err, expected)
	}
}
<commit_msg>gpio: fix failing tests<commit_after>package embd

import (
	"testing"
	"time"
)

type fakeDigitalPin struct {
	n int
}

func (p *fakeDigitalPin) N() int {
	return p.n
}

func (*fakeDigitalPin) SetDirection(dir Direction) error {
	return nil
}

func (*fakeDigitalPin) Read() (int, error) {
	return 0, nil
}

func (*fakeDigitalPin) Write(val int) error {
	return nil
}

func (*fakeDigitalPin) TimePulse(state int) (time.Duration, error) {
	return 0, nil
}

func (*fakeDigitalPin) ActiveLow(b bool) error {
	return nil
}

func (*fakeDigitalPin) PullUp() error {
	return nil
}

func (*fakeDigitalPin) PullDown() error {
	return nil
}

func (*fakeDigitalPin) Close() error {
	return nil
}

func newFakeDigitalPin(n int) DigitalPin {
	return &fakeDigitalPin{n}
}

func TestGpioDriverDigitalPin(t *testing.T) {
	tests := []struct {
		key interface{}
		n   int
	}{
		{1, 1},
	}
	pinMap := PinMap{
		&PinDesc{ID: "P1_1", Aliases: []string{"1"}, Caps: CapDigital, DigitalLogical: 1},
	}
	driver := newGPIODriver(pinMap, newFakeDigitalPin, nil, nil)
	for _, test := range tests {
		pin, err := driver.DigitalPin(test.key)
		if err != nil {
			t.Errorf("Looking up %v: unexpected error: %v", test.key, err)
			continue
		}
		if pin.N() != test.n {
			t.Errorf("Looking up %v: got %v, want %v", test.key, pin.N(), test.n)
		}
	}
}

type fakeAnalogPin struct {
	n int
}

func (p *fakeAnalogPin) N() int {
	return p.n
}

func (*fakeAnalogPin) Read() (int, error) {
	return 0, nil
}

func (*fakeAnalogPin) Write(val int) error {
	return nil
}

func (*fakeAnalogPin) Close() error {
	return nil
}

func newFakeAnalogPin(n int) AnalogPin {
	return &fakeAnalogPin{n}
}

func TestGpioDriverAnalogPin(t *testing.T) {
	tests := []struct {
		key interface{}
		n   int
	}{
		{1, 1},
	}
	pinMap := PinMap{
		&PinDesc{ID: "P1_1", Aliases: []string{"1"}, Caps: CapAnalog, AnalogLogical: 1},
	}
	driver := newGPIODriver(pinMap, nil, newFakeAnalogPin, nil)
	for _, test := range tests {
		pin, err := driver.AnalogPin(test.key)
		if err != nil {
			t.Errorf("Looking up %v: unexpected error: %v", test.key, err)
			continue
		}
		if pin.N() != test.n {
			t.Errorf("Looking up %v: got %v, want %v", test.key, pin.N(), test.n)
		}
	}
}

func TestGpioDriverUnavailablePinType(t *testing.T) {
	pinMap := PinMap{
		&PinDesc{ID: "P1_1", Aliases: []string{"1"}, Caps: CapDigital, DigitalLogical: 1},
		&PinDesc{ID: "P1_2", Aliases: []string{"1"}, Caps: CapAnalog, AnalogLogical: 1},
	}
	driver := newGPIODriver(pinMap, nil, nil, nil)
	_, err := driver.DigitalPin(1)
	if err == nil {
		t.Fatal("Looking up digital pin 1: did not get error")
	}
	expected := "gpio: digital io not supported on this host"
	if err.Error() != expected {
		t.Fatalf("Looking up digital pin 1: got error %q, expected %q", err, expected)
	}
	_, err = driver.AnalogPin(1)
	if err == nil {
		t.Fatal("Looking up analog pin 1: did not get error")
	}
	expected = "gpio: analog io not supported on this host"
	if err.Error() != expected {
		t.Fatalf("Looking up analog pin 1: got error %q, expected %q", err, expected)
	}
}
<|endoftext|>