<commit_before>package geodata

import (
	"fmt"

	"github.com/golang/geo/s2"
	"github.com/golang/protobuf/ptypes/struct"
	spb "github.com/golang/protobuf/ptypes/struct"
	"github.com/pkg/errors"
	"github.com/twpayne/go-geom"
	"github.com/twpayne/go-geom/encoding/geojson"
)

// GeomToGeoData update gd with geo data gathered from g
func GeomToGeoData(g geom.T, gd *GeoData) error {
	geo := &Geometry{}

	switch g := g.(type) {
	case *geom.Point:
		geo.Coordinates = g.Coords()
		geo.Type = Geometry_POINT
		//case *geom.MultiPolygon:
		//	geo.Type = geodata.Geometry_MULTIPOLYGON

	case *geom.Polygon:
		// only supports outer ring
		geo.Type = Geometry_POLYGON
		geo.Coordinates = g.FlatCoords()

	default:
		return errors.Errorf("unsupported geo type %T", g)
	}

	gd.Geometry = geo
	return nil
}

// GeoDataToGeom
func GeoDataToGeom(gd *GeoData) (geom.T, error) {
	switch gd.Geometry.Type {
	case Geometry_POINT:
		return geom.NewPointFlat(geom.XY, gd.Geometry.Coordinates), nil
	case Geometry_POLYGON:
		return geom.NewPolygonFlat(geom.XY, gd.Geometry.Coordinates, []int{len(gd.Geometry.Coordinates)}), nil
	default:
		return nil, errors.Errorf("unsupported geodata type")
	}
}

// GeoJSONFeatureToGeoData fill gd with the GeoJSON data f
func GeoJSONFeatureToGeoData(f *geojson.Feature, gd *GeoData) error {
	err := PropertiesToGeoData(f, gd)
	if err != nil {
		return errors.Wrap(err, "while converting feature properties to GeoData")
	}

	err = GeomToGeoData(f.Geometry, gd)
	if err != nil {
		return errors.Wrap(err, "while converting feature to GeoData")
	}

	return nil
}

// PropertiesToGeoData update gd.Properties with the properties found in f
func PropertiesToGeoData(f *geojson.Feature, gd *GeoData) error {
	for k, vi := range f.Properties {
		switch tv := vi.(type) {
		case bool:
			gd.Properties[k] = &structpb.Value{Kind: &structpb.Value_BoolValue{BoolValue: tv}}
		case int:
			gd.Properties[k] = &structpb.Value{Kind: &structpb.Value_NumberValue{NumberValue: float64(tv)}}
		case string:
			gd.Properties[k] = &structpb.Value{Kind: &structpb.Value_StringValue{StringValue: tv}}
		case float64:
			gd.Properties[k] = &structpb.Value{Kind: &structpb.Value_NumberValue{NumberValue: tv}}
		case nil:
			// pass
		default:
			return fmt.Errorf("GeoJSON property %s unsupported type %T", k, tv)
		}
	}
	return nil
}

// GeoDataToFlatCellUnion generate an s2 cover for GeoData gd
func GeoDataToFlatCellUnion(gd *GeoData, coverer *s2.RegionCoverer) (s2.CellUnion, error) {
	var cu s2.CellUnion
	switch gd.Geometry.Type {
	case Geometry_POINT:
		c := s2.CellIDFromLatLng(s2.LatLngFromDegrees(gd.Geometry.Coordinates[1], gd.Geometry.Coordinates[0]))
		cu = append(cu, c.Parent(coverer.MinLevel))

	case Geometry_POLYGON:
		cup, err := coverPolygon(gd.Geometry.Coordinates, coverer)
		if err != nil {
			return nil, errors.Wrap(err, "can't cover polygon")
		}
		cu = append(cu, cup...)

	case Geometry_MULTIPOLYGON:
		for _, g := range gd.Geometry.Geometries {
			cup, err := coverPolygon(g.Coordinates, coverer)
			if err != nil {
				return nil, errors.Wrap(err, "can't cover multipolygon")
			}

			cu = append(cu, cup...)
		}

	case Geometry_LINESTRING:
		if len(gd.Geometry.Coordinates)%2 != 0 {
			return nil, errors.New("invalid coordinates count for line")
		}

		// the s2 cover returns bogus results
		// pl := make(s2.Polyline, len(gd.Geometry.Coordinates))
		// for i := 0; i <= len(gd.Geometry.Coordinates)/2+2; i += 2 {
		// 	ll := s2.LatLngFromDegrees(gd.Geometry.Coordinates[i+1], gd.Geometry.Coordinates[i])
		// 	pl[i/2] = s2.PointFromLatLng(ll)
		// }

		// cupl := coverer.Covering(&pl)
		// cu = append(cu, cupl...)

		// uncomplete tempory solution
		// for each points add the cell, it is invalid because 2 distant points could be in more than 2 different cells
		m := make(map[s2.CellID]struct{})
		for i := 0; i <= len(gd.Geometry.Coordinates)/2+2; i += 2 {
			ll := s2.LatLngFromDegrees(gd.Geometry.Coordinates[i+1], gd.Geometry.Coordinates[i])
			c := s2.CellIDFromLatLng(ll).Parent(coverer.MinLevel)
			m[c] = struct{}{}
		}
		for c := range m {
			cu = append(cu, c)
		}
	default:
		return nil, errors.New("unsupported data type")
	}

	return cu, nil
}

// returns an s2 cover from a list of lng, lat forming a closed polygon
func coverPolygon(c []float64, coverer *s2.RegionCoverer) (s2.CellUnion, error) {
	if len(c) < 6 {
		return nil, errors.New("invalid polygons not enough coordinates for a closed polygon")
	}
	if len(c)%2 != 0 {
		return nil, errors.New("invalid polygons odd coordinates number")
	}
	l := LoopFromCoordinates(c)
	if l.IsEmpty() || l.IsFull() || l.ContainsOrigin() {
		return nil, errors.New("invalid polygons")
	}

	return coverer.Covering(l), nil
}

// ToGeoJSONFeatureCollection converts a list of GeoData to a GeoJSON Feature Collection
func ToGeoJSONFeatureCollection(geos []*GeoData) ([]byte, error) {
	fc := geojson.FeatureCollection{}
	for _, g := range geos {
		f := &geojson.Feature{}
		switch g.Geometry.Type {
		case Geometry_POINT:
			ng := geom.NewPointFlat(geom.XY, g.Geometry.Coordinates)
			f.Geometry = ng
		case Geometry_POLYGON:
			ng := geom.NewPolygonFlat(geom.XY, g.Geometry.Coordinates, []int{len(g.Geometry.Coordinates)})
			f.Geometry = ng
		case Geometry_MULTIPOLYGON:
			mp := geom.NewMultiPolygon(geom.XY)
			for _, poly := range g.Geometry.Geometries {
				ng := geom.NewPolygonFlat(geom.XY, poly.Coordinates, []int{len(poly.Coordinates)})
				mp.Push(ng)
			}
			f.Geometry = mp
		}
		f.Properties = PropertiesToJSONMap(g.Properties)
		fc.Features = append(fc.Features, f)
	}

	return fc.MarshalJSON()
}

// PointsToGeoJSONPolyLines converts a list of GeoDatato containing points to a polylines GeoJSON
func PointsToGeoJSONPolyLines(geos []*GeoData) ([]byte, error) {
	f := geojson.Feature{}
	var flatCoords []float64

	if len(geos) == 0 {
		return f.MarshalJSON()
	}

	for _, g := range geos {
		switch g.Geometry.Type {
		case Geometry_POINT:
			flatCoords = append(flatCoords, g.Geometry.Coordinates...)
		default:
			return nil, errors.Errorf("unsupported geometry")

		}

	}
	f.Properties = PropertiesToJSONMap(geos[0].Properties)
	g := geom.NewLineStringFlat(geom.XY, flatCoords)
	f.Geometry = g

	return f.MarshalJSON()
}

// PropertiesToJSONMap converts a protobuf map to it's JSON serializable map equivalent
func PropertiesToJSONMap(src map[string]*spb.Value) map[string]interface{} {
	res := make(map[string]interface{})

	for k, v := range src {
		switch x := v.Kind.(type) {
		case *spb.Value_NumberValue:
			res[k] = x.NumberValue
		case *spb.Value_StringValue:
			res[k] = x.StringValue
		case *spb.Value_BoolValue:
			res[k] = x.BoolValue
		}
	}
	return res
}
<commit_msg>update tools for line<commit_after>package geodata

import (
	"fmt"

	"github.com/golang/geo/s2"
	"github.com/golang/protobuf/ptypes/struct"
	spb "github.com/golang/protobuf/ptypes/struct"
	"github.com/pkg/errors"
	"github.com/twpayne/go-geom"
	"github.com/twpayne/go-geom/encoding/geojson"
)

// GeomToGeoData update gd with geo data gathered from g
func GeomToGeoData(g geom.T, gd *GeoData) error {
	geo := &Geometry{}

	switch g := g.(type) {
	case *geom.Point:
		geo.Coordinates = g.Coords()
		geo.Type = Geometry_POINT
		//case *geom.MultiPolygon:
		//	geo.Type = geodata.Geometry_MULTIPOLYGON

	case *geom.Polygon:
		// only supports outer ring
		geo.Type = Geometry_POLYGON
		geo.Coordinates = g.FlatCoords()

	case *geom.LineString:
		geo.Type = Geometry_LINESTRING
		geo.Coordinates = g.FlatCoords()

	default:
		return errors.Errorf("unsupported geo type %T", g)
	}

	gd.Geometry = geo
	return nil
}

// GeoDataToGeom
func GeoDataToGeom(gd *GeoData) (geom.T, error) {
	switch gd.Geometry.Type {
	case Geometry_POINT:
		return geom.NewPointFlat(geom.XY, gd.Geometry.Coordinates), nil
	case Geometry_POLYGON:
		return geom.NewPolygonFlat(geom.XY, gd.Geometry.Coordinates, []int{len(gd.Geometry.Coordinates)}), nil
	case Geometry_LINESTRING:
		return geom.NewLineStringFlat(geo, XY, gd.Geometry.Coordinates)
	default:
		return nil, errors.Errorf("unsupported geodata type")
	}
}

// GeoJSONFeatureToGeoData fill gd with the GeoJSON data f
func GeoJSONFeatureToGeoData(f *geojson.Feature, gd *GeoData) error {
	err := PropertiesToGeoData(f, gd)
	if err != nil {
		return errors.Wrap(err, "while converting feature properties to GeoData")
	}

	err = GeomToGeoData(f.Geometry, gd)
	if err != nil {
		return errors.Wrap(err, "while converting feature to GeoData")
	}

	return nil
}

// PropertiesToGeoData update gd.Properties with the properties found in f
func PropertiesToGeoData(f *geojson.Feature, gd *GeoData) error {
	for k, vi := range f.Properties {
		switch tv := vi.(type) {
		case bool:
			gd.Properties[k] = &structpb.Value{Kind: &structpb.Value_BoolValue{BoolValue: tv}}
		case int:
			gd.Properties[k] = &structpb.Value{Kind: &structpb.Value_NumberValue{NumberValue: float64(tv)}}
		case string:
			gd.Properties[k] = &structpb.Value{Kind: &structpb.Value_StringValue{StringValue: tv}}
		case float64:
			gd.Properties[k] = &structpb.Value{Kind: &structpb.Value_NumberValue{NumberValue: tv}}
		case nil:
			// pass
		default:
			return fmt.Errorf("GeoJSON property %s unsupported type %T", k, tv)
		}
	}
	return nil
}

// GeoDataToFlatCellUnion generate an s2 cover for GeoData gd
func GeoDataToFlatCellUnion(gd *GeoData, coverer *s2.RegionCoverer) (s2.CellUnion, error) {
	var cu s2.CellUnion
	switch gd.Geometry.Type {
	case Geometry_POINT:
		c := s2.CellIDFromLatLng(s2.LatLngFromDegrees(gd.Geometry.Coordinates[1], gd.Geometry.Coordinates[0]))
		cu = append(cu, c.Parent(coverer.MinLevel))

	case Geometry_POLYGON:
		cup, err := coverPolygon(gd.Geometry.Coordinates, coverer)
		if err != nil {
			return nil, errors.Wrap(err, "can't cover polygon")
		}
		cu = append(cu, cup...)

	case Geometry_MULTIPOLYGON:
		for _, g := range gd.Geometry.Geometries {
			cup, err := coverPolygon(g.Coordinates, coverer)
			if err != nil {
				return nil, errors.Wrap(err, "can't cover multipolygon")
			}

			cu = append(cu, cup...)
		}

	case Geometry_LINESTRING:
		if len(gd.Geometry.Coordinates)%2 != 0 {
			return nil, errors.New("invalid coordinates count for line")
		}

		// the s2 cover returns bogus results
		// pl := make(s2.Polyline, len(gd.Geometry.Coordinates))
		// for i := 0; i <= len(gd.Geometry.Coordinates)/2+2; i += 2 {
		// 	ll := s2.LatLngFromDegrees(gd.Geometry.Coordinates[i+1], gd.Geometry.Coordinates[i])
		// 	pl[i/2] = s2.PointFromLatLng(ll)
		// }

		// cupl := coverer.Covering(&pl)
		// cu = append(cu, cupl...)

		// uncomplete tempory solution
		// for each points add the cell, it is invalid because 2 distant points could be in more than 2 different cells
		m := make(map[s2.CellID]struct{})
		for i := 0; i <= len(gd.Geometry.Coordinates)/2+2; i += 2 {
			ll := s2.LatLngFromDegrees(gd.Geometry.Coordinates[i+1], gd.Geometry.Coordinates[i])
			c := s2.CellIDFromLatLng(ll).Parent(coverer.MinLevel)
			m[c] = struct{}{}
		}
		for c := range m {
			cu = append(cu, c)
		}
	default:
		return nil, errors.New("unsupported data type")
	}

	return cu, nil
}

// returns an s2 cover from a list of lng, lat forming a closed polygon
func coverPolygon(c []float64, coverer *s2.RegionCoverer) (s2.CellUnion, error) {
	if len(c) < 6 {
		return nil, errors.New("invalid polygons not enough coordinates for a closed polygon")
	}
	if len(c)%2 != 0 {
		return nil, errors.New("invalid polygons odd coordinates number")
	}
	l := LoopFromCoordinates(c)
	if l.IsEmpty() || l.IsFull() || l.ContainsOrigin() {
		return nil, errors.New("invalid polygons")
	}

	return coverer.Covering(l), nil
}

// ToGeoJSONFeatureCollection converts a list of GeoData to a GeoJSON Feature Collection
func ToGeoJSONFeatureCollection(geos []*GeoData) ([]byte, error) {
	fc := geojson.FeatureCollection{}
	for _, g := range geos {
		f := &geojson.Feature{}
		switch g.Geometry.Type {
		case Geometry_POINT:
			ng := geom.NewPointFlat(geom.XY, g.Geometry.Coordinates)
			f.Geometry = ng
		case Geometry_POLYGON:
			ng := geom.NewPolygonFlat(geom.XY, g.Geometry.Coordinates, []int{len(g.Geometry.Coordinates)})
			f.Geometry = ng
		case Geometry_MULTIPOLYGON:
			mp := geom.NewMultiPolygon(geom.XY)
			for _, poly := range g.Geometry.Geometries {
				ng := geom.NewPolygonFlat(geom.XY, poly.Coordinates, []int{len(poly.Coordinates)})
				mp.Push(ng)
			}
			f.Geometry = mp
		case Geometry_LINESTRING:
			ls := geom.NewLineStringFlat(geom.XY, g.Geometry.Coordinates)
			f.Geometry = ls
		}
		f.Properties = PropertiesToJSONMap(g.Properties)
		fc.Features = append(fc.Features, f)
	}

	return fc.MarshalJSON()
}

// PointsToGeoJSONPolyLines converts a list of GeoData containing points to a polylines GeoJSON
func PointsToGeoJSONPolyLines(geos []*GeoData) ([]byte, error) {
	f := geojson.Feature{}
	var flatCoords []float64

	if len(geos) == 0 {
		return f.MarshalJSON()
	}

	for _, g := range geos {
		switch g.Geometry.Type {
		case Geometry_POINT:
			flatCoords = append(flatCoords, g.Geometry.Coordinates...)
		default:
			return nil, errors.Errorf("unsupported geometry")
		}

	}
	f.Properties = PropertiesToJSONMap(geos[0].Properties)
	g := geom.NewLineStringFlat(geom.XY, flatCoords)
	f.Geometry = g

	return f.MarshalJSON()
}

// PropertiesToJSONMap converts a protobuf map to it's JSON serializable map equivalent
func PropertiesToJSONMap(src map[string]*spb.Value) map[string]interface{} {
	res := make(map[string]interface{})

	for k, v := range src {
		switch x := v.Kind.(type) {
		case *spb.Value_NumberValue:
			res[k] = x.NumberValue
		case *spb.Value_StringValue:
			res[k] = x.StringValue
		case *spb.Value_BoolValue:
			res[k] = x.BoolValue
		}
	}
	return res
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/sloonz/cfeedparser"
	"github.com/sloonz/go-maildir"
	"github.com/sloonz/go-mime-message"
	"github.com/sloonz/go-qprintable"
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"regexp"
	"strings"
	"time"
)

type Cache struct {
	data map[string]bool
	path string
}

func (c *Cache) load() error {
	cacheFile, err := os.Open(c.path)
	if err != nil {
		return err
	}

	data, err := ioutil.ReadAll(cacheFile)
	if err != nil {
		return err
	}

	return json.Unmarshal(data, &c.data)
}

func (c *Cache) dump() error {
	cacheFile, err := os.Create(c.path + ".new")
	if err != nil {
		return err
	}
	defer cacheFile.Close()

	enc := json.NewEncoder(cacheFile)
	if err = enc.Encode(c.data); err != nil {
		return err
	}

	return os.Rename(c.path+".new", c.path)
}

var cache Cache

func firstNonEmpty(s ...string) string {
	var val string
	for _, val = range s {
		if val != "" {
			break
		}
	}
	return val
}

func getRFC822Date(e *feedparser.Entry) string {
	emptyTime := time.Time{}
	if e.PublicationDateParsed != emptyTime {
		return e.PublicationDateParsed.Format(time.RFC1123Z)
	}
	if e.ModificationDateParsed != emptyTime {
		return e.ModificationDateParsed.Format(time.RFC1123Z)
	}
	if e.PublicationDate != "" {
		return e.PublicationDate
	}
	if e.ModificationDate != "" {
		return e.ModificationDate
	}
	return time.Now().UTC().Format(time.RFC1123Z)
}

func getFrom(e *feedparser.Entry) string {
	name := strings.TrimSpace(message.EncodeWord(firstNonEmpty(e.Author.Name, e.Author.Uri, e.Author.Text)))
	if e.Author.Email != "" {
		name += " <" + strings.TrimSpace(e.Author.Email) + ">"
	}
	return name
}

var convertEOLReg = regexp.MustCompile("\r\n?")

func convertEOL(s string) string {
	return convertEOLReg.ReplaceAllString(s, "\n")
}

func process(rawUrl string) error {
	url_, err := url.Parse(rawUrl)
	if err != nil {
		return err
	}

	md, err := maildir.New(".", false)
	if err != nil {
		return err
	}

	feed, err := feedparser.ParseURL(url_)
	if err != nil {
		return err
	}

	fmt.Printf("[%s]\n", feed.Title)
	for _, entry := range feed.Entries {
		postId := firstNonEmpty(entry.Id, entry.Link, entry.PublicationDate+":"+entry.Title)
		if _, hasId := cache.data[postId]; hasId {
			continue
		}

		body := convertEOL(firstNonEmpty(entry.Content, entry.Summary))
		body += "\n
View post
\n"

		title := strings.TrimSpace(entry.Title)
		msg := message.NewTextMessage(qprintable.UnixTextEncoding, bytes.NewBufferString(body))
		msg.SetHeader("Date", getRFC822Date(&entry))
		msg.SetHeader("From", getFrom(&entry))
		msg.SetHeader("To", "Feeds <feeds@localhost>")
		msg.SetHeader("Subject", message.EncodeWord(title))
		msg.SetHeader("Content-Type", "text/html; charset=\"UTF-8\"")

		_, err = md.CreateMail(msg)
		if err != nil {
			return err
		}

		fmt.Printf("  %s\n", title)
		cache.data[postId] = true
	}

	return nil
}

func main() {
	url_ := os.Args[1]

	cache.path = path.Join(os.Getenv("HOME"), ".cache", "rss2maildir", strings.Replace(url_, "/", "_", -1))
	cache.data = make(map[string]bool)

	err := cache.load()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Warning: can't read cache: %s\n", err.Error())
	}

	err = process(url_)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Can't process feed: %s\n", err.Error())
		os.Exit(1)
	}

	err = cache.dump()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Can't write cache: %s\n", err.Error())
		os.Exit(1)
	}
}
<commit_msg>Alway add an adress email in the From header<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/sloonz/cfeedparser"
	"github.com/sloonz/go-maildir"
	"github.com/sloonz/go-mime-message"
	"github.com/sloonz/go-qprintable"
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"regexp"
	"strings"
	"time"
)

type Cache struct {
	data map[string]bool
	path string
}

func (c *Cache) load() error {
	cacheFile, err := os.Open(c.path)
	if err != nil {
		return err
	}

	data, err := ioutil.ReadAll(cacheFile)
	if err != nil {
		return err
	}

	return json.Unmarshal(data, &c.data)
}

func (c *Cache) dump() error {
	cacheFile, err := os.Create(c.path + ".new")
	if err != nil {
		return err
	}
	defer cacheFile.Close()

	enc := json.NewEncoder(cacheFile)
	if err = enc.Encode(c.data); err != nil {
		return err
	}

	return os.Rename(c.path+".new", c.path)
}

var cache Cache

func firstNonEmpty(s ...string) string {
	var val string
	for _, val = range s {
		if val != "" {
			break
		}
	}
	return val
}

func getRFC822Date(e *feedparser.Entry) string {
	emptyTime := time.Time{}
	if e.PublicationDateParsed != emptyTime {
		return e.PublicationDateParsed.Format(time.RFC1123Z)
	}
	if e.ModificationDateParsed != emptyTime {
		return e.ModificationDateParsed.Format(time.RFC1123Z)
	}
	if e.PublicationDate != "" {
		return e.PublicationDate
	}
	if e.ModificationDate != "" {
		return e.ModificationDate
	}
	return time.Now().UTC().Format(time.RFC1123Z)
}

func getFrom(e *feedparser.Entry) string {
	name := strings.TrimSpace(message.EncodeWord(firstNonEmpty(e.Author.Name, e.Author.Uri, e.Author.Text)))
	if e.Author.Email != "" {
		name += " <" + strings.TrimSpace(e.Author.Email) + ">"
	} else {
		name += " <noreply@localhost>"
	}
	return name
}

var convertEOLReg = regexp.MustCompile("\r\n?")

func convertEOL(s string) string {
	return convertEOLReg.ReplaceAllString(s, "\n")
}

func process(rawUrl string) error {
	url_, err := url.Parse(rawUrl)
	if err != nil {
		return err
	}

	md, err := maildir.New(".", false)
	if err != nil {
		return err
	}

	feed, err := feedparser.ParseURL(url_)
	if err != nil {
		return err
	}

	fmt.Printf("[%s]\n", feed.Title)
	for _, entry := range feed.Entries {
		postId := firstNonEmpty(entry.Id, entry.Link, entry.PublicationDate+":"+entry.Title)
		if _, hasId := cache.data[postId]; hasId {
			continue
		}

		body := convertEOL(firstNonEmpty(entry.Content, entry.Summary))
		body += "\n
View post
\n"

		title := strings.TrimSpace(entry.Title)
		msg := message.NewTextMessage(qprintable.UnixTextEncoding, bytes.NewBufferString(body))
		msg.SetHeader("Date", getRFC822Date(&entry))
		msg.SetHeader("From", getFrom(&entry))
		msg.SetHeader("To", "Feeds <feeds@localhost>")
		msg.SetHeader("Subject", message.EncodeWord(title))
		msg.SetHeader("Content-Type", "text/html; charset=\"UTF-8\"")

		_, err = md.CreateMail(msg)
		if err != nil {
			return err
		}

		fmt.Printf("  %s\n", title)
		cache.data[postId] = true
	}

	return nil
}

func main() {
	url_ := os.Args[1]

	cache.path = path.Join(os.Getenv("HOME"), ".cache", "rss2maildir", strings.Replace(url_, "/", "_", -1))
	cache.data = make(map[string]bool)

	err := cache.load()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Warning: can't read cache: %s\n", err.Error())
	}

	err = process(url_)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Can't process feed: %s\n", err.Error())
		os.Exit(1)
	}

	err = cache.dump()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Can't write cache: %s\n", err.Error())
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package tunnel

import (
	"sync"
	"testing"

	"github.com/micro/go-micro/transport"
)

// testAccept will accept connections on the transport, create a new link and tunnel on top
func testAccept(t *testing.T, tun Tunnel, wg *sync.WaitGroup) {
	// listen on some virtual address
	tl, err := tun.Listen("test-tunnel")
	if err != nil {
		t.Fatal(err)
	}

	// accept a connection
	c, err := tl.Accept()
	if err != nil {
		t.Fatal(err)
	}

	// get a message
	for {
		m := new(transport.Message)
		if err := c.Recv(m); err != nil {
			t.Fatal(err)
		}
		wg.Done()
		return
	}
}

// testSend will create a new link to an address and then a tunnel on top
func testSend(t *testing.T, tun Tunnel) {
	// dial a new session
	c, err := tun.Dial("test-tunnel")
	if err != nil {
		t.Fatal(err)
	}
	defer c.Close()

	m := transport.Message{
		Header: map[string]string{
			"test": "header",
		},
	}

	if err := c.Send(&m); err != nil {
		t.Fatal(err)
	}
}

func TestTunnel(t *testing.T) {
	// create a new listener
	tun := NewTunnel(Nodes(":9096"))
	err := tun.Connect()
	if err != nil {
		t.Fatal(err)
	}
	defer tun.Close()

	var wg sync.WaitGroup

	// start accepting connections
	wg.Add(1)
	go testAccept(t, tun, &wg)

	// send a message
	testSend(t, tun)

	// wait until message is received
	wg.Wait()
}

func TestTwoTunnel(t *testing.T) {
	// create a new tunnel client
	tunA := NewTunnel(
		Address(":9096"),
		Nodes(":9097"),
	)

	// create a new tunnel server
	tunB := NewTunnel(
		Address(":9097"),
	)

	// start tunB
	err := tunB.Connect()
	if err != nil {
		t.Fatal(err)
	}
	defer tunB.Close()

	// start tunA
	err = tunA.Connect()
	if err != nil {
		t.Fatal(err)
	}
	defer tunA.Close()

	var wg sync.WaitGroup

	// start accepting connections
	wg.Add(1)
	go testAccept(t, tunB, &wg)

	// send a message
	testSend(t, tunA)

	// wait until done
	wg.Wait()
}
<commit_msg>Fix travis test?<commit_after>package tunnel

import (
	"sync"
	"testing"

	"github.com/micro/go-micro/transport"
)

// testAccept will accept connections on the transport, create a new link and tunnel on top
func testAccept(t *testing.T, tun Tunnel, wg *sync.WaitGroup) {
	// listen on some virtual address
	tl, err := tun.Listen("test-tunnel")
	if err != nil {
		t.Fatal(err)
	}

	// accept a connection
	c, err := tl.Accept()
	if err != nil {
		t.Fatal(err)
	}

	// get a message
	for {
		m := new(transport.Message)
		if err := c.Recv(m); err != nil {
			t.Fatal(err)
		}
		wg.Done()
		return
	}
}

// testSend will create a new link to an address and then a tunnel on top
func testSend(t *testing.T, tun Tunnel) {
	// dial a new session
	c, err := tun.Dial("test-tunnel")
	if err != nil {
		t.Fatal(err)
	}
	defer c.Close()

	m := transport.Message{
		Header: map[string]string{
			"test": "header",
		},
	}

	if err := c.Send(&m); err != nil {
		t.Fatal(err)
	}
}

func TestTunnel(t *testing.T) {
	// create a new listener
	tun := NewTunnel(Nodes("127.0.0.1:9096"))
	err := tun.Connect()
	if err != nil {
		t.Fatal(err)
	}
	defer tun.Close()

	var wg sync.WaitGroup

	// start accepting connections
	wg.Add(1)
	go testAccept(t, tun, &wg)

	// send a message
	testSend(t, tun)

	// wait until message is received
	wg.Wait()
}

func TestTwoTunnel(t *testing.T) {
	// create a new tunnel client
	tunA := NewTunnel(
		Address("127.0.0.1:9096"),
		Nodes("127.0.0.1:9097"),
	)

	// create a new tunnel server
	tunB := NewTunnel(
		Address("127.0.0.1:9097"),
	)

	// start tunB
	err := tunB.Connect()
	if err != nil {
		t.Fatal(err)
	}
	defer tunB.Close()

	// start tunA
	err = tunA.Connect()
	if err != nil {
		t.Fatal(err)
	}
	defer tunA.Close()

	var wg sync.WaitGroup

	// start accepting connections
	wg.Add(1)
	go testAccept(t, tunB, &wg)

	// send a message
	testSend(t, tunA)

	// wait until done
	wg.Wait()
}
<|endoftext|>


<commit_before>// Copyright (c) 2017-2018 Tigera, Inc. All rights reserved.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package clientv3

import (
	"context"
	"fmt"
	"strings"

	"github.com/projectcalico/libcalico-go/lib/apiconfig"
	apiv3 "github.com/projectcalico/libcalico-go/lib/apis/v3"
	"github.com/projectcalico/libcalico-go/lib/options"
	validator "github.com/projectcalico/libcalico-go/lib/validator/v3"
	"github.com/projectcalico/libcalico-go/lib/watch"
)

// GlobalNetworkPolicyInterface has methods to work with GlobalNetworkPolicy resources.
type GlobalNetworkPolicyInterface interface {
	Create(ctx context.Context, res *apiv3.GlobalNetworkPolicy, opts options.SetOptions) (*apiv3.GlobalNetworkPolicy, error)
	Update(ctx context.Context, res *apiv3.GlobalNetworkPolicy, opts options.SetOptions) (*apiv3.GlobalNetworkPolicy, error)
	Delete(ctx context.Context, name string, opts options.DeleteOptions) (*apiv3.GlobalNetworkPolicy, error)
	Get(ctx context.Context, name string, opts options.GetOptions) (*apiv3.GlobalNetworkPolicy, error)
	List(ctx context.Context, opts options.ListOptions) (*apiv3.GlobalNetworkPolicyList, error)
	Watch(ctx context.Context, opts options.ListOptions) (watch.Interface, error)
}

// globalNetworkPolicies implements GlobalNetworkPolicyInterface
type globalNetworkPolicies struct {
	client client
}

// Create takes the representation of a GlobalNetworkPolicy and creates it.  Returns the stored
// representation of the GlobalNetworkPolicy, and an error, if there is any.
func (r globalNetworkPolicies) Create(ctx context.Context, res *apiv3.GlobalNetworkPolicy, opts options.SetOptions) (*apiv3.GlobalNetworkPolicy, error) {
	if res != nil {
		// Since we're about to default some fields, take a (shallow) copy of the input data
		// before we do so.
		resCopy := *res
		res = &resCopy
	}
	defaultPolicyTypesField(res.Spec.Ingress, res.Spec.Egress, &res.Spec.Types)

	if err := validator.Validate(res); err != nil {
		return nil, err
	}

	// Check if it is permitted to create the network policy with the specific alpha feature support.
	if err := r.validateAlphaFeatures(res); err != nil {
		return nil, err
	}

	// Properly prefix the name
	res.GetObjectMeta().SetName(convertPolicyNameForStorage(res.GetObjectMeta().GetName()))
	out, err := r.client.resources.Create(ctx, opts, apiv3.KindGlobalNetworkPolicy, res)
	if out != nil {
		// Remove the prefix out of the returned policy name.
		out.GetObjectMeta().SetName(convertPolicyNameFromStorage(out.GetObjectMeta().GetName()))
		return out.(*apiv3.GlobalNetworkPolicy), err
	}

	// Remove the prefix out of the returned policy name.
	res.GetObjectMeta().SetName(convertPolicyNameFromStorage(res.GetObjectMeta().GetName()))
	return nil, err
}

// Update takes the representation of a GlobalNetworkPolicy and updates it. Returns the stored
// representation of the GlobalNetworkPolicy, and an error, if there is any.
func (r globalNetworkPolicies) Update(ctx context.Context, res *apiv3.GlobalNetworkPolicy, opts options.SetOptions) (*apiv3.GlobalNetworkPolicy, error) {
	if res != nil {
		// Since we're about to default some fields, take a (shallow) copy of the input data
		// before we do so.
		resCopy := *res
		res = &resCopy
	}
	defaultPolicyTypesField(res.Spec.Ingress, res.Spec.Egress, &res.Spec.Types)

	if err := validator.Validate(res); err != nil {
		return nil, err
	}

	// Check if it is permitted to create the network policy with the specific alpha feature support.
	if err := r.validateAlphaFeatures(res); err != nil {
		return nil, err
	}

	// Properly prefix the name
	res.GetObjectMeta().SetName(convertPolicyNameForStorage(res.GetObjectMeta().GetName()))
	out, err := r.client.resources.Update(ctx, opts, apiv3.KindGlobalNetworkPolicy, res)
	if out != nil {
		// Remove the prefix out of the returned policy name.
		out.GetObjectMeta().SetName(convertPolicyNameFromStorage(out.GetObjectMeta().GetName()))
		return out.(*apiv3.GlobalNetworkPolicy), err
	}

	// Remove the prefix out of the returned policy name.
	res.GetObjectMeta().SetName(convertPolicyNameFromStorage(res.GetObjectMeta().GetName()))
	return nil, err
}

// Delete takes name of the GlobalNetworkPolicy and deletes it. Returns an error if one occurs.
func (r globalNetworkPolicies) Delete(ctx context.Context, name string, opts options.DeleteOptions) (*apiv3.GlobalNetworkPolicy, error) {
	out, err := r.client.resources.Delete(ctx, opts, apiv3.KindGlobalNetworkPolicy, noNamespace, convertPolicyNameForStorage(name))
	if out != nil {
		// Remove the prefix out of the returned policy name.
		out.GetObjectMeta().SetName(convertPolicyNameFromStorage(out.GetObjectMeta().GetName()))
		return out.(*apiv3.GlobalNetworkPolicy), err
	}
	return nil, err
}

// Get takes name of the GlobalNetworkPolicy, and returns the corresponding GlobalNetworkPolicy object,
// and an error if there is any.
func (r globalNetworkPolicies) Get(ctx context.Context, name string, opts options.GetOptions) (*apiv3.GlobalNetworkPolicy, error) {
	out, err := r.client.resources.Get(ctx, opts, apiv3.KindGlobalNetworkPolicy, noNamespace, convertPolicyNameForStorage(name))
	if out != nil {
		// Remove the prefix out of the returned policy name.
		out.GetObjectMeta().SetName(convertPolicyNameFromStorage(out.GetObjectMeta().GetName()))
		return out.(*apiv3.GlobalNetworkPolicy), err
	}
	return nil, err
}

// List returns the list of GlobalNetworkPolicy objects that match the supplied options.
func (r globalNetworkPolicies) List(ctx context.Context, opts options.ListOptions) (*apiv3.GlobalNetworkPolicyList, error) {
	res := &apiv3.GlobalNetworkPolicyList{}
	// Add the name prefix if name is provided
	if opts.Name != "" {
		opts.Name = convertPolicyNameForStorage(opts.Name)
	}

	if err := r.client.resources.List(ctx, opts, apiv3.KindGlobalNetworkPolicy, apiv3.KindGlobalNetworkPolicyList, res); err != nil {
		return nil, err
	}

	// Remove the prefix off of each policy name
	for i, _ := range res.Items {
		name := res.Items[i].GetObjectMeta().GetName()
		res.Items[i].GetObjectMeta().SetName(convertPolicyNameFromStorage(name))
	}

	return res, nil
}

// Watch returns a watch.Interface that watches the globalNetworkPolicies that match the
// supplied options.
func (r globalNetworkPolicies) Watch(ctx context.Context, opts options.ListOptions) (watch.Interface, error) {
	// Add the name prefix if name is provided
	if opts.Name != "" {
		opts.Name = convertPolicyNameForStorage(opts.Name)
	}

	return r.client.resources.Watch(ctx, opts, apiv3.KindGlobalNetworkPolicy, &policyConverter{})
}

func (r globalNetworkPolicies) validateAlphaFeatures(res *apiv3.GlobalNetworkPolicy) error {
	if apiconfig.IsAlphaFeatureSet(r.client.config.Spec.AlphaFeatures, apiconfig.AlphaFeatureSA) == false {
		err := validator.ValidateNoServiceAccountRules(res.Spec.Ingress, res.Spec.Egress)
		if err != nil {
			return fmt.Errorf("Global NP %s: %s", res.GetObjectMeta().GetName(), err.Error())
		}
	}

	if apiconfig.IsAlphaFeatureSet(r.client.config.Spec.AlphaFeatures, apiconfig.AlphaFeatureHTTP) == false {
		err := validator.ValidateNoHTTPRules(res.Spec.Ingress, res.Spec.Egress)
		if err != nil {
			return fmt.Errorf("Global NP %s: %s", res.GetObjectMeta().GetName(), err.Error())
		}
	}

	return nil
}

func defaultPolicyTypesField(ingressRules, egressRules []apiv3.Rule, types *[]apiv3.PolicyType) {
	if len(*types) == 0 {
		// Default the Types field according to what inbound and outbound rules are present
		// in the policy.
		if len(egressRules) == 0 {
			// Policy has no egress rules, so apply this policy to ingress only.  (Note:
			// intentionally including the case where the policy also has no ingress
			// rules.)
			*types = []apiv3.PolicyType{apiv3.PolicyTypeIngress}
		} else if len(ingressRules) == 0 {
			// Policy has egress rules but no ingress rules, so apply this policy to
			// egress only.
			*types = []apiv3.PolicyType{apiv3.PolicyTypeEgress}
		} else {
			// Policy has both ingress and egress rules, so apply this policy to both
			// ingress and egress.
			*types = []apiv3.PolicyType{apiv3.PolicyTypeIngress, apiv3.PolicyTypeEgress}
		}
	}
}

func convertPolicyNameForStorage(name string) string {
	// Do nothing on names prefixed with "knp."
	if strings.HasPrefix(name, "knp.") {
		return name
	}
	return "default." + name
}

func convertPolicyNameFromStorage(name string) string {
	// Do nothing on names prefixed with "knp."
	if strings.HasPrefix(name, "knp.") {
		return name
	}
	parts := strings.SplitN(name, ".", 2)
	return parts[len(parts)-1]
}

type policyConverter struct{}

func (pc *policyConverter) Convert(r resource) resource {
	r.GetObjectMeta().SetName(convertPolicyNameFromStorage(r.GetObjectMeta().GetName()))
	return r
}
<commit_msg>Don't add/remove 'ossg.default.' NP prefix when to/from storage<commit_after>// Copyright (c) 2017-2018 Tigera, Inc. All rights reserved.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package clientv3

import (
	"context"
	"fmt"
	"strings"

	"github.com/projectcalico/libcalico-go/lib/apiconfig"
	apiv3 "github.com/projectcalico/libcalico-go/lib/apis/v3"
	"github.com/projectcalico/libcalico-go/lib/options"
	validator "github.com/projectcalico/libcalico-go/lib/validator/v3"
	"github.com/projectcalico/libcalico-go/lib/watch"
)

// GlobalNetworkPolicyInterface has methods to work with GlobalNetworkPolicy resources.
type GlobalNetworkPolicyInterface interface {
	Create(ctx context.Context, res *apiv3.GlobalNetworkPolicy, opts options.SetOptions) (*apiv3.GlobalNetworkPolicy, error)
	Update(ctx context.Context, res *apiv3.GlobalNetworkPolicy, opts options.SetOptions) (*apiv3.GlobalNetworkPolicy, error)
	Delete(ctx context.Context, name string, opts options.DeleteOptions) (*apiv3.GlobalNetworkPolicy, error)
	Get(ctx context.Context, name string, opts options.GetOptions) (*apiv3.GlobalNetworkPolicy, error)
	List(ctx context.Context, opts options.ListOptions) (*apiv3.GlobalNetworkPolicyList, error)
	Watch(ctx context.Context, opts options.ListOptions) (watch.Interface, error)
}

// globalNetworkPolicies implements GlobalNetworkPolicyInterface
type globalNetworkPolicies struct {
	client client
}

// Create takes the representation of a GlobalNetworkPolicy and creates it.  Returns the stored
// representation of the GlobalNetworkPolicy, and an error, if there is any.
func (r globalNetworkPolicies) Create(ctx context.Context, res *apiv3.GlobalNetworkPolicy, opts options.SetOptions) (*apiv3.GlobalNetworkPolicy, error) {
	if res != nil {
		// Since we're about to default some fields, take a (shallow) copy of the input data
		// before we do so.
		resCopy := *res
		res = &resCopy
	}
	defaultPolicyTypesField(res.Spec.Ingress, res.Spec.Egress, &res.Spec.Types)

	if err := validator.Validate(res); err != nil {
		return nil, err
	}

	// Check if it is permitted to create the network policy with the specific alpha feature support.
	if err := r.validateAlphaFeatures(res); err != nil {
		return nil, err
	}

	// Properly prefix the name
	res.GetObjectMeta().SetName(convertPolicyNameForStorage(res.GetObjectMeta().GetName()))
	out, err := r.client.resources.Create(ctx, opts, apiv3.KindGlobalNetworkPolicy, res)
	if out != nil {
		// Remove the prefix out of the returned policy name.
		out.GetObjectMeta().SetName(convertPolicyNameFromStorage(out.GetObjectMeta().GetName()))
		return out.(*apiv3.GlobalNetworkPolicy), err
	}

	// Remove the prefix out of the returned policy name.
	res.GetObjectMeta().SetName(convertPolicyNameFromStorage(res.GetObjectMeta().GetName()))
	return nil, err
}

// Update takes the representation of a GlobalNetworkPolicy and updates it. Returns the stored
// representation of the GlobalNetworkPolicy, and an error, if there is any.
func (r globalNetworkPolicies) Update(ctx context.Context, res *apiv3.GlobalNetworkPolicy, opts options.SetOptions) (*apiv3.GlobalNetworkPolicy, error) {
	if res != nil {
		// Since we're about to default some fields, take a (shallow) copy of the input data
		// before we do so.
		resCopy := *res
		res = &resCopy
	}
	defaultPolicyTypesField(res.Spec.Ingress, res.Spec.Egress, &res.Spec.Types)

	if err := validator.Validate(res); err != nil {
		return nil, err
	}

	// Check if it is permitted to create the network policy with the specific alpha feature support.
	if err := r.validateAlphaFeatures(res); err != nil {
		return nil, err
	}

	// Properly prefix the name
	res.GetObjectMeta().SetName(convertPolicyNameForStorage(res.GetObjectMeta().GetName()))
	out, err := r.client.resources.Update(ctx, opts, apiv3.KindGlobalNetworkPolicy, res)
	if out != nil {
		// Remove the prefix out of the returned policy name.
		out.GetObjectMeta().SetName(convertPolicyNameFromStorage(out.GetObjectMeta().GetName()))
		return out.(*apiv3.GlobalNetworkPolicy), err
	}

	// Remove the prefix out of the returned policy name.
	res.GetObjectMeta().SetName(convertPolicyNameFromStorage(res.GetObjectMeta().GetName()))
	return nil, err
}

// Delete takes name of the GlobalNetworkPolicy and deletes it. Returns an error if one occurs.
func (r globalNetworkPolicies) Delete(ctx context.Context, name string, opts options.DeleteOptions) (*apiv3.GlobalNetworkPolicy, error) {
	out, err := r.client.resources.Delete(ctx, opts, apiv3.KindGlobalNetworkPolicy, noNamespace, convertPolicyNameForStorage(name))
	if out != nil {
		// Remove the prefix out of the returned policy name.
		out.GetObjectMeta().SetName(convertPolicyNameFromStorage(out.GetObjectMeta().GetName()))
		return out.(*apiv3.GlobalNetworkPolicy), err
	}
	return nil, err
}

// Get takes name of the GlobalNetworkPolicy, and returns the corresponding GlobalNetworkPolicy object,
// and an error if there is any.
func (r globalNetworkPolicies) Get(ctx context.Context, name string, opts options.GetOptions) (*apiv3.GlobalNetworkPolicy, error) {
	out, err := r.client.resources.Get(ctx, opts, apiv3.KindGlobalNetworkPolicy, noNamespace, convertPolicyNameForStorage(name))
	if out != nil {
		// Remove the prefix out of the returned policy name.
		out.GetObjectMeta().SetName(convertPolicyNameFromStorage(out.GetObjectMeta().GetName()))
		return out.(*apiv3.GlobalNetworkPolicy), err
	}
	return nil, err
}

// List returns the list of GlobalNetworkPolicy objects that match the supplied options.
func (r globalNetworkPolicies) List(ctx context.Context, opts options.ListOptions) (*apiv3.GlobalNetworkPolicyList, error) {
	res := &apiv3.GlobalNetworkPolicyList{}
	// Add the name prefix if name is provided
	if opts.Name != "" {
		opts.Name = convertPolicyNameForStorage(opts.Name)
	}

	if err := r.client.resources.List(ctx, opts, apiv3.KindGlobalNetworkPolicy, apiv3.KindGlobalNetworkPolicyList, res); err != nil {
		return nil, err
	}

	// Remove the prefix off of each policy name
	for i, _ := range res.Items {
		name := res.Items[i].GetObjectMeta().GetName()
		res.Items[i].GetObjectMeta().SetName(convertPolicyNameFromStorage(name))
	}

	return res, nil
}

// Watch returns a watch.Interface that watches the globalNetworkPolicies that match the
// supplied options.
func (r globalNetworkPolicies) Watch(ctx context.Context, opts options.ListOptions) (watch.Interface, error) {
	// Add the name prefix if name is provided
	if opts.Name != "" {
		opts.Name = convertPolicyNameForStorage(opts.Name)
	}

	return r.client.resources.Watch(ctx, opts, apiv3.KindGlobalNetworkPolicy, &policyConverter{})
}

func (r globalNetworkPolicies) validateAlphaFeatures(res *apiv3.GlobalNetworkPolicy) error {
	if apiconfig.IsAlphaFeatureSet(r.client.config.Spec.AlphaFeatures, apiconfig.AlphaFeatureSA) == false {
		err := validator.ValidateNoServiceAccountRules(res.Spec.Ingress, res.Spec.Egress)
		if err != nil {
			return fmt.Errorf("Global NP %s: %s", res.GetObjectMeta().GetName(), err.Error())
		}
	}

	if apiconfig.IsAlphaFeatureSet(r.client.config.Spec.AlphaFeatures, apiconfig.AlphaFeatureHTTP) == false {
		err := validator.ValidateNoHTTPRules(res.Spec.Ingress, res.Spec.Egress)
		if err != nil {
			return fmt.Errorf("Global NP %s: %s", res.GetObjectMeta().GetName(), err.Error())
		}
	}

	return nil
}

func defaultPolicyTypesField(ingressRules, egressRules []apiv3.Rule, types *[]apiv3.PolicyType) {
	if len(*types) == 0 {
		// Default the Types field according to what inbound and outbound rules are present
		// in the policy.
		if len(egressRules) == 0 {
			// Policy has no egress rules, so apply this policy to ingress only.  (Note:
			// intentionally including the case where the policy also has no ingress
			// rules.)
			*types = []apiv3.PolicyType{apiv3.PolicyTypeIngress}
		} else if len(ingressRules) == 0 {
			// Policy has egress rules but no ingress rules, so apply this policy to
			// egress only.
			*types = []apiv3.PolicyType{apiv3.PolicyTypeEgress}
		} else {
			// Policy has both ingress and egress rules, so apply this policy to both
			// ingress and egress.
			*types = []apiv3.PolicyType{apiv3.PolicyTypeIngress, apiv3.PolicyTypeEgress}
		}
	}
}

func convertPolicyNameForStorage(name string) string {
	// Do nothing on names prefixed with "knp."
	if strings.HasPrefix(name, "knp.") {
		return name
	}
	// Similarly for "ossg."
	if strings.HasPrefix(name, "ossg.") {
		return name
	}
	return "default." + name
}

func convertPolicyNameFromStorage(name string) string {
	// Do nothing on names prefixed with "knp."
	if strings.HasPrefix(name, "knp.") {
		return name
	}
	// Similarly for "ossg."
	if strings.HasPrefix(name, "ossg.") {
		return name
	}
	parts := strings.SplitN(name, ".", 2)
	return parts[len(parts)-1]
}

type policyConverter struct{}

func (pc *policyConverter) Convert(r resource) resource {
	r.GetObjectMeta().SetName(convertPolicyNameFromStorage(r.GetObjectMeta().GetName()))
	return r
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"net/url"
	"testing"
	"time"
)

// start when there is no config should report that fact right away
func TestStartWithoutConfig(t *testing.T) {
	done := start("/not/notaconfig.conf")
	message := <- done
	if message != "Could not read file at /not/notaconfig.conf" {
		t.Errorf("did not report the file that was missing")
	}
}

// start when there is a bad config should report that fact right away
func TestStartWithBadConfig(t *testing.T) {
	done := start("./test_bad.conf")
	message := <- done
	if message != "Could not decode config" {
		t.Errorf("incorrect bad config mesage\n\"%s\"", message)
	}
}

// start with a valid file should not be done right away
func TestStartWithGoodConfig(t *testing.T) {
	done := start("./test_good.conf")
	select {
	case message := <-done:
		t.Errorf("done with message\n \"%s\"", message)
	case <-time.After(time.Millisecond * 50):
		fmt.Print("Stayed up with good config\n")
	}
}

// Stabilizer.ServeHTTP should return the first good response
func TestStabilizerReturnsFirstResponse(t *testing.T) {
	// mock handler that first errors, then takes a long time then returns a
	// a good response
	reqCount := 0
	mockHandler := func(w http.ResponseWriter, r *http.Request) {
		reqCount++
		if reqCount % 3 == 1 {
			http.Error(w, "test error", 1234567890)
		}
		if reqCount % 3 == 2 {
			time.Sleep(100 * time.Millisecond)
			fmt.Fprintf(w, "slow response")
		}
		if reqCount % 3 == 0 {
			fmt.Fprintf(w, "fast response")
		}
	}

	mockUnstableBackend := httptest.NewServer(http.HandlerFunc(mockHandler))
	defer mockUnstableBackend.Close()
	u, err := url.Parse(mockUnstableBackend.URL)
	if err != nil { t.Errorf("error parsing backend url test broken") }
	testStabilizer := &Stabilizer{u, 4}
	testStableServer := httptest.NewServer(
		http.TimeoutHandler(testStabilizer, 5 * time.Second, "timeout"),
	)
	defer testStableServer.Close()

	// make many requests and make sure they are all the fast response
	for i := 0; i < 50; i++ {
		res, err := http.Get(testStableServer.URL)
		if err != nil { t.Errorf("error response from stable server") }

		message, err := ioutil.ReadAll(res.Body)
		if err != nil { t.Errorf("error reading response body from stable server") }
		res.Body.Close()

		// ensure that all responses are the fast response
		if string(message) != "fast response" {
			t.Errorf(string(message))
		}
	}
}

// TestCanPass proves that tests are running
func TestCanPass(t *testing.T) {
	if true != true {
		t.Errorf("true is not true,\ncheck your premises,\n consider clojure?")
	}
}
<commit_msg>pause slightly before each set of requests so the pipes can clear<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"net/url"
	"testing"
	"time"
)

// start when there is no config should report that fact right away
func TestStartWithoutConfig(t *testing.T) {
	done := start("/not/notaconfig.conf")
	message := <- done
	if message != "Could not read file at /not/notaconfig.conf" {
		t.Errorf("did not report the file that was missing")
	}
}

// start when there is a bad config should report that fact right away
func TestStartWithBadConfig(t *testing.T) {
	done := start("./test_bad.conf")
	message := <- done
	if message != "Could not decode config" {
		t.Errorf("incorrect bad config mesage\n\"%s\"", message)
	}
}

// start with a valid file should not be done right away
func TestStartWithGoodConfig(t *testing.T) {
	done := start("./test_good.conf")
	select {
	case message := <-done:
		t.Errorf("done with message\n \"%s\"", message)
	case <-time.After(time.Millisecond * 50):
		fmt.Print("Stayed up with good config\n")
	}
}

// Stabilizer.ServeHTTP should return the first good response
func TestStabilizerReturnsFirstResponse(t *testing.T) {
	// mock handler that first errors, then takes a long time then returns a
	// a good response
	reqCount := 0
	mockHandler := func(w http.ResponseWriter, r *http.Request) {
		reqCount++
		if reqCount % 3 == 1 {
			http.Error(w, "test error", 1234567890)
		}
		if reqCount % 3 == 2 {
			time.Sleep(100 * time.Millisecond)
			fmt.Fprintf(w, "slow response")
		}
		if reqCount % 3 == 0 {
			fmt.Fprintf(w, "fast response")
		}
	}

	mockUnstableBackend := httptest.NewServer(http.HandlerFunc(mockHandler))
	defer mockUnstableBackend.Close()
	u, err := url.Parse(mockUnstableBackend.URL)
	if err != nil { t.Errorf("error parsing backend url test broken") }
	testStabilizer := &Stabilizer{u, 4}
	testStableServer := httptest.NewServer(
		http.TimeoutHandler(testStabilizer, 5 * time.Second, "timeout"),
	)
	defer testStableServer.Close()

	// make many requests and make sure they are all the fast response
	for i := 0; i < 100; i++ {
		res, err := http.Get(testStableServer.URL)
		if err != nil { t.Errorf("error response from stable server") }

		message, err := ioutil.ReadAll(res.Body)
		if err != nil { t.Errorf("error reading response body from stable server") }
		res.Body.Close()

		// ensure that all responses are the fast response
		if string(message) != "fast response" {
			t.Errorf(string(message))
		}

		time.Sleep(10)
	}
}

// TestCanPass proves that tests are running
func TestCanPass(t *testing.T) {
	if true != true {
		t.Errorf("true is not true,\ncheck your premises,\n consider clojure?")
	}
}
<|endoftext|>


<commit_before>package download

import (
	"fmt"
	"io"
	"net"
	"net/http"
	"os"
	"golang.org/x/sync/errgroup"
	"github.com/pivotal-cf/go-pivnet/logger"
	"syscall"
)

//go:generate counterfeiter -o ./fakes/ranger.go --fake-name Ranger . ranger
type ranger interface {
	BuildRange(contentLength int64) ([]Range, error)
}

//go:generate counterfeiter -o ./fakes/http_client.go --fake-name HTTPClient . httpClient
type httpClient interface {
	Do(*http.Request) (*http.Response, error)
}

type downloadLinkFetcher interface {
	NewDownloadLink() (string, error)
}

//go:generate counterfeiter -o ./fakes/bar.go --fake-name Bar . bar
type bar interface {
	SetTotal(contentLength int64)
	SetOutput(output io.Writer)
	Add(totalWritten int) int
	Kickoff()
	Finish()
	NewProxyReader(reader io.Reader) io.Reader
}

type Client struct {
	HTTPClient httpClient
	Ranger     ranger
	Bar        bar
	Logger     logger.Logger
}

func (c Client) Get(
	location *os.File,
	downloadLinkFetcher downloadLinkFetcher,
	progressWriter io.Writer,
) error {
	contentURL, err := downloadLinkFetcher.NewDownloadLink()
	if err != nil {
		return err
	}

	req, err := http.NewRequest("HEAD", contentURL, nil)
	if err != nil {
		return fmt.Errorf("failed to construct HEAD request: %s", err)
	}

	resp, err := c.HTTPClient.Do(req)
	if err != nil {
		return fmt.Errorf("failed to make HEAD request: %s", err)
	}

	contentURL = resp.Request.URL.String()

	ranges, err := c.Ranger.BuildRange(resp.ContentLength)
	if err != nil {
		return fmt.Errorf("failed to construct range: %s", err)
	}

	c.Bar.SetOutput(progressWriter)
	c.Bar.SetTotal(resp.ContentLength)
	c.Bar.Kickoff()

	defer c.Bar.Finish()
	fileInfo, err := location.Stat()
	if err != nil {
		return fmt.Errorf("failed to read information from output file: %s", err)
	}

	var g errgroup.Group
	for _, r := range ranges {
		byteRange := r

		fileWriter, err := os.OpenFile(location.Name(), os.O_RDWR, fileInfo.Mode())
		if err != nil {
			return fmt.Errorf("failed to open file for writing: %s", err)
		}

		g.Go(func() error {
			err := c.retryableRequest(contentURL, byteRange.HTTPHeader, fileWriter, byteRange.Lower, downloadLinkFetcher)
			if err != nil {
				return fmt.Errorf("failed during retryable request: %s", err)
			}

			return nil
		})
	}

	if err := g.Wait(); err != nil {
		return err
	}

	return nil
}

func (c Client) retryableRequest(contentURL string, rangeHeader http.Header, fileWriter *os.File, startingByte int64, downloadLinkFetcher downloadLinkFetcher) (error) {
	currentURL := contentURL
	defer fileWriter.Close()

	var err error
Retry:
	_, err = fileWriter.Seek(startingByte, 0)
	if err != nil {
		return fmt.Errorf("failed to seek to correct byte of output file: %s", err)
	}

	req, err := http.NewRequest("GET", currentURL, nil)
	if err != nil {
		return err
	}

	req.Header = rangeHeader

	resp, err := c.HTTPClient.Do(req)
	if err != nil {
		if netErr, ok := err.(net.Error); ok {
			if netErr.Temporary() {
				goto Retry
			}
		}

		return fmt.Errorf("download request failed: %s", err)
	}

	defer resp.Body.Close()

	if resp.StatusCode == http.StatusForbidden {
		c.Logger.Debug("received unsuccessful status code: %d", logger.Data{"statusCode": resp.StatusCode})
		currentURL, err = downloadLinkFetcher.NewDownloadLink()
		if err != nil {
			return err
		}
		c.Logger.Debug("fetched new download url: %d", logger.Data{"url": currentURL})

		goto Retry
	}

	if resp.StatusCode != http.StatusPartialContent {
		return fmt.Errorf("during GET unexpected status code was returned: %d", resp.StatusCode)
	}

	var proxyReader io.Reader
	proxyReader = c.Bar.NewProxyReader(resp.Body)

	bytesWritten, err := io.Copy(fileWriter, proxyReader)
	if err != nil {
		if err == io.ErrUnexpectedEOF {
			c.Bar.Add(int(-1 * bytesWritten))
			goto Retry
		}
		operr, _ := err.(*net.OpError)
		if operr.Err.Error() == syscall.ECONNRESET.Error() {
			c.Bar.Add(int(-1 * bytesWritten))
			goto Retry
		}
		return fmt.Errorf("failed to write file during io.Copy: %s", err)
	}

	return nil
}
<commit_msg>Temporarily add debugging to cloudfront requests<commit_after>package download

import (
	"fmt"
	"io"
	"net"
	"net/http"
	"os"
	"golang.org/x/sync/errgroup"
	"github.com/pivotal-cf/go-pivnet/logger"
	"syscall"
)

//go:generate counterfeiter -o ./fakes/ranger.go --fake-name Ranger . ranger
type ranger interface {
	BuildRange(contentLength int64) ([]Range, error)
}

//go:generate counterfeiter -o ./fakes/http_client.go --fake-name HTTPClient . httpClient
type httpClient interface {
	Do(*http.Request) (*http.Response, error)
}

type downloadLinkFetcher interface {
	NewDownloadLink() (string, error)
}

//go:generate counterfeiter -o ./fakes/bar.go --fake-name Bar . bar
type bar interface {
	SetTotal(contentLength int64)
	SetOutput(output io.Writer)
	Add(totalWritten int) int
	Kickoff()
	Finish()
	NewProxyReader(reader io.Reader) io.Reader
}

type Client struct {
	HTTPClient httpClient
	Ranger     ranger
	Bar        bar
	Logger     logger.Logger
}

func (c Client) Get(
	location *os.File,
	downloadLinkFetcher downloadLinkFetcher,
	progressWriter io.Writer,
) error {
	contentURL, err := downloadLinkFetcher.NewDownloadLink()
	if err != nil {
		return err
	}

	req, err := http.NewRequest("HEAD", contentURL, nil)
	if err != nil {
		return fmt.Errorf("failed to construct HEAD request: %s", err)
	}

	resp, err := c.HTTPClient.Do(req)
	if err != nil {
		return fmt.Errorf("failed to make HEAD request: %s", err)
	}

	contentURL = resp.Request.URL.String()

	ranges, err := c.Ranger.BuildRange(resp.ContentLength)
	if err != nil {
		return fmt.Errorf("failed to construct range: %s", resp)
	}

	c.Bar.SetOutput(progressWriter)
	c.Bar.SetTotal(resp.ContentLength)
	c.Bar.Kickoff()

	defer c.Bar.Finish()
	fileInfo, err := location.Stat()
	if err != nil {
		return fmt.Errorf("failed to read information from output file: %s", err)
	}

	var g errgroup.Group
	for _, r := range ranges {
		byteRange := r

		fileWriter, err := os.OpenFile(location.Name(), os.O_RDWR, fileInfo.Mode())
		if err != nil {
			return fmt.Errorf("failed to open file for writing: %s", err)
		}

		g.Go(func() error {
			err := c.retryableRequest(contentURL, byteRange.HTTPHeader, fileWriter, byteRange.Lower, downloadLinkFetcher)
			if err != nil {
				return fmt.Errorf("failed during retryable request: %s", err)
			}

			return nil
		})
	}

	if err := g.Wait(); err != nil {
		return err
	}

	return nil
}

func (c Client) retryableRequest(contentURL string, rangeHeader http.Header, fileWriter *os.File, startingByte int64, downloadLinkFetcher downloadLinkFetcher) (error) {
	currentURL := contentURL
	defer fileWriter.Close()

	var err error
Retry:
	_, err = fileWriter.Seek(startingByte, 0)
	if err != nil {
		return fmt.Errorf("failed to seek to correct byte of output file: %s", err)
	}

	req, err := http.NewRequest("GET", currentURL, nil)
	if err != nil {
		return err
	}

	req.Header = rangeHeader

	resp, err := c.HTTPClient.Do(req)
	if err != nil {
		if netErr, ok := err.(net.Error); ok {
			if netErr.Temporary() {
				goto Retry
			}
		}

		return fmt.Errorf("download request failed: %s", err)
	}

	defer resp.Body.Close()

	if resp.StatusCode == http.StatusForbidden {
		c.Logger.Debug("received unsuccessful status code: %d", logger.Data{"statusCode": resp.StatusCode})
		currentURL, err = downloadLinkFetcher.NewDownloadLink()
		if err != nil {
			return err
		}
		c.Logger.Debug("fetched new download url: %d", logger.Data{"url": currentURL})

		goto Retry
	}

	if resp.StatusCode != http.StatusPartialContent {
		return fmt.Errorf("during GET unexpected status code was returned: %d", resp.StatusCode)
	}

	var proxyReader io.Reader
	proxyReader = c.Bar.NewProxyReader(resp.Body)

	bytesWritten, err := io.Copy(fileWriter, proxyReader)
	if err != nil {
		if err == io.ErrUnexpectedEOF {
			c.Bar.Add(int(-1 * bytesWritten))
			goto Retry
		}
		operr, _ := err.(*net.OpError)
		if operr.Err.Error() == syscall.ECONNRESET.Error() {
			c.Bar.Add(int(-1 * bytesWritten))
			goto Retry
		}
		return fmt.Errorf("failed to write file during io.Copy: %s", err)
	}

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
	"fmt"
	"io/ioutil"
	"os"
	"sort"
	"strings"
	"testing"

	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/test-infra/prow/config"
	"k8s.io/test-infra/prow/config/org"
	"k8s.io/test-infra/prow/github"

	"github.com/ghodss/yaml"
)

type owners struct {
	Reviewers []string `json:"reviewers,omitempty"`
	Approvers []string `json:"approvers"`
}

func readInto(path string, i interface{}) error {
	buf, err := ioutil.ReadFile(path)
	if err != nil {
		return fmt.Errorf("read: %v", err)
	}
	if err := yaml.Unmarshal(buf, i); err != nil {
		return fmt.Errorf("unmarshal: %v", err)
	}
	return nil
}

func loadOwners(dir string) (*owners, error) {
	var own owners
	if err := readInto(dir+"/OWNERS", &own); err != nil {
		return nil, err
	}
	return &own, nil
}

func loadOrg(dir string) (*org.Config, error) {
	var cfg org.Config
	if err := readInto(dir+"/org.yaml", &cfg); err != nil {
		return nil, err
	}
	return &cfg, nil
}

func testDuplicates(list sets.String) error {
	found := sets.String{}
	dups := sets.String{}
	all := list.List()
	for _, i := range all {
		if found.Has(i) {
			dups.Insert(i)
		}
		found.Insert(i)
	}
	if n := len(dups); n > 0 {
		return fmt.Errorf("%d duplicate names: %s", n, strings.Join(dups.List(), ", "))
	}
	return nil
}

func isSorted(list []string) bool {
	items := make([]string, len(list))
	for _, l := range list {
		items = append(items, strings.ToLower(l))
	}

	return sort.StringsAreSorted(items)
}

func normalize(s sets.String) sets.String {
	out := sets.String{}
	for i := range s {
		out.Insert(github.NormLogin(i))
	}
	return out
}

// testTeamMembers ensures that a user is not a maintainer and member at the same time,
// there are no duplicate names in the list and all users are org members.
func testTeamMembers(teams map[string]org.Team, admins sets.String, orgMembers sets.String, orgName string) []error {
	var errs []error
	for teamName, team := range teams {
		teamMaintainers := sets.NewString(team.Maintainers...)
		teamMembers := sets.NewString(team.Members...)

		teamMaintainers = normalize(teamMaintainers)
		teamMembers = normalize(teamMembers)

		// check for users in both maintainers and members
		if both := teamMaintainers.Intersection(teamMembers); len(both) > 0 {
			errs = append(errs, fmt.Errorf("The team %s in org %s has users in both maintainer admin and member roles: %s", teamName, orgName, strings.Join(both.List(), ", ")))
		}

		// check for duplicates
		if err := testDuplicates(teamMaintainers); err != nil {
			errs = append(errs, fmt.Errorf("The team %s in org %s has duplicate maintainers: %v", teamName, orgName, err))
		}
		if err := testDuplicates(teamMembers); err != nil {
			errs = append(errs, fmt.Errorf("The team %s in org %s has duplicate members: %v", teamMembers, orgName, err))
		}

		// check if all are org members
		if missing := teamMaintainers.Difference(orgMembers); len(missing) > 0 {
			errs = append(errs, fmt.Errorf("The following maintainers of team %s are not %s org members: %s", teamName, orgName, strings.Join(missing.List(), ", ")))
		}
		if missing := teamMembers.Difference(orgMembers); len(missing) > 0 {
			errs = append(errs, fmt.Errorf("The following members of team %s are not %s org members: %s", teamName, orgName, strings.Join(missing.List(), ", ")))
		}

		// check if admins are a regular member of team
		if adminTeamMembers := teamMembers.Intersection(admins); len(adminTeamMembers) > 0 {
			errs = append(errs, fmt.Errorf("The team %s in org %s has org admins listed as members; these users should be in the maintainers list instead, and cannot be on the members list: %s", teamName, orgName, strings.Join(adminTeamMembers.List(), ", ")))
		}

		// check if lists are sorted
		if !isSorted(team.Maintainers) {
			errs = append(errs, fmt.Errorf("The team %s in org %s has an unsorted list of maintainers", teamName, orgName))
		}
		if !isSorted(team.Members) {
			errs = append(errs, fmt.Errorf("The team %s in org %s has an unsorted list of members", teamName, orgName))
		}

		if team.Children != nil {
			errs = append(errs, testTeamMembers(team.Children, admins, orgMembers, orgName)...)
		}
	}
	return errs
}

func testOrg(targetDir string, t *testing.T) {
	cfg, err := loadOrg(targetDir)
	if err != nil {
		t.Fatalf("failed to load org.yaml: %v", err)
	}
	own, err := loadOwners(targetDir)
	if err != nil {
		t.Fatalf("failed to load OWNERS: %v", err)
	}

	members := normalize(sets.NewString(cfg.Members...))
	admins := normalize(sets.NewString(cfg.Admins...))
	allOrgMembers := members.Union(admins)

	reviewers := normalize(sets.NewString(own.Reviewers...))
	approvers := normalize(sets.NewString(own.Approvers...))

	if n := len(approvers); n < 5 {
		t.Errorf("Require at least 5 approvers, found %d: %s", n, strings.Join(approvers.List(), ", "))
	}

	if missing := reviewers.Difference(allOrgMembers); len(missing) > 0 {
		t.Errorf("The following reviewers must be members: %s", strings.Join(missing.List(), ", "))
	}
	if missing := approvers.Difference(allOrgMembers); len(missing) > 0 {
		t.Errorf("The following approvers must be members: %s", strings.Join(missing.List(), ", "))
	}
	if err := testDuplicates(reviewers); err != nil {
		t.Errorf("duplicate reviewers: %v", err)
	}
	if err := testDuplicates(approvers); err != nil {
		t.Errorf("duplicate approvers: %v", err)
	}
}

func TestAllOrgs(t *testing.T) {
	cfg, err := config.Load("config.yaml", "")
	if err != nil {
		t.Fatalf("cannot read config.yaml from //config:gen-config.yaml: %v", err)
	}
	f, err := os.Open(".")
	if err != nil {
		t.Fatalf("cannot read config: %v", err)
	}
	infos, err := f.Readdir(0)
	if err != nil {
		t.Fatalf("cannot read subdirs: %v", err)
	}
	for _, i := range infos {
		if !i.IsDir() {
			continue
		}
		n := i.Name()
		if strings.HasPrefix(n, "linux_") || strings.HasPrefix(n, "darwin_") {
			continue
		}
		t.Run(n, func(t *testing.T) {
			if _, ok := cfg.Orgs[n]; !ok {
				t.Errorf("%s missing from generated config.yaml", n)
			}
			testOrg(n, t)
		})
	}

	for _, org := range cfg.Orgs {
		members := normalize(sets.NewString(org.Members...))
		admins := normalize(sets.NewString(org.Admins...))
		allOrgMembers := members.Union(admins)

		if both := admins.Intersection(members); len(both) > 0 {
			t.Errorf("users in both org admin and member roles: %s", strings.Join(both.List(), ", "))
		}

		if !admins.Has("k8s-ci-robot") {
			t.Errorf("k8s-ci-robot must be an admin")
		}

		if org.BillingEmail != nil {
			t.Errorf("billing_email must be unset")
		}

		if err := testDuplicates(admins); err != nil {
			t.Errorf("duplicate admins: %v", err)
		}
		if err := testDuplicates(allOrgMembers); err != nil {
			t.Errorf("duplicate members: %v", err)
		}
		if !isSorted(org.Admins) {
			t.Errorf("admins for %s org are unsorted", *org.Name)
		}
		if !isSorted(org.Members) {
			t.Errorf("members for %s org are unsorted", *org.Name)
		}

		if errs := testTeamMembers(org.Teams, admins, allOrgMembers, *org.Name); errs != nil {
			for _, err := range errs {
				t.Error(err)
			}
		}

	}
}
<commit_msg>Readd test to enforce non-admins shouldn't be maintainers<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
	"fmt"
	"io/ioutil"
	"os"
	"sort"
	"strings"
	"testing"

	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/test-infra/prow/config"
	"k8s.io/test-infra/prow/config/org"
	"k8s.io/test-infra/prow/github"

	"github.com/ghodss/yaml"
)

type owners struct {
	Reviewers []string `json:"reviewers,omitempty"`
	Approvers []string `json:"approvers"`
}

func readInto(path string, i interface{}) error {
	buf, err := ioutil.ReadFile(path)
	if err != nil {
		return fmt.Errorf("read: %v", err)
	}
	if err := yaml.Unmarshal(buf, i); err != nil {
		return fmt.Errorf("unmarshal: %v", err)
	}
	return nil
}

func loadOwners(dir string) (*owners, error) {
	var own owners
	if err := readInto(dir+"/OWNERS", &own); err != nil {
		return nil, err
	}
	return &own, nil
}

func loadOrg(dir string) (*org.Config, error) {
	var cfg org.Config
	if err := readInto(dir+"/org.yaml", &cfg); err != nil {
		return nil, err
	}
	return &cfg, nil
}

func testDuplicates(list sets.String) error {
	found := sets.String{}
	dups := sets.String{}
	all := list.List()
	for _, i := range all {
		if found.Has(i) {
			dups.Insert(i)
		}
		found.Insert(i)
	}
	if n := len(dups); n > 0 {
		return fmt.Errorf("%d duplicate names: %s", n, strings.Join(dups.List(), ", "))
	}
	return nil
}

func isSorted(list []string) bool {
	items := make([]string, len(list))
	for _, l := range list {
		items = append(items, strings.ToLower(l))
	}

	return sort.StringsAreSorted(items)
}

func normalize(s sets.String) sets.String {
	out := sets.String{}
	for i := range s {
		out.Insert(github.NormLogin(i))
	}
	return out
}

// testTeamMembers ensures that a user is not a maintainer and member at the same time,
// there are no duplicate names in the list and all users are org members.
func testTeamMembers(teams map[string]org.Team, admins sets.String, orgMembers sets.String, orgName string) []error {
	var errs []error
	for teamName, team := range teams {
		teamMaintainers := sets.NewString(team.Maintainers...)
		teamMembers := sets.NewString(team.Members...)

		teamMaintainers = normalize(teamMaintainers)
		teamMembers = normalize(teamMembers)

		// check for non-admins in maintainers list
		if nonAdminMaintainers := teamMaintainers.Difference(admins); len(nonAdminMaintainers) > 0 {
			errs = append(errs, fmt.Errorf("The team %s in org %s has non-admins listed as maintainers; these users should be in the members list instead: %s", teamName, orgName, strings.Join(nonAdminMaintainers.List(), ",")))
		}

		// check for users in both maintainers and members
		if both := teamMaintainers.Intersection(teamMembers); len(both) > 0 {
			errs = append(errs, fmt.Errorf("The team %s in org %s has users in both maintainer admin and member roles: %s", teamName, orgName, strings.Join(both.List(), ", ")))
		}

		// check for duplicates
		if err := testDuplicates(teamMaintainers); err != nil {
			errs = append(errs, fmt.Errorf("The team %s in org %s has duplicate maintainers: %v", teamName, orgName, err))
		}
		if err := testDuplicates(teamMembers); err != nil {
			errs = append(errs, fmt.Errorf("The team %s in org %s has duplicate members: %v", teamMembers, orgName, err))
		}

		// check if all are org members
		if missing := teamMembers.Difference(orgMembers); len(missing) > 0 {
			errs = append(errs, fmt.Errorf("The following members of team %s are not %s org members: %s", teamName, orgName, strings.Join(missing.List(), ", ")))
		}

		// check if admins are a regular member of team
		if adminTeamMembers := teamMembers.Intersection(admins); len(adminTeamMembers) > 0 {
			errs = append(errs, fmt.Errorf("The team %s in org %s has org admins listed as members; these users should be in the maintainers list instead, and cannot be on the members list: %s", teamName, orgName, strings.Join(adminTeamMembers.List(), ", ")))
		}

		// check if lists are sorted
		if !isSorted(team.Maintainers) {
			errs = append(errs, fmt.Errorf("The team %s in org %s has an unsorted list of maintainers", teamName, orgName))
		}
		if !isSorted(team.Members) {
			errs = append(errs, fmt.Errorf("The team %s in org %s has an unsorted list of members", teamName, orgName))
		}

		if team.Children != nil {
			errs = append(errs, testTeamMembers(team.Children, admins, orgMembers, orgName)...)
		}
	}
	return errs
}

func testOrg(targetDir string, t *testing.T) {
	cfg, err := loadOrg(targetDir)
	if err != nil {
		t.Fatalf("failed to load org.yaml: %v", err)
	}
	own, err := loadOwners(targetDir)
	if err != nil {
		t.Fatalf("failed to load OWNERS: %v", err)
	}

	members := normalize(sets.NewString(cfg.Members...))
	admins := normalize(sets.NewString(cfg.Admins...))
	allOrgMembers := members.Union(admins)

	reviewers := normalize(sets.NewString(own.Reviewers...))
	approvers := normalize(sets.NewString(own.Approvers...))

	if n := len(approvers); n < 5 {
		t.Errorf("Require at least 5 approvers, found %d: %s", n, strings.Join(approvers.List(), ", "))
	}

	if missing := reviewers.Difference(allOrgMembers); len(missing) > 0 {
		t.Errorf("The following reviewers must be members: %s", strings.Join(missing.List(), ", "))
	}
	if missing := approvers.Difference(allOrgMembers); len(missing) > 0 {
		t.Errorf("The following approvers must be members: %s", strings.Join(missing.List(), ", "))
	}
	if err := testDuplicates(reviewers); err != nil {
		t.Errorf("duplicate reviewers: %v", err)
	}
	if err := testDuplicates(approvers); err != nil {
		t.Errorf("duplicate approvers: %v", err)
	}
}

func TestAllOrgs(t *testing.T) {
	cfg, err := config.Load("config.yaml", "")
	if err != nil {
		t.Fatalf("cannot read config.yaml from //config:gen-config.yaml: %v", err)
	}
	f, err := os.Open(".")
	if err != nil {
		t.Fatalf("cannot read config: %v", err)
	}
	infos, err := f.Readdir(0)
	if err != nil {
		t.Fatalf("cannot read subdirs: %v", err)
	}
	for _, i := range infos {
		if !i.IsDir() {
			continue
		}
		n := i.Name()
		if strings.HasPrefix(n, "linux_") || strings.HasPrefix(n, "darwin_") {
			continue
		}
		t.Run(n, func(t *testing.T) {
			if _, ok := cfg.Orgs[n]; !ok {
				t.Errorf("%s missing from generated config.yaml", n)
			}
			testOrg(n, t)
		})
	}

	for _, org := range cfg.Orgs {
		members := normalize(sets.NewString(org.Members...))
		admins := normalize(sets.NewString(org.Admins...))
		allOrgMembers := members.Union(admins)

		if both := admins.Intersection(members); len(both) > 0 {
			t.Errorf("users in both org admin and member roles: %s", strings.Join(both.List(), ", "))
		}

		if !admins.Has("k8s-ci-robot") {
			t.Errorf("k8s-ci-robot must be an admin")
		}

		if org.BillingEmail != nil {
			t.Errorf("billing_email must be unset")
		}

		if err := testDuplicates(admins); err != nil {
			t.Errorf("duplicate admins: %v", err)
		}
		if err := testDuplicates(allOrgMembers); err != nil {
			t.Errorf("duplicate members: %v", err)
		}
		if !isSorted(org.Admins) {
			t.Errorf("admins for %s org are unsorted", *org.Name)
		}
		if !isSorted(org.Members) {
			t.Errorf("members for %s org are unsorted", *org.Name)
		}

		if errs := testTeamMembers(org.Teams, admins, allOrgMembers, *org.Name); errs != nil {
			for _, err := range errs {
				t.Error(err)
			}
		}

	}
}
<|endoftext|>


<commit_before>package config

import (
	"os"
	"strings"
	"sync"
)

// EnvFetcher is an implementation of the Fetcher type for communicating with
// the system's environment.
//
// It is safe to use across multiple goroutines.
type EnvFetcher struct {
	// vmu guards read/write access to vals
	vmu sync.Mutex
	// vals maintains a local cache of the system's enviornment variables
	// for fast repeat lookups of a given key.
	vals map[string]string
}

// NewEnvFetcher returns a new *EnvFetcher.
func NewEnvFetcher() *EnvFetcher {
	return &EnvFetcher{
		vals: make(map[string]string),
	}
}

// Get returns the value associated with the given key as stored in the local
// cache, or in the operating system's environment variables.
//
// If there was a cache-hit, the value will be returned from the cache, skipping
// a check against os.Getenv. Otherwise, the value will be fetched from the
// system, stored in the cache, and then returned. If no value was present in
// the cache or in the system, an empty string will be returned.
//
// Get is safe to call across multiple goroutines.
func (e *EnvFetcher) Get(key string) (val string) {
	e.vmu.Lock()
	defer e.vmu.Unlock()

	if i, ok := e.vals[key]; ok {
		return i
	}

	v := os.Getenv(key)
	e.vals[key] = v

	return v
}

// Bool	returns the boolean state assosicated with a given key, or the value
// "def", if no value was assosicated.
//
// The "boolean state assosicated with a given key" is defined as the
// case-insensitive string comparsion with the following:
//
// 1) true if...
//   "true", "1", "on", "yes", or "t"
// 2) false if...
//   "false", "0", "off", "no", "f", or otherwise.
func (e *EnvFetcher) Bool(key string, def bool) (val bool) {
	s := e.String(key)
	if len(s) == 0 {
		return def
	}

	switch strings.ToLower(s) {
	case "true", "1", "on", "yes", "t":
		return true
	case "false", "0", "off", "no", "f":
		return false
	default:
		return false
	}
}

// Set replaces a given key-value pair in the cache (if previously present in
// the cache) and in the system's environment variables. It returns an error if
// one was encountered in setting the environment variable, or `nil` if
// successful.
//
// Note: this method is a temporary measure while some of the old tests still
// rely on this mutable behavior.
func (e *EnvFetcher) Set(key, val string) error {
	e.vmu.Lock()
	defer e.vmu.Unlock()

	if _, ok := e.vals[key]; ok {
		e.vals[key] = val
	}

	return os.Setenv(key, val)
}

// SetAll replaces all key-value pairs with the given set, but does not modify
// the system's environment.
//
// Note: this method is a temporary measure while some of the old tests still
// rely on this mutable behavior.
func (e *EnvFetcher) SetAll(env map[string]string) {
	e.vmu.Lock()
	defer e.vmu.Unlock()

	e.vals = make(map[string]string)
	for k, v := range env {
		e.vals[k] = v
	}
}
<commit_msg>config/env: fix outdated reference to EnvFetcher.String<commit_after>package config

import (
	"os"
	"strings"
	"sync"
)

// EnvFetcher is an implementation of the Fetcher type for communicating with
// the system's environment.
//
// It is safe to use across multiple goroutines.
type EnvFetcher struct {
	// vmu guards read/write access to vals
	vmu sync.Mutex
	// vals maintains a local cache of the system's enviornment variables
	// for fast repeat lookups of a given key.
	vals map[string]string
}

// NewEnvFetcher returns a new *EnvFetcher.
func NewEnvFetcher() *EnvFetcher {
	return &EnvFetcher{
		vals: make(map[string]string),
	}
}

// Get returns the value associated with the given key as stored in the local
// cache, or in the operating system's environment variables.
//
// If there was a cache-hit, the value will be returned from the cache, skipping
// a check against os.Getenv. Otherwise, the value will be fetched from the
// system, stored in the cache, and then returned. If no value was present in
// the cache or in the system, an empty string will be returned.
//
// Get is safe to call across multiple goroutines.
func (e *EnvFetcher) Get(key string) (val string) {
	e.vmu.Lock()
	defer e.vmu.Unlock()

	if i, ok := e.vals[key]; ok {
		return i
	}

	v := os.Getenv(key)
	e.vals[key] = v

	return v
}

// Bool	returns the boolean state assosicated with a given key, or the value
// "def", if no value was assosicated.
//
// The "boolean state assosicated with a given key" is defined as the
// case-insensitive string comparsion with the following:
//
// 1) true if...
//   "true", "1", "on", "yes", or "t"
// 2) false if...
//   "false", "0", "off", "no", "f", or otherwise.
func (e *EnvFetcher) Bool(key string, def bool) (val bool) {
	s := e.Get(key)
	if len(s) == 0 {
		return def
	}

	switch strings.ToLower(s) {
	case "true", "1", "on", "yes", "t":
		return true
	case "false", "0", "off", "no", "f":
		return false
	default:
		return false
	}
}

// Set replaces a given key-value pair in the cache (if previously present in
// the cache) and in the system's environment variables. It returns an error if
// one was encountered in setting the environment variable, or `nil` if
// successful.
//
// Note: this method is a temporary measure while some of the old tests still
// rely on this mutable behavior.
func (e *EnvFetcher) Set(key, val string) error {
	e.vmu.Lock()
	defer e.vmu.Unlock()

	if _, ok := e.vals[key]; ok {
		e.vals[key] = val
	}

	return os.Setenv(key, val)
}

// SetAll replaces all key-value pairs with the given set, but does not modify
// the system's environment.
//
// Note: this method is a temporary measure while some of the old tests still
// rely on this mutable behavior.
func (e *EnvFetcher) SetAll(env map[string]string) {
	e.vmu.Lock()
	defer e.vmu.Unlock()

	e.vals = make(map[string]string)
	for k, v := range env {
		e.vals[k] = v
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package libhttpserver

import (
	"net/http"
	"strings"
)

type contentTypeOverridingResponseWriter struct {
	original http.ResponseWriter
}

var _ http.ResponseWriter = (*contentTypeOverridingResponseWriter)(nil)

func newContentTypeOverridingResponseWriter(
	original http.ResponseWriter) *contentTypeOverridingResponseWriter {
	return &contentTypeOverridingResponseWriter{
		original: original,
	}
}

func (w *contentTypeOverridingResponseWriter) calculateOverride(
	mimeType string) (newMimeType, disposition string) {
	// Send text/plain for all HTML and JS files to avoid them being executed
	// by the frontend WebView.
	ty := strings.ToLower(mimeType)
	switch {
	// First anything textual as text/plain, also javascript.
	case strings.HasPrefix(ty, "text/") ||
		ty == "application/javascript":
		return "text/plain", "inline"
	// Rest of html, xml. (note that the type may be e.g. application/xhtml+xml)
	case strings.Contains(ty, "xml") ||
		strings.Contains(ty, "html"):
		return "text/plain", "attachment"
	// Pass multimedia types through, and pdf too.
	case strings.HasPrefix(ty, "audio/") ||
		strings.HasPrefix(ty, "image/") ||
		strings.HasPrefix(ty, "video/") ||
		ty == "application/pdf":
		return ty, "inline"
	// Otherwise default to text + attachment.
	default:
		return "text/plain", "attachment"
	}
}

func (w *contentTypeOverridingResponseWriter) override() {
	t := w.original.Header().Get("Content-Type")
	if len(t) > 0 {
		ct, disp := w.calculateOverride(t)
		w.original.Header().Set("Content-Type", ct)
		w.original.Header().Set("Content-Disposition", disp)
	}
	w.original.Header().Set("X-Content-Type-Options", "nosniff")
}

func (w *contentTypeOverridingResponseWriter) Header() http.Header {
	return w.original.Header()
}

func (w *contentTypeOverridingResponseWriter) WriteHeader(statusCode int) {
	w.override()
	w.original.WriteHeader(statusCode)
}

func (w *contentTypeOverridingResponseWriter) Write(data []byte) (int, error) {
	w.override()
	return w.original.Write(data)
}

var additionalMimeTypes = map[string]string{
	".go":    "text/plain",
	".py":    "text/plain",
	".zsh":   "text/plain",
	".fish":  "text/plain",
	".cs":    "text/plain",
	".rb":    "text/plain",
	".m":     "text/plain",
	".mm":    "text/plain",
	".swift": "text/plain",
	".flow":  "text/plain",
	".php":   "text/plain",
	".pl":    "text/plain",
	".sh":    "text/plain",
	".js":    "text/plain",
	".json":  "text/plain",
	".sql":   "text/plain",
	".rs":    "text/plain",
	".xml":   "text/plain",
	".tex":   "text/plain",
	".pub":   "text/plain",
}
<commit_msg>libhttpserver: Get mime types right for even more filetypes<commit_after>// Copyright 2018 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package libhttpserver

import (
	"net/http"
	"strings"
)

type contentTypeOverridingResponseWriter struct {
	original http.ResponseWriter
}

var _ http.ResponseWriter = (*contentTypeOverridingResponseWriter)(nil)

func newContentTypeOverridingResponseWriter(
	original http.ResponseWriter) *contentTypeOverridingResponseWriter {
	return &contentTypeOverridingResponseWriter{
		original: original,
	}
}

// supportedContentTypes has exceptions to the libmime stuff because some types
// need special handling or are unsupported by frontend. The boolean value
// decides on whether this will be shown inline or as an attachment.
// We don't want to render SVG unless that has been audited, even if
// the file lacks a .svg extension.
var supportedContentTypes = map[string]bool{
	// Media
	"image/tiff":         false,
	"image/x-jng":        false,
	"image/vnd.wap.wbmp": false,
	"image/svg+xml":      false,
}

// displayInlineDefault decides on the Content-Disposition value (inline vs attachment) for
// the given mimeType by consulting the supportedContentTypes map and using the defaultValue
// parameter.
func displayInlineDefault(defaultValue bool, mimeType string) string {
	res, found := supportedContentTypes[mimeType]
	if (found && res) || (!found && defaultValue) {
		return "inline"
	}
	return "attachment"
}

func (w *contentTypeOverridingResponseWriter) calculateOverride(
	mimeType string) (newMimeType, disposition string) {
	// Send text/plain for all HTML and JS files to avoid them being executed
	// by the frontend WebView.
	ty := strings.ToLower(mimeType)
	switch {
	// First anything textual as text/plain.
	// Javascript is set to plain text by additionalMimeTypes map.
	// If text/something-dangerous would get here, we set it to plaintext.
	// If application/javascript somehow gets here it would be handled safely
	// by the default handler below.
	case strings.HasPrefix(ty, "text/"):
		return "text/plain", "inline"
	// Pass multimedia types through, and pdf too.
	// Some types get special handling here and are not shown inline (e.g. SVG).
	case strings.HasPrefix(ty, "audio/") ||
		strings.HasPrefix(ty, "image/") ||
		strings.HasPrefix(ty, "video/") ||
		ty == "application/pdf":
		return ty, displayInlineDefault(true, ty)
	// Otherwise default to text + attachment.
	// This is safe for all files.
	default:
		return "text/plain", "attachment"
	}
}

func (w *contentTypeOverridingResponseWriter) override() {
	t := w.original.Header().Get("Content-Type")
	if len(t) > 0 {
		ct, disp := w.calculateOverride(t)
		w.original.Header().Set("Content-Type", ct)
		w.original.Header().Set("Content-Disposition", disp)
	}
	w.original.Header().Set("X-Content-Type-Options", "nosniff")
}

func (w *contentTypeOverridingResponseWriter) Header() http.Header {
	return w.original.Header()
}

func (w *contentTypeOverridingResponseWriter) WriteHeader(statusCode int) {
	w.override()
	w.original.WriteHeader(statusCode)
}

func (w *contentTypeOverridingResponseWriter) Write(data []byte) (int, error) {
	w.override()
	return w.original.Write(data)
}

var additionalMimeTypes = map[string]string{
	".go":    "text/plain",
	".py":    "text/plain",
	".zsh":   "text/plain",
	".fish":  "text/plain",
	".cs":    "text/plain",
	".rb":    "text/plain",
	".m":     "text/plain",
	".mm":    "text/plain",
	".swift": "text/plain",
	".flow":  "text/plain",
	".php":   "text/plain",
	".pl":    "text/plain",
	".pm":    "text/plain",
	".sh":    "text/plain",
	".js":    "text/plain",
	".json":  "text/plain",
	".sql":   "text/plain",
	".rs":    "text/plain",
	".xml":   "text/plain",
	".tex":   "text/plain",
	".pub":   "text/plain",
	".atom":  "text/plain",
	".xhtml": "text/plain",
	".rss":   "text/plain",
	".tcl":   "text/plain",
	".tk":    "text/plain",
}
<|endoftext|>


<commit_before>package node

import (
	"fmt"
	"reflect"
	"testing"
	"time"

	"github.com/mosaicnetworks/babble/src/common"
	"github.com/mosaicnetworks/babble/src/crypto"
	"github.com/mosaicnetworks/babble/src/peers"
)

func TestMonologue(t *testing.T) {
	logger := common.NewTestLogger(t)
	keys, peers := initPeers(1)
	nodes := initNodes(keys, peers, 100000, 1000, "inmem", 5*time.Millisecond, logger, t)
	//defer drawGraphs(nodes, t)

	target := 50
	err := gossip(nodes, target, true, 3*time.Second)
	if err != nil {
		t.Fatal(err)
	}

	checkGossip(nodes, 0, t)
}

func TestJoinRequest(t *testing.T) {
	logger := common.NewTestLogger(t)
	keys, peerSet := initPeers(4)
	nodes := initNodes(keys, peerSet, 1000000, 1000, "inmem", 5*time.Millisecond, logger, t)
	defer shutdownNodes(nodes)
	//defer drawGraphs(nodes, t)

	target := 30
	err := gossip(nodes, target, false, 3*time.Second)
	if err != nil {
		t.Fatal(err)
	}
	checkGossip(nodes, 0, t)

	key, _ := crypto.GenerateECDSAKey()
	peer := peers.NewPeer(
		fmt.Sprintf("0x%X", crypto.FromECDSAPub(&key.PublicKey)),
		fmt.Sprint("127.0.0.1:4242"),
		"monika",
	)
	newNode := newNode(peer, key, "new node", peerSet, 1000, 1000, "inmem", 5*time.Millisecond, logger, t)
	defer newNode.Shutdown()

	err = newNode.join()
	if err != nil {
		t.Fatal(err)
	}

	//Gossip some more
	secondTarget := target + 30
	err = bombardAndWait(nodes, secondTarget, 6*time.Second)
	if err != nil {
		t.Fatal(err)
	}
	checkGossip(nodes, 0, t)
	checkPeerSets(nodes, t)
}

func TestLeaveRequest(t *testing.T) {
	n := 1
	f := func() {
		logger := common.NewTestLogger(t)
		keys, peerSet := initPeers(n)
		nodes := initNodes(keys, peerSet, 1000000, 1000, "inmem", 5*time.Millisecond, logger, t)
		defer shutdownNodes(nodes)
		//defer drawGraphs(nodes, t)

		target := 30
		err := gossip(nodes, target, false, 3*time.Second)
		if err != nil {
			t.Fatal(err)
		}
		checkGossip(nodes, 0, t)

		leavingNode := nodes[n-1]

		err = leavingNode.Leave()
		if err != nil {
			t.Fatal(err)
		}

		if n == 1 {
			return
		}

		//Gossip some more
		secondTarget := target + 50
		err = bombardAndWait(nodes[0:n-1], secondTarget, 6*time.Second)
		if err != nil {
			t.Fatal(err)
		}
		checkGossip(nodes[0:n-1], 0, t)
		checkPeerSets(nodes[0:n-1], t)
	}

	for n <= 4 {
		f()
		n++
	}
}

func TestSuccessiveJoinRequest(t *testing.T) {
	logger := common.NewTestLogger(t)
	keys, peerSet := initPeers(1)

	node0 := newNode(peerSet.Peers[0], keys[0], "new node", peerSet, 1000000, 400, "inmem", 10*time.Millisecond, logger, t)
	defer node0.Shutdown()
	node0.RunAsync(true)

	nodes := []*Node{node0}
	//defer drawGraphs(nodes, t)

	target := 20
	for i := 1; i <= 3; i++ {
		peerSet := peers.NewPeerSet(node0.GetPeers())

		key, _ := crypto.GenerateECDSAKey()
		peer := peers.NewPeer(
			fmt.Sprintf("0x%X", crypto.FromECDSAPub(&key.PublicKey)),
			fmt.Sprintf("127.0.0.1:%d", 4240+i),
			"monika",
		)
		newNode := newNode(peer, key, "new node", peerSet, 1000000, 400, "inmem", 10*time.Millisecond, logger, t)

		logger.Debugf("starting new node %d, %d", i, newNode.ID())
		defer newNode.Shutdown()
		newNode.RunAsync(true)

		nodes = append(nodes, newNode)

		//Gossip some more
		err := bombardAndWait(nodes, target, 10*time.Second)
		if err != nil {
			t.Fatal(err)
		}

		start := newNode.core.hg.FirstConsensusRound
		checkGossip(nodes, *start, t)

		target = target + 40
	}
}

func TestSuccessiveLeaveRequest(t *testing.T) {
	n := 4

	logger := common.NewTestLogger(t)
	keys, peerSet := initPeers(n)
	nodes := initNodes(keys, peerSet, 1000000, 1000, "inmem", 5*time.Millisecond, logger, t)
	defer shutdownNodes(nodes)

	target := 0

	f := func() {
		//defer drawGraphs(nodes, t)
		target += 30
		err := gossip(nodes, target, false, 3*time.Second)
		if err != nil {
			t.Fatal(err)
		}
		checkGossip(nodes, 0, t)

		leavingNode := nodes[n-1]

		err = leavingNode.Leave()
		if err != nil {
			t.Fatal(err)
		}

		if n == 1 {
			return
		}

		nodes = nodes[0 : n-1]

		//Gossip some more
		target += 50
		err = bombardAndWait(nodes, target, 6*time.Second)
		if err != nil {
			t.Fatal(err)
		}
		checkGossip(nodes, 0, t)
		checkPeerSets(nodes, t)
	}

	for n > 0 {
		f()
		n--
	}
}

func TestSimultaneusLeaveRequest(t *testing.T) {
	logger := common.NewTestLogger(t)
	keys, peerSet := initPeers(4)
	nodes := initNodes(keys, peerSet, 1000000, 1000, "inmem", 5*time.Millisecond, logger, t)
	defer shutdownNodes(nodes)
	//defer drawGraphs(nodes, t)

	target := 30
	err := gossip(nodes, target, false, 3*time.Second)
	if err != nil {
		t.Fatal(err)
	}
	checkGossip(nodes, 0, t)

	leavingNode := nodes[3]
	leavingNode2 := nodes[2]

	err = leavingNode.Leave()
	if err != nil {
		t.Fatal(err)
	}

	err = leavingNode2.Leave()
	if err != nil {
		t.Fatal(err)
	}

	//Gossip some more
	secondTarget := target + 50
	err = bombardAndWait(nodes[0:2], secondTarget, 6*time.Second)
	if err != nil {
		t.Fatal(err)
	}
	checkGossip(nodes[0:2], 0, t)
	checkPeerSets(nodes[0:2], t)
}

func TestJoinLeaveRequest(t *testing.T) {
	logger := common.NewTestLogger(t)
	keys, peerSet := initPeers(4)
	nodes := initNodes(keys, peerSet, 1000000, 1000, "inmem", 5*time.Millisecond, logger, t)
	defer shutdownNodes(nodes)
	//defer drawGraphs(nodes, t)

	target := 30
	err := gossip(nodes, target, false, 3*time.Second)
	if err != nil {
		t.Fatal(err)
	}
	checkGossip(nodes, 0, t)

	leavingNode := nodes[3]

	err = leavingNode.Leave()
	if err != nil {
		t.Fatal(err)
	}

	key, _ := crypto.GenerateECDSAKey()
	peer := peers.NewPeer(
		fmt.Sprintf("0x%X", crypto.FromECDSAPub(&key.PublicKey)),
		fmt.Sprint("127.0.0.1:4242"),
		"new node",
	)
	newNode := newNode(peer, key, peer.Moniker, peerSet, 1000000, 400, "inmem", 10*time.Millisecond, logger, t)
	defer newNode.Shutdown()

	// Run parallel routine to check newNode eventually reaches CatchingUp state.
	timeout := time.After(6 * time.Second)
	go func() {
		for {
			select {
			case <-timeout:
				t.Fatalf("Timeout waiting for newNode to enter CatchingUp state")
			default:
			}
			if newNode.getState() == CatchingUp {
				break
			}
		}
	}()

	newNode.RunAsync(true)

	// remove leaving node
	nodes = nodes[0:3]

	// add new node
	// nodes = append(nodes, newNode)

	//Gossip some more
	secondTarget := target + 50
	err = bombardAndWait(nodes, secondTarget, 6*time.Second)
	if err != nil {
		t.Fatal(err)
	}
	checkGossip(nodes, 0, t)
	checkPeerSets(nodes, t)

	peerSet2 := nodes[0].GetPeers()

	if len(peerSet2) != 4 {
		t.Fatalf("Invalid peerSet size")
	}
}

func TestJoinFull(t *testing.T) {
	logger := common.NewTestLogger(t)
	keys, peerSet := initPeers(4)
	initialNodes := initNodes(keys, peerSet, 1000000, 400, "inmem", 10*time.Millisecond, logger, t)
	defer shutdownNodes(initialNodes)

	target := 30
	err := gossip(initialNodes, target, false, 6*time.Second)
	if err != nil {
		t.Fatal(err)
	}
	checkGossip(initialNodes, 0, t)

	key, _ := crypto.GenerateECDSAKey()
	peer := peers.NewPeer(
		fmt.Sprintf("0x%X", crypto.FromECDSAPub(&key.PublicKey)),
		fmt.Sprint("127.0.0.1:4242"),
		"monika",
	)
	newNode := newNode(peer, key, "new node", peerSet, 1000000, 400, "inmem", 10*time.Millisecond, logger, t)
	defer newNode.Shutdown()

	//Run parallel routine to check newNode eventually reaches CatchingUp state.
	timeout := time.After(6 * time.Second)
	go func() {
		for {
			select {
			case <-timeout:
				t.Fatalf("Timeout waiting for newNode to enter CatchingUp state")
			default:
			}
			if newNode.getState() == CatchingUp {
				break
			}
		}
	}()

	newNode.RunAsync(true)

	nodes := append(initialNodes, newNode)

	//defer drawGraphs(nodes, t)

	//Gossip some more
	secondTarget := target + 50
	err = bombardAndWait(nodes, secondTarget, 10*time.Second)
	if err != nil {
		t.Fatal(err)
	}

	start := newNode.core.hg.FirstConsensusRound
	checkGossip(nodes, *start, t)
	checkPeerSets(nodes, t)
}

func checkPeerSets(nodes []*Node, t *testing.T) {
	node0FP, err := nodes[0].core.hg.Store.GetAllPeerSets()
	if err != nil {
		t.Fatal(err)
	}
	for i := range nodes[1:] {
		nodeiFP, err := nodes[i].core.hg.Store.GetAllPeerSets()
		if err != nil {
			t.Fatal(err)
		}
		if !reflect.DeepEqual(node0FP, nodeiFP) {
			t.Logf("Node 0 PeerSets: %v", node0FP)
			t.Logf("Node %d PeerSets: %v", i, nodeiFP)
			t.Fatalf("PeerSets defer")
		}
	}
}
<commit_msg>Remove JoinLeave test<commit_after>package node

import (
	"fmt"
	"reflect"
	"testing"
	"time"

	"github.com/mosaicnetworks/babble/src/common"
	"github.com/mosaicnetworks/babble/src/crypto"
	"github.com/mosaicnetworks/babble/src/peers"
)

func TestMonologue(t *testing.T) {
	logger := common.NewTestLogger(t)
	keys, peers := initPeers(1)
	nodes := initNodes(keys, peers, 100000, 1000, "inmem", 5*time.Millisecond, logger, t)
	//defer drawGraphs(nodes, t)

	target := 50
	err := gossip(nodes, target, true, 3*time.Second)
	if err != nil {
		t.Fatal(err)
	}

	checkGossip(nodes, 0, t)
}

func TestJoinRequest(t *testing.T) {
	logger := common.NewTestLogger(t)
	keys, peerSet := initPeers(4)
	nodes := initNodes(keys, peerSet, 1000000, 1000, "inmem", 5*time.Millisecond, logger, t)
	defer shutdownNodes(nodes)
	//defer drawGraphs(nodes, t)

	target := 30
	err := gossip(nodes, target, false, 3*time.Second)
	if err != nil {
		t.Fatal(err)
	}
	checkGossip(nodes, 0, t)

	key, _ := crypto.GenerateECDSAKey()
	peer := peers.NewPeer(
		fmt.Sprintf("0x%X", crypto.FromECDSAPub(&key.PublicKey)),
		fmt.Sprint("127.0.0.1:4242"),
		"monika",
	)
	newNode := newNode(peer, key, "new node", peerSet, 1000, 1000, "inmem", 5*time.Millisecond, logger, t)
	defer newNode.Shutdown()

	err = newNode.join()
	if err != nil {
		t.Fatal(err)
	}

	//Gossip some more
	secondTarget := target + 30
	err = bombardAndWait(nodes, secondTarget, 6*time.Second)
	if err != nil {
		t.Fatal(err)
	}
	checkGossip(nodes, 0, t)
	checkPeerSets(nodes, t)
}

func TestLeaveRequest(t *testing.T) {
	n := 1
	f := func() {
		logger := common.NewTestLogger(t)
		keys, peerSet := initPeers(n)
		nodes := initNodes(keys, peerSet, 1000000, 1000, "inmem", 5*time.Millisecond, logger, t)
		defer shutdownNodes(nodes)
		//defer drawGraphs(nodes, t)

		target := 30
		err := gossip(nodes, target, false, 3*time.Second)
		if err != nil {
			t.Fatal(err)
		}
		checkGossip(nodes, 0, t)

		leavingNode := nodes[n-1]

		err = leavingNode.Leave()
		if err != nil {
			t.Fatal(err)
		}

		if n == 1 {
			return
		}

		//Gossip some more
		secondTarget := target + 50
		err = bombardAndWait(nodes[0:n-1], secondTarget, 6*time.Second)
		if err != nil {
			t.Fatal(err)
		}
		checkGossip(nodes[0:n-1], 0, t)
		checkPeerSets(nodes[0:n-1], t)
	}

	for n <= 4 {
		f()
		n++
	}
}

func TestSuccessiveJoinRequest(t *testing.T) {
	logger := common.NewTestLogger(t)
	keys, peerSet := initPeers(1)

	node0 := newNode(peerSet.Peers[0], keys[0], "new node", peerSet, 1000000, 400, "inmem", 10*time.Millisecond, logger, t)
	defer node0.Shutdown()
	node0.RunAsync(true)

	nodes := []*Node{node0}
	//defer drawGraphs(nodes, t)

	target := 20
	for i := 1; i <= 3; i++ {
		peerSet := peers.NewPeerSet(node0.GetPeers())

		key, _ := crypto.GenerateECDSAKey()
		peer := peers.NewPeer(
			fmt.Sprintf("0x%X", crypto.FromECDSAPub(&key.PublicKey)),
			fmt.Sprintf("127.0.0.1:%d", 4240+i),
			"monika",
		)
		newNode := newNode(peer, key, "new node", peerSet, 1000000, 400, "inmem", 10*time.Millisecond, logger, t)

		logger.Debugf("starting new node %d, %d", i, newNode.ID())
		defer newNode.Shutdown()
		newNode.RunAsync(true)

		nodes = append(nodes, newNode)

		//Gossip some more
		err := bombardAndWait(nodes, target, 10*time.Second)
		if err != nil {
			t.Fatal(err)
		}

		start := newNode.core.hg.FirstConsensusRound
		checkGossip(nodes, *start, t)

		target = target + 40
	}
}

func TestSuccessiveLeaveRequest(t *testing.T) {
	n := 4

	logger := common.NewTestLogger(t)
	keys, peerSet := initPeers(n)
	nodes := initNodes(keys, peerSet, 1000000, 1000, "inmem", 5*time.Millisecond, logger, t)
	defer shutdownNodes(nodes)

	target := 0

	f := func() {
		//defer drawGraphs(nodes, t)
		target += 30
		err := gossip(nodes, target, false, 3*time.Second)
		if err != nil {
			t.Fatal(err)
		}
		checkGossip(nodes, 0, t)

		leavingNode := nodes[n-1]

		err = leavingNode.Leave()
		if err != nil {
			t.Fatal(err)
		}

		if n == 1 {
			return
		}

		nodes = nodes[0 : n-1]

		//Gossip some more
		target += 50
		err = bombardAndWait(nodes, target, 6*time.Second)
		if err != nil {
			t.Fatal(err)
		}
		checkGossip(nodes, 0, t)
		checkPeerSets(nodes, t)
	}

	for n > 0 {
		f()
		n--
	}
}

func TestSimultaneusLeaveRequest(t *testing.T) {
	logger := common.NewTestLogger(t)
	keys, peerSet := initPeers(4)
	nodes := initNodes(keys, peerSet, 1000000, 1000, "inmem", 5*time.Millisecond, logger, t)
	defer shutdownNodes(nodes)
	//defer drawGraphs(nodes, t)

	target := 30
	err := gossip(nodes, target, false, 3*time.Second)
	if err != nil {
		t.Fatal(err)
	}
	checkGossip(nodes, 0, t)

	leavingNode := nodes[3]
	leavingNode2 := nodes[2]

	err = leavingNode.Leave()
	if err != nil {
		t.Fatal(err)
	}

	err = leavingNode2.Leave()
	if err != nil {
		t.Fatal(err)
	}

	//Gossip some more
	secondTarget := target + 50
	err = bombardAndWait(nodes[0:2], secondTarget, 6*time.Second)
	if err != nil {
		t.Fatal(err)
	}
	checkGossip(nodes[0:2], 0, t)
	checkPeerSets(nodes[0:2], t)
}

func TestJoinFull(t *testing.T) {
	logger := common.NewTestLogger(t)
	keys, peerSet := initPeers(4)
	initialNodes := initNodes(keys, peerSet, 1000000, 400, "inmem", 10*time.Millisecond, logger, t)
	defer shutdownNodes(initialNodes)

	target := 30
	err := gossip(initialNodes, target, false, 6*time.Second)
	if err != nil {
		t.Fatal(err)
	}
	checkGossip(initialNodes, 0, t)

	key, _ := crypto.GenerateECDSAKey()
	peer := peers.NewPeer(
		fmt.Sprintf("0x%X", crypto.FromECDSAPub(&key.PublicKey)),
		fmt.Sprint("127.0.0.1:4242"),
		"monika",
	)
	newNode := newNode(peer, key, "new node", peerSet, 1000000, 400, "inmem", 10*time.Millisecond, logger, t)
	defer newNode.Shutdown()

	//Run parallel routine to check newNode eventually reaches CatchingUp state.
	timeout := time.After(6 * time.Second)
	go func() {
		for {
			select {
			case <-timeout:
				t.Fatalf("Timeout waiting for newNode to enter CatchingUp state")
			default:
			}
			if newNode.getState() == CatchingUp {
				break
			}
		}
	}()

	newNode.RunAsync(true)

	nodes := append(initialNodes, newNode)

	//defer drawGraphs(nodes, t)

	//Gossip some more
	secondTarget := target + 50
	err = bombardAndWait(nodes, secondTarget, 10*time.Second)
	if err != nil {
		t.Fatal(err)
	}

	start := newNode.core.hg.FirstConsensusRound
	checkGossip(nodes, *start, t)
	checkPeerSets(nodes, t)
}

func checkPeerSets(nodes []*Node, t *testing.T) {
	node0FP, err := nodes[0].core.hg.Store.GetAllPeerSets()
	if err != nil {
		t.Fatal(err)
	}
	for i := range nodes[1:] {
		nodeiFP, err := nodes[i].core.hg.Store.GetAllPeerSets()
		if err != nil {
			t.Fatal(err)
		}
		if !reflect.DeepEqual(node0FP, nodeiFP) {
			t.Logf("Node 0 PeerSets: %v", node0FP)
			t.Logf("Node %d PeerSets: %v", i, nodeiFP)
			t.Fatalf("PeerSets defer")
		}
	}
}
<|endoftext|>