<commit_before>package graph

import (
	"database/sql/driver"
	"fmt"
)

// StatType is the name of a symbol statistic (see below for a listing).
type StatType string

// Stats holds statistics for a symbol.
type Stats map[StatType]int

const (
	// StatXRefs is the number of external references to a symbol (i.e.,
	// references from other repositories). It is only computed for abstract
	// symbols (see the docs for SymbolKey) because it is not easy to determine
	// which specific commit a ref references (for external refs).
	StatXRefs = "xrefs"

	// StatRRefs is the number of references to a symbol from the same
	// repository in which the symbol is defined. It is inclusive of the
	// StatURefs count. It is only computed for concrete symbols (see the docs
	// for SymbolKey) because otherwise it would count 1 rref for each unique
	// revision of the repository that we have processed. (It is easy to
	// determine which specific commit an internal ref references; we just
	// assume it references a symbol in the same commit.)
	StatRRefs = "rrefs"

	// StatURefs is the number of references to a symbol from the same source
	// unit in which the symbol is defined. It is included in the StatRRefs
	// count. It is only computed for concrete symbols (see the docs for
	// SymbolKey) because otherwise it would count 1 uref for each revision of
	// the repository that we have processed.
	StatURefs = "urefs"

	// StatAuthors is the number of distinct resolved people who contributed
	// code to a symbol's definition (according to a VCS "blame" of the
	// version). It is only computed for concrete symbols (see the docs for
	// SymbolKey).
	StatAuthors = "authors"

	// StatClients is the number of distinct resolved people who have committed
	// refs that reference a symbol. It is only computed for abstract symbols
	// (see the docs for SymbolKey) because it is not easy to determine which
	// specific commit a ref references.
	StatClients = "clients"

	// StatDependents is the number of distinct repositories that contain refs
	// that reference a symbol. It is only computed for abstract symbols (see
	// the docs for SymbolKey) because it is not easy to determine which
	// specific commit a ref references.
	StatDependents = "dependents"

	// StatExportedElements is the number of exported symbols whose path is a
	// descendant of this symbol's path (and that is in the same repository and
	// source unit). It is only computed for concrete symbols (see the docs for
	// SymbolKey) because otherwise it would count 1 exported element for each
	// revision of the repository that we have processed.
	StatExportedElements = "exported-elements"

	// StatInterfaces is the number of interfaces that a symbol implements (in
	// its own repository or other repositories). TODO(sqs): it is not currently
	// being computed.
	StatInterfaces = "interfaces"

	// StatImplementations is the number of implementations of an interface
	// symbol (in its own repository or other repositories). TODO(sqs): it is
	// not currently being computed.
	StatImplementations = "implementations"
)

// Value implements database/sql/driver.Valuer.
func (x StatType) Value() (driver.Value, error) {
	return string(x), nil
}

// Scan implements database/sql.Scanner.
func (x *StatType) Scan(v interface{}) error {
	if data, ok := v.([]byte); ok {
		*x = StatType(data)
		return nil
	}
	return fmt.Errorf("%T.Scan failed: %v", x, v)
}

// UniqueRefSymbols groups refs by the RefSymbolKey field and returns a map of
// how often each RefSymbolKey appears. If m is non-nil, counts are incremented
// and a new map is not created.
func UniqueRefSymbols(refs []*Ref, m map[RefSymbolKey]int) map[RefSymbolKey]int {
	if m == nil {
		m = make(map[RefSymbolKey]int)
	}
	for _, ref := range refs {
		m[ref.RefSymbolKey()]++
	}
	return m
}
<commit_msg>support both concrete and abstract stats in Symbols.List, support sorting by stats<commit_after>package graph

import (
	"database/sql/driver"
	"fmt"
)

// StatType is the name of a symbol statistic (see below for a listing).
type StatType string

// Stats holds statistics for a symbol.
type Stats map[StatType]int

const (
	// StatXRefs is the number of external references to a symbol (i.e.,
	// references from other repositories). It is only computed for abstract
	// symbols (see the docs for SymbolKey) because it is not easy to determine
	// which specific commit a ref references (for external refs).
	StatXRefs = "xrefs"

	// StatRRefs is the number of references to a symbol from the same
	// repository in which the symbol is defined. It is inclusive of the
	// StatURefs count. It is only computed for concrete symbols (see the docs
	// for SymbolKey) because otherwise it would count 1 rref for each unique
	// revision of the repository that we have processed. (It is easy to
	// determine which specific commit an internal ref references; we just
	// assume it references a symbol in the same commit.)
	StatRRefs = "rrefs"

	// StatURefs is the number of references to a symbol from the same source
	// unit in which the symbol is defined. It is included in the StatRRefs
	// count. It is only computed for concrete symbols (see the docs for
	// SymbolKey) because otherwise it would count 1 uref for each revision of
	// the repository that we have processed.
	StatURefs = "urefs"

	// StatAuthors is the number of distinct resolved people who contributed
	// code to a symbol's definition (according to a VCS "blame" of the
	// version). It is only computed for concrete symbols (see the docs for
	// SymbolKey).
	StatAuthors = "authors"

	// StatClients is the number of distinct resolved people who have committed
	// refs that reference a symbol. It is only computed for abstract symbols
	// (see the docs for SymbolKey) because it is not easy to determine which
	// specific commit a ref references.
	StatClients = "clients"

	// StatDependents is the number of distinct repositories that contain refs
	// that reference a symbol. It is only computed for abstract symbols (see
	// the docs for SymbolKey) because it is not easy to determine which
	// specific commit a ref references.
	StatDependents = "dependents"

	// StatExportedElements is the number of exported symbols whose path is a
	// descendant of this symbol's path (and that is in the same repository and
	// source unit). It is only computed for concrete symbols (see the docs for
	// SymbolKey) because otherwise it would count 1 exported element for each
	// revision of the repository that we have processed.
	StatExportedElements = "exported_elements"

	// StatInterfaces is the number of interfaces that a symbol implements (in
	// its own repository or other repositories). TODO(sqs): it is not currently
	// being computed.
	StatInterfaces = "interfaces"

	// StatImplementations is the number of implementations of an interface
	// symbol (in its own repository or other repositories). TODO(sqs): it is
	// not currently being computed.
	StatImplementations = "implementations"
)

var AllStatTypes = []StatType{StatXRefs, StatRRefs, StatURefs, StatAuthors, StatClients, StatDependents, StatExportedElements, StatInterfaces, StatImplementations}

func (x StatType) IsAbstract() bool {
	switch x {
	case StatXRefs:
		fallthrough
	case StatClients:
		fallthrough
	case StatDependents:
		return true
	default:
		return false
	}
}

// Value implements database/sql/driver.Valuer.
func (x StatType) Value() (driver.Value, error) {
	return string(x), nil
}

// Scan implements database/sql.Scanner.
func (x *StatType) Scan(v interface{}) error {
	if data, ok := v.([]byte); ok {
		*x = StatType(data)
		return nil
	}
	return fmt.Errorf("%T.Scan failed: %v", x, v)
}

// UniqueRefSymbols groups refs by the RefSymbolKey field and returns a map of
// how often each RefSymbolKey appears. If m is non-nil, counts are incremented
// and a new map is not created.
func UniqueRefSymbols(refs []*Ref, m map[RefSymbolKey]int) map[RefSymbolKey]int {
	if m == nil {
		m = make(map[RefSymbolKey]int)
	}
	for _, ref := range refs {
		m[ref.RefSymbolKey()]++
	}
	return m
}
<|endoftext|>


<commit_before>package whois

import (
	"testing"

	"github.com/domainr/whoistest"
	"github.com/nbio/st"
)

func TestResponse_Text(t *testing.T) {
	r := NewResponse("google.com", "whois.verisign-grs.com")
	r.Charset = "utf-8"
	r.Body = []byte("hello")
	text, err := r.Text()
	st.Expect(t, err, nil)
	st.Expect(t, string(text), "hello")
}

func TestResponse_String(t *testing.T) {
	r := NewResponse("google.com", "whois.verisign-grs.com")
	r.Charset = "utf-8"
	r.Body = []byte("hello")
	st.Expect(t, r.String(), "hello")
}

func TestReadMIME(t *testing.T) {
	fns, err := whoistest.ResponseFiles()
	st.Assert(t, err, nil)
	for _, fn := range fns {
		// fmt.Printf("%s\n", fn)
		res, err := ReadMIMEFile(fn)
		st.Reject(t, res, nil)
		st.Expect(t, err, nil)
		// fmt.Printf("%#v\n\n", res)
	}
}

func TestPIRRateLimitText(t *testing.T) {
	req, err := NewRequest("google.org")
	st.Assert(t, err, nil)
	res, err := DefaultClient.Fetch(req)
	st.Assert(t, err, nil)
	st.Expect(t, res.MediaType, "text/plain")
	st.Expect(t, res.Charset, "windows-1252")
	res.Body = []byte("WHOIS LIMIT EXCEEDED - SEE WWW.PIR.ORG/WHOIS FOR DETAILS\n")
	res.DetectContentType("")
	st.Expect(t, res.MediaType, "text/plain")
	st.Expect(t, res.Charset, "windows-1252")
	h := res.Header()
	st.Expect(t, h.Get("Content-Type"), "text/plain; charset=windows-1252")
}
<commit_msg>add test for empty body<commit_after>package whois

import (
	"io"
	"strings"
	"testing"

	"github.com/domainr/whoistest"
	"github.com/nbio/st"
)

func TestResponse_Text(t *testing.T) {
	r := NewResponse("google.com", "whois.verisign-grs.com")
	r.Charset = "utf-8"
	r.Body = []byte("hello")
	text, err := r.Text()
	st.Expect(t, err, nil)
	st.Expect(t, string(text), "hello")
}

func TestResponse_String(t *testing.T) {
	r := NewResponse("google.com", "whois.verisign-grs.com")
	r.Charset = "utf-8"
	r.Body = []byte("hello")
	st.Expect(t, r.String(), "hello")
}

func TestReadMIME(t *testing.T) {
	fns, err := whoistest.ResponseFiles()
	st.Assert(t, err, nil)
	for _, fn := range fns {
		// fmt.Printf("%s\n", fn)
		res, err := ReadMIMEFile(fn)
		st.Reject(t, res, nil)
		st.Expect(t, err, nil)
		// fmt.Printf("%#v\n\n", res)
	}
}

func TestReadMIMEEmpty(t *testing.T) {
	res, err := ReadMIME(strings.NewReader(""))
	st.Reject(t, res, nil)
	st.Expect(t, err, io.EOF)
}

func TestPIRRateLimitText(t *testing.T) {
	req, err := NewRequest("google.org")
	st.Assert(t, err, nil)
	res, err := DefaultClient.Fetch(req)
	st.Assert(t, err, nil)
	st.Expect(t, res.MediaType, "text/plain")
	st.Expect(t, res.Charset, "iso-8859-1")
	res.Body = []byte("WHOIS LIMIT EXCEEDED - SEE WWW.PIR.ORG/WHOIS FOR DETAILS\n")
	res.DetectContentType("")
	st.Expect(t, res.MediaType, "text/plain")
	st.Expect(t, res.Charset, "windows-1252")
	h := res.Header()
	st.Expect(t, h.Get("Content-Type"), "text/plain; charset=windows-1252")
}
<|endoftext|>


<commit_before>// Copyright 2014 Manu Martinez-Almeida.  All rights reserved.
// Use of this source code is governed by a MIT style
// license that can be found in the LICENSE file.

package authz

import (
	"net/http"

	"github.com/casbin/casbin"
	"github.com/gin-gonic/gin"
)

// NewAuthorizer returns the authorizer, uses a Casbin enforcer as input
func NewAuthorizer(e *casbin.Enforcer) gin.HandlerFunc {
	return func(c *gin.Context) {
		a := &BasicAuthorizer{enforcer: e}

		if !a.CheckPermission(c.Request) {
			a.RequirePermission(c.Writer)
		}
	}
}

// BasicAuthorizer stores the casbin handler
type BasicAuthorizer struct {
	enforcer *casbin.Enforcer
}

// GetUserName gets the user name from the request.
// Currently, only HTTP basic authentication is supported
func (a *BasicAuthorizer) GetUserName(r *http.Request) string {
	username, _, _ := r.BasicAuth()
	return username
}

// CheckPermission checks the user/method/path combination from the request.
// Returns true (permission granted) or false (permission forbidden)
func (a *BasicAuthorizer) CheckPermission(r *http.Request) bool {
	user := a.GetUserName(r)
	method := r.Method
	path := r.URL.Path
	return a.enforcer.Enforce(user, path, method)
}

// RequirePermission returns the 403 Forbidden to the client
func (a *BasicAuthorizer) RequirePermission(w http.ResponseWriter) {
	w.WriteHeader(403)
	w.Write([]byte("403 Forbidden\n"))
}
<commit_msg>fixed response header overwriting<commit_after>// Copyright 2014 Manu Martinez-Almeida.  All rights reserved.
// Use of this source code is governed by a MIT style
// license that can be found in the LICENSE file.

package authz

import (
	"net/http"

	"github.com/casbin/casbin"
	"github.com/gin-gonic/gin"
)

// NewAuthorizer returns the authorizer, uses a Casbin enforcer as input
func NewAuthorizer(e *casbin.Enforcer) gin.HandlerFunc {
	return func(c *gin.Context) {
		a := &BasicAuthorizer{enforcer: e}

		if !a.CheckPermission(c.Request) {
			a.RequirePermission(c)
			return
		}
	}
}

// BasicAuthorizer stores the casbin handler
type BasicAuthorizer struct {
	enforcer *casbin.Enforcer
}

// GetUserName gets the user name from the request.
// Currently, only HTTP basic authentication is supported
func (a *BasicAuthorizer) GetUserName(r *http.Request) string {
	username, _, _ := r.BasicAuth()
	return username
}

// CheckPermission checks the user/method/path combination from the request.
// Returns true (permission granted) or false (permission forbidden)
func (a *BasicAuthorizer) CheckPermission(r *http.Request) bool {
	user := a.GetUserName(r)
	method := r.Method
	path := r.URL.Path
	return a.enforcer.Enforce(user, path, method)
}

// RequirePermission returns the 403 Forbidden to the client
func (a *BasicAuthorizer) RequirePermission(c *gin.Context) {
	c.AbortWithStatus(403)
	return
}
<|endoftext|>


<commit_before>package mocks_test

import (
	"go/ast"
	"go/format"
	"testing"

	"github.com/a8m/expect"
	"github.com/nelsam/hel/mocks"
)

func TestNewErrorsForNonInterfaceTypes(t *testing.T) {
	expect := expect.New(t)

	spec := typeSpec(expect, "type Foo func()")
	_, err := mocks.For(spec)
	expect(err).Not.To.Be.Nil()
	expect(err.Error()).To.Equal("TypeSpec.Type expected to be *ast.InterfaceType, was *ast.FuncType")
}

func TestMockName(t *testing.T) {
	expect := expect.New(t)

	spec := typeSpec(expect, "type Foo interface{}")
	m, err := mocks.For(spec)
	expect(err).To.Be.Nil()
	expect(m).Not.To.Be.Nil()
	expect(m.Name()).To.Equal("mockFoo")
}

func TestMockTypeDecl(t *testing.T) {
	expect := expect.New(t)

	spec := typeSpec(expect, `
 type Foo interface {
  Foo(foo string) int
  Bar(bar int) string
  Baz()
 }
 `)
	m, err := mocks.For(spec)
	expect(err).To.Be.Nil()
	expect(m).Not.To.Be.Nil()

	expected, err := format.Source([]byte(`
 package foo

 type mockFoo struct {
  FooCalled chan bool
  FooInput struct {
   foo chan string
  }
  FooOutput struct {
   ret0 chan int
  }
  BarCalled chan bool
  BarInput struct {
   bar chan int
  }
  BarOutput struct {
   ret0 chan string
  }
  BazCalled chan bool
 }
 `))
	expect(err).To.Be.Nil()

	src := source(expect, "foo", []ast.Decl{m.Decl()}, nil)
	expect(src).To.Equal(string(expected))
}

func TestMockConstructor(t *testing.T) {
	expect := expect.New(t)

	spec := typeSpec(expect, `
 type Foo interface {
  Foo(foo string) int
  Bar(bar int) string
 }
 `)
	m, err := mocks.For(spec)
	expect(err).To.Be.Nil()
	expect(m).Not.To.Be.Nil()

	expected, err := format.Source([]byte(`
 package foo

 func newMockFoo() *mockFoo {
  m := &mockFoo{}
  m.FooCalled = make(chan bool, 300)
  m.FooInput.foo = make(chan string, 300)
  m.FooOutput.ret0 = make(chan int, 300)
  m.BarCalled = make(chan bool, 300)
  m.BarInput.bar = make(chan int, 300)
  m.BarOutput.ret0 = make(chan string, 300)
  return m
 }`))
	expect(err).To.Be.Nil()

	src := source(expect, "foo", []ast.Decl{m.Constructor(300)}, nil)
	expect(src).To.Equal(string(expected))
}

func TestMockAst(t *testing.T) {
	expect := expect.New(t)

	spec := typeSpec(expect, `
 type Foo interface {
  Bar(bar string)
  Baz() (baz int)
 }`)
	m, err := mocks.For(spec)
	expect(err).To.Be.Nil()
	expect(m).Not.To.Be.Nil()

	expect(m.Methods()).To.Have.Len(2)

	decls := m.Ast(300)
	expect(decls).To.Have.Len(4)
	expect(decls[0]).To.Equal(m.Decl())
	expect(decls[1]).To.Equal(m.Constructor(300))
	expect(decls[2]).To.Equal(m.Methods()[0].Ast())
	expect(decls[3]).To.Equal(m.Methods()[1].Ast())
}
<commit_msg>Add test for adding package name to local types<commit_after>package mocks_test

import (
	"go/ast"
	"go/format"
	"testing"

	"github.com/a8m/expect"
	"github.com/nelsam/hel/mocks"
)

func TestNewErrorsForNonInterfaceTypes(t *testing.T) {
	expect := expect.New(t)

	spec := typeSpec(expect, "type Foo func()")
	_, err := mocks.For(spec)
	expect(err).Not.To.Be.Nil()
	expect(err.Error()).To.Equal("TypeSpec.Type expected to be *ast.InterfaceType, was *ast.FuncType")
}

func TestMockName(t *testing.T) {
	expect := expect.New(t)

	spec := typeSpec(expect, "type Foo interface{}")
	m, err := mocks.For(spec)
	expect(err).To.Be.Nil()
	expect(m).Not.To.Be.Nil()
	expect(m.Name()).To.Equal("mockFoo")
}

func TestMockTypeDecl(t *testing.T) {
	expect := expect.New(t)

	spec := typeSpec(expect, `
 type Foo interface {
  Foo(foo string) int
  Bar(bar int) Foo
  Baz()
 }
 `)
	m, err := mocks.For(spec)
	expect(err).To.Be.Nil()
	expect(m).Not.To.Be.Nil()

	expected, err := format.Source([]byte(`
 package foo

 type mockFoo struct {
  FooCalled chan bool
  FooInput struct {
   foo chan string
  }
  FooOutput struct {
   ret0 chan int
  }
  BarCalled chan bool
  BarInput struct {
   bar chan int
  }
  BarOutput struct {
   ret0 chan Foo
  }
  BazCalled chan bool
 }
 `))
	expect(err).To.Be.Nil()

	src := source(expect, "foo", []ast.Decl{m.Decl()}, nil)
	expect(src).To.Equal(string(expected))

	m.PrependLocalPackage("foo")

	expected, err = format.Source([]byte(`
 package foo

 type mockFoo struct {
  FooCalled chan bool
  FooInput struct {
   foo chan string
  }
  FooOutput struct {
   ret0 chan int
  }
  BarCalled chan bool
  BarInput struct {
   bar chan int
  }
  BarOutput struct {
   ret0 chan foo.Foo
  }
  BazCalled chan bool
 }
 `))
	expect(err).To.Be.Nil()

	src = source(expect, "foo", []ast.Decl{m.Decl()}, nil)
	expect(src).To.Equal(string(expected))
}

func TestMockConstructor(t *testing.T) {
	expect := expect.New(t)

	spec := typeSpec(expect, `
 type Foo interface {
  Foo(foo string) int
  Bar(bar int) string
 }
 `)
	m, err := mocks.For(spec)
	expect(err).To.Be.Nil()
	expect(m).Not.To.Be.Nil()

	expected, err := format.Source([]byte(`
 package foo

 func newMockFoo() *mockFoo {
  m := &mockFoo{}
  m.FooCalled = make(chan bool, 300)
  m.FooInput.foo = make(chan string, 300)
  m.FooOutput.ret0 = make(chan int, 300)
  m.BarCalled = make(chan bool, 300)
  m.BarInput.bar = make(chan int, 300)
  m.BarOutput.ret0 = make(chan string, 300)
  return m
 }`))
	expect(err).To.Be.Nil()

	src := source(expect, "foo", []ast.Decl{m.Constructor(300)}, nil)
	expect(src).To.Equal(string(expected))
}

func TestMockAst(t *testing.T) {
	expect := expect.New(t)

	spec := typeSpec(expect, `
 type Foo interface {
  Bar(bar string)
  Baz() (baz int)
 }`)
	m, err := mocks.For(spec)
	expect(err).To.Be.Nil()
	expect(m).Not.To.Be.Nil()

	expect(m.Methods()).To.Have.Len(2)

	decls := m.Ast(300)
	expect(decls).To.Have.Len(4)
	expect(decls[0]).To.Equal(m.Decl())
	expect(decls[1]).To.Equal(m.Constructor(300))
	expect(decls[2]).To.Equal(m.Methods()[0].Ast())
	expect(decls[3]).To.Equal(m.Methods()[1].Ast())
}
<|endoftext|>


<commit_before>package city

import (
	"errors"

	mgo "gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	redis "gopkg.in/redis.v5"
)

type City struct {
	Id       bson.ObjectId `json:"id" bson:"_id,omitempty"`
	Name     string        `json:"name"`
	District int           `json:"district"`
	Location GeoJson       `json:"location"`
}

type GeoJson struct {
	Type        string    `json:"type"`
	Coordinates []float64 `json:"coordinates"`
}

// Global variable for storing database connection.
var mongo *mgo.Session
var redisConn *redis.Client

func (c *City) GetConn(mongoConnection *mgo.Session, redisConnection *redis.Client) {
	mongo = mongoConnection
	redisConn = redisConnection
}

// check mongo connection if error return it.
func checkMongoConnection(mongoConnection *mgo.Session) error {
	if mongoConnection == nil {
		return errors.New("No Mongo Connection")
	}

	return nil
}

func (c *City) CreateIndex(collectionName string) error {
	var err error
	err = checkMongoConnection(mongo)
	if err != nil {
		return err
	}

	index := mgo.Index{
		Key: []string{"$2dsphere:location"},
	}

	// create index from given collection
	collection := mongo.DB("Driver").C(collectionName)
	err = collection.EnsureIndex(index)
	if err != nil {
		return err
	}

	return nil

}

// Inserting district to mongo database
func (c *City) InsertDistrict(city string, distric int, lat, lon float64) error {
	var err error
	err = checkMongoConnection(mongo)
	if err != nil {
		return err
	}

	collection := mongo.DB("Driver").C(city)

	c.District = distric
	c.Name = city
	c.Location = GeoJson{Type: "Point", Coordinates: []float64{lon, lat}} // lon, lat order rules from mongodb

	err = collection.Insert(c)
	if err != nil {
		return err
	}

	return nil
}

func (c *City) AllDistrict(city string) ([]City, error) {
	var cities []City
	var err error

	err = checkMongoConnection(mongo)
	if err != nil {
		return cities, err
	}

	collection := mongo.DB("Driver").C(city)
	err = collection.Find(bson.M{}).All(&cities)
	if err != nil {
		return cities, err
	}

	return cities, nil
}

// get Near district in the city with given distance in meters
func (c *City) GetNearestDistrict(cityName string, lat, lon float64, distance int64) (City, error) {
	var err error
	var city City

	err = checkMongoConnection(mongo)
	if err != nil {
		return city, err
	}

	collection := mongo.DB("Driver").C(cityName)

	err = collection.Find(bson.M{
		"location": bson.M{
			"$nearSphere": bson.M{
				"$geometry": bson.M{
					"type":        "Point",
					"coordinates": []float64{lon, lat}, // lon,lat in order is the rule from mongodb
				},
				"$maxDistance": distance,
			},
		},
	}).One(&city)
	if err != nil {
		return city, err
	}

	return city, nil
}
<commit_msg>add logger<commit_after>package city

import (
	"errors"
	"log"
	"os"

	mgo "gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	redis "gopkg.in/redis.v5"
)

type City struct {
	Id       bson.ObjectId `json:"id" bson:"_id,omitempty"`
	Name     string        `json:"name"`
	District int           `json:"district"`
	Location GeoJson       `json:"location"`
}

type GeoJson struct {
	Type        string    `json:"type"`
	Coordinates []float64 `json:"coordinates"`
}

// Global variable for storing database connection.
var mongo *mgo.Session
var redisConn *redis.Client
var logger *log.Logger

func init() {
	logger = log.New(os.Stderr,
		"City Model :: ",
		log.Ldate|log.Ltime|log.Lshortfile)
}

func (c *City) GetConn(mongoConnection *mgo.Session, redisConnection *redis.Client) {
	mongo = mongoConnection
	redisConn = redisConnection
}

// check mongo connection if error return it.
func checkMongoConnection(mongoConnection *mgo.Session) error {
	if mongoConnection == nil {
		return errors.New("No Mongo Connection")
	}

	return nil
}

func (c *City) CreateIndex(collectionName string) error {
	var err error
	err = checkMongoConnection(mongo)
	if err != nil {
		logger.Println(err)
		return err
	}

	index := mgo.Index{
		Key: []string{"$2dsphere:location"},
	}

	// create index from given collection
	collection := mongo.DB("Driver").C(collectionName)
	err = collection.EnsureIndex(index)
	if err != nil {
		logger.Println(err)
		return err
	}

	return nil

}

// Inserting district to mongo database
func (c *City) InsertDistrict(city string, distric int, lat, lon float64) error {
	var err error
	err = checkMongoConnection(mongo)
	if err != nil {
		return err
	}

	collection := mongo.DB("Driver").C(city)

	c.District = distric
	c.Name = city
	c.Location = GeoJson{Type: "Point", Coordinates: []float64{lon, lat}} // lon, lat order rules from mongodb

	err = collection.Insert(c)
	if err != nil {
		return err
	}

	return nil
}

func (c *City) AllDistrict(city string) ([]City, error) {
	var cities []City
	var err error

	err = checkMongoConnection(mongo)
	if err != nil {
		return cities, err
	}

	collection := mongo.DB("Driver").C(city)
	err = collection.Find(bson.M{}).All(&cities)
	if err != nil {
		return cities, err
	}

	return cities, nil
}

// get Near district in the city with given distance in meters
func (c *City) GetNearestDistrict(cityName string, lat, lon float64, distance int64) (City, error) {
	var err error
	var city City

	err = checkMongoConnection(mongo)
	if err != nil {
		logger.Println(err)
		return city, err
	}

	collection := mongo.DB("Driver").C(cityName)

	err = collection.Find(bson.M{
		"location": bson.M{
			"$nearSphere": bson.M{
				"$geometry": bson.M{
					"type":        "Point",
					"coordinates": []float64{lon, lat}, // lon,lat in order is the rule from mongodb
				},
				"$maxDistance": distance,
			},
		},
	}).One(&city)
	if err != nil {
		logger.Println(err)
		return city, err
	}

	return city, nil
}
<|endoftext|>


<commit_before>//go:generate scaneo $GOFILE

package model

import "time"

// User returns model object for user.
type User struct {
	ID      int64      `json:"id"`
	Name    string     `json:"name"`
	Email   string     `json:"email"`
	Salt    string     `json:"salt"`
	Salted  string     `json:"salted"`
	Created *time.Time `json:"created"`
	Updated *time.Time `json:"updated"`
}

// Mask masks user information
func (u User) Mask() User {
	return User{
		ID:      u.ID,
		Name:    u.Name,
		Email:   u.Email,
		Created: u.Created,
		Updated: u.Updated,
	}
}

// Article returns model object for article.
type Article struct {
	ID      int64      `json:"id"`
	Title   string     `json:"title"`
	Body    string     `json:"body"`
	Created *time.Time `json:"created"`
	Updated *time.Time `json:"updated"`
}
<commit_msg>remove unused Mask func<commit_after>//go:generate scaneo $GOFILE

package model

import "time"

// User returns model object for user.
type User struct {
	ID      int64      `json:"id"`
	Name    string     `json:"name"`
	Email   string     `json:"email"`
	Salt    string     `json:"salt"`
	Salted  string     `json:"salted"`
	Created *time.Time `json:"created"`
	Updated *time.Time `json:"updated"`
}

// Article returns model object for article.
type Article struct {
	ID      int64      `json:"id"`
	Title   string     `json:"title"`
	Body    string     `json:"body"`
	Created *time.Time `json:"created"`
	Updated *time.Time `json:"updated"`
}
<|endoftext|>


<commit_before>package rc

import (
	"crypto/tls"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"golang.org/x/oauth2"

	"github.com/concourse/go-concourse/concourse"
	"github.com/mattn/go-isatty"

	"gopkg.in/yaml.v2"
)

var ErrNoTargetSpecified = errors.New("no target specified")

type UnknownTargetError struct {
	TargetName TargetName
}

func (err UnknownTargetError) Error() string {
	return fmt.Sprintf("unknown target: %s", err.TargetName)
}

type TargetProps struct {
	API      string       `yaml:"api"`
	Insecure bool         `yaml:"insecure,omitempty"`
	Token    *TargetToken `yaml:"token,omitempty"`
}

type TargetToken struct {
	Type  string `yaml:"type"`
	Value string `yaml:"value"`
}

type targetDetailsYAML struct {
	Targets map[TargetName]TargetProps
}

func NewTarget(api string, insecure bool, token *TargetToken) TargetProps {
	return TargetProps{
		API:      strings.TrimRight(api, "/"),
		Insecure: insecure,
		Token:    token,
	}
}

func SaveTarget(targetName TargetName, api string, insecure bool, token *TargetToken) error {
	flyrc := filepath.Join(userHomeDir(), ".flyrc")
	flyTargets, err := loadTargets(flyrc)
	if err != nil {
		return err
	}

	newInfo := flyTargets.Targets[targetName]
	newInfo.API = api
	newInfo.Insecure = insecure
	newInfo.Token = token

	flyTargets.Targets[targetName] = newInfo

	return writeTargets(flyrc, flyTargets)
}

func SelectTarget(selectedTarget TargetName) (TargetProps, error) {
	if selectedTarget == "" {
		return TargetProps{}, ErrNoTargetSpecified
	}

	flyrc := filepath.Join(userHomeDir(), ".flyrc")
	flyTargets, err := loadTargets(flyrc)
	if err != nil {
		return TargetProps{}, err
	}

	target, ok := flyTargets.Targets[selectedTarget]
	if !ok {
		return TargetProps{}, UnknownTargetError{selectedTarget}
	}

	return target, nil
}

func NewClient(atcURL string, insecure bool) concourse.Client {
	var tlsConfig *tls.Config
	if insecure {
		tlsConfig = &tls.Config{InsecureSkipVerify: insecure}
	}

	var transport http.RoundTripper

	transport = &http.Transport{
		TLSClientConfig: tlsConfig,
		Dial: (&net.Dialer{
			Timeout: 10 * time.Second,
		}).Dial,
	}

	return concourse.NewClient(atcURL, &http.Client{
		Transport: transport,
	})
}

func TargetClient(selectedTarget TargetName) (concourse.Client, error) {
	targetClient, err := CommandTargetClient(selectedTarget, nil)
	if err != nil {
		return nil, err
	}
	if isatty.IsTerminal(os.Stdout.Fd()) {
		fmt.Printf("targeting %s\n\n", targetClient.URL())
	}
	return targetClient, nil
}

func CommandTargetClient(selectedTarget TargetName, commandInsecure *bool) (concourse.Client, error) {
	target, err := SelectTarget(selectedTarget)
	if err != nil {
		return nil, err
	}

	var token *oauth2.Token
	if target.Token != nil {
		token = &oauth2.Token{
			TokenType:   target.Token.Type,
			AccessToken: target.Token.Value,
		}
	}

	var tlsConfig *tls.Config
	if commandInsecure != nil {
		tlsConfig = &tls.Config{InsecureSkipVerify: *commandInsecure}
	} else if target.Insecure {
		tlsConfig = &tls.Config{InsecureSkipVerify: true}
	}

	var transport http.RoundTripper

	transport = &http.Transport{
		TLSClientConfig: tlsConfig,
		Dial: (&net.Dialer{
			Timeout: 10 * time.Second,
		}).Dial,
	}

	if token != nil {
		transport = &oauth2.Transport{
			Source: oauth2.StaticTokenSource(token),
			Base:   transport,
		}
	}

	httpClient := &http.Client{
		Transport: transport,
	}

	return concourse.NewClient(target.API, httpClient), nil
}

func userHomeDir() string {
	if runtime.GOOS == "windows" {
		home := os.Getenv("USERPROFILE")
		if home == "" {
			home = os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
		}

		if home == "" {
			panic("could not detect home directory for .flyrc")
		}

		return home
	}

	return os.Getenv("HOME")
}

func loadTargets(configFileLocation string) (*targetDetailsYAML, error) {
	var flyTargets *targetDetailsYAML

	if _, err := os.Stat(configFileLocation); err == nil {
		flyTargetsBytes, err := ioutil.ReadFile(configFileLocation)
		if err != nil {
			return nil, fmt.Errorf("could not read %s", configFileLocation)
		}

		err = yaml.Unmarshal(flyTargetsBytes, &flyTargets)
		if err != nil {
			return nil, fmt.Errorf("could not unmarshal %s", configFileLocation)
		}
	}

	if flyTargets == nil {
		return &targetDetailsYAML{Targets: map[TargetName]TargetProps{}}, nil
	}

	return flyTargets, nil
}

func writeTargets(configFileLocation string, targetsToWrite *targetDetailsYAML) error {
	yamlBytes, err := yaml.Marshal(targetsToWrite)
	if err != nil {
		return fmt.Errorf("could not marshal %s", configFileLocation)
	}

	err = ioutil.WriteFile(configFileLocation, yamlBytes, os.ModePerm)
	if err != nil {
		return fmt.Errorf("could not write %s", configFileLocation)
	}

	return nil
}
<commit_msg>Fix targetting a concourse over a proxy<commit_after>package rc

import (
	"crypto/tls"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"golang.org/x/oauth2"

	"github.com/concourse/go-concourse/concourse"
	"github.com/mattn/go-isatty"

	"gopkg.in/yaml.v2"
)

var ErrNoTargetSpecified = errors.New("no target specified")

type UnknownTargetError struct {
	TargetName TargetName
}

func (err UnknownTargetError) Error() string {
	return fmt.Sprintf("unknown target: %s", err.TargetName)
}

type TargetProps struct {
	API      string       `yaml:"api"`
	Insecure bool         `yaml:"insecure,omitempty"`
	Token    *TargetToken `yaml:"token,omitempty"`
}

type TargetToken struct {
	Type  string `yaml:"type"`
	Value string `yaml:"value"`
}

type targetDetailsYAML struct {
	Targets map[TargetName]TargetProps
}

func NewTarget(api string, insecure bool, token *TargetToken) TargetProps {
	return TargetProps{
		API:      strings.TrimRight(api, "/"),
		Insecure: insecure,
		Token:    token,
	}
}

func SaveTarget(targetName TargetName, api string, insecure bool, token *TargetToken) error {
	flyrc := filepath.Join(userHomeDir(), ".flyrc")
	flyTargets, err := loadTargets(flyrc)
	if err != nil {
		return err
	}

	newInfo := flyTargets.Targets[targetName]
	newInfo.API = api
	newInfo.Insecure = insecure
	newInfo.Token = token

	flyTargets.Targets[targetName] = newInfo

	return writeTargets(flyrc, flyTargets)
}

func SelectTarget(selectedTarget TargetName) (TargetProps, error) {
	if selectedTarget == "" {
		return TargetProps{}, ErrNoTargetSpecified
	}

	flyrc := filepath.Join(userHomeDir(), ".flyrc")
	flyTargets, err := loadTargets(flyrc)
	if err != nil {
		return TargetProps{}, err
	}

	target, ok := flyTargets.Targets[selectedTarget]
	if !ok {
		return TargetProps{}, UnknownTargetError{selectedTarget}
	}

	return target, nil
}

func NewClient(atcURL string, insecure bool) concourse.Client {
	var tlsConfig *tls.Config
	if insecure {
		tlsConfig = &tls.Config{InsecureSkipVerify: insecure}
	}

	var transport http.RoundTripper

	transport = &http.Transport{
		TLSClientConfig: tlsConfig,
		Dial: (&net.Dialer{
			Timeout: 10 * time.Second,
		}).Dial,
		Proxy: http.ProxyFromEnvironment,
	}

	return concourse.NewClient(atcURL, &http.Client{
		Transport: transport,
	})
}

func TargetClient(selectedTarget TargetName) (concourse.Client, error) {
	targetClient, err := CommandTargetClient(selectedTarget, nil)
	if err != nil {
		return nil, err
	}
	if isatty.IsTerminal(os.Stdout.Fd()) {
		fmt.Printf("targeting %s\n\n", targetClient.URL())
	}
	return targetClient, nil
}

func CommandTargetClient(selectedTarget TargetName, commandInsecure *bool) (concourse.Client, error) {
	target, err := SelectTarget(selectedTarget)
	if err != nil {
		return nil, err
	}

	var token *oauth2.Token
	if target.Token != nil {
		token = &oauth2.Token{
			TokenType:   target.Token.Type,
			AccessToken: target.Token.Value,
		}
	}

	var tlsConfig *tls.Config
	if commandInsecure != nil {
		tlsConfig = &tls.Config{InsecureSkipVerify: *commandInsecure}
	} else if target.Insecure {
		tlsConfig = &tls.Config{InsecureSkipVerify: true}
	}

	var transport http.RoundTripper

	transport = &http.Transport{
		TLSClientConfig: tlsConfig,
		Dial: (&net.Dialer{
			Timeout: 10 * time.Second,
		}).Dial,
		Proxy: http.ProxyFromEnvironment,
	}

	if token != nil {
		transport = &oauth2.Transport{
			Source: oauth2.StaticTokenSource(token),
			Base:   transport,
		}
	}

	httpClient := &http.Client{
		Transport: transport,
	}

	return concourse.NewClient(target.API, httpClient), nil
}

func userHomeDir() string {
	if runtime.GOOS == "windows" {
		home := os.Getenv("USERPROFILE")
		if home == "" {
			home = os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
		}

		if home == "" {
			panic("could not detect home directory for .flyrc")
		}

		return home
	}

	return os.Getenv("HOME")
}

func loadTargets(configFileLocation string) (*targetDetailsYAML, error) {
	var flyTargets *targetDetailsYAML

	if _, err := os.Stat(configFileLocation); err == nil {
		flyTargetsBytes, err := ioutil.ReadFile(configFileLocation)
		if err != nil {
			return nil, fmt.Errorf("could not read %s", configFileLocation)
		}

		err = yaml.Unmarshal(flyTargetsBytes, &flyTargets)
		if err != nil {
			return nil, fmt.Errorf("could not unmarshal %s", configFileLocation)
		}
	}

	if flyTargets == nil {
		return &targetDetailsYAML{Targets: map[TargetName]TargetProps{}}, nil
	}

	return flyTargets, nil
}

func writeTargets(configFileLocation string, targetsToWrite *targetDetailsYAML) error {
	yamlBytes, err := yaml.Marshal(targetsToWrite)
	if err != nil {
		return fmt.Errorf("could not marshal %s", configFileLocation)
	}

	err = ioutil.WriteFile(configFileLocation, yamlBytes, os.ModePerm)
	if err != nil {
		return fmt.Errorf("could not write %s", configFileLocation)
	}

	return nil
}
<|endoftext|>


<commit_before>package models

import (
	"io/ioutil"

	"gopkg.in/yaml.v2"
)

type Setting struct {
	Database struct {
		Host       string
		DbName     string
		TokenTable string
		UserTable  string
	}
	Ssl struct {
		Key         string
		Certificate string
	}
	Router struct {
		Register string
		Login    string
		Validate string
	}
	Domain string
	IP     string
}

var Set Setting

func LoadSettings(path string) error {
	text, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}

	if err = yaml.Unmarshal(text, &Set); err != nil {
		return err
	}

	return nil
}
<commit_msg>Add logout config<commit_after>package models

import (
	"io/ioutil"

	"gopkg.in/yaml.v2"
)

type Setting struct {
	Database struct {
		Host       string
		DbName     string
		TokenTable string
		UserTable  string
	}
	Ssl struct {
		Key         string
		Certificate string
	}
	Router struct {
		Register string
		Login    string
		Validate string
		Logout   string
	}
	Domain string
	IP     string
}

var Set Setting

func LoadSettings(path string) error {
	text, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}

	if err = yaml.Unmarshal(text, &Set); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2018 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mock

import (
	"fmt"
	"reflect"
	"sync"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"github.com/projectcalico/felix/config"
	"github.com/projectcalico/felix/proto"
	"github.com/projectcalico/libcalico-go/lib/set"
)

type MockDataplane struct {
	sync.Mutex

	inSync                         bool
	ipSets                         map[string]set.Set
	activePolicies                 set.Set
	activeUntrackedPolicies        set.Set
	activePreDNATPolicies          set.Set
	activeProfiles                 set.Set
	endpointToPolicyOrder          map[string][]TierInfo
	endpointToUntrackedPolicyOrder map[string][]TierInfo
	endpointToPreDNATPolicyOrder   map[string][]TierInfo
	config                         map[string]string
}

func (d *MockDataplane) InSync() bool {
	d.Lock()
	defer d.Unlock()

	return d.inSync
}

func (d *MockDataplane) IPSets() map[string]set.Set {
	d.Lock()
	defer d.Unlock()

	copy := map[string]set.Set{}
	for k, v := range d.ipSets {
		copy[k] = v.Copy()
	}
	return copy
}

func (d *MockDataplane) ActivePolicies() set.Set {
	d.Lock()
	defer d.Unlock()

	return d.activePolicies.Copy()
}
func (d *MockDataplane) ActiveUntrackedPolicies() set.Set {
	d.Lock()
	defer d.Unlock()

	return d.activeUntrackedPolicies.Copy()
}
func (d *MockDataplane) ActivePreDNATPolicies() set.Set {
	d.Lock()
	defer d.Unlock()

	return d.activePreDNATPolicies.Copy()
}
func (d *MockDataplane) ActiveProfiles() set.Set {
	d.Lock()
	defer d.Unlock()

	return d.activeProfiles.Copy()
}
func (d *MockDataplane) EndpointToPolicyOrder() map[string][]TierInfo {
	d.Lock()
	defer d.Unlock()

	return copyPolOrder(d.endpointToPolicyOrder)
}
func (d *MockDataplane) EndpointToUntrackedPolicyOrder() map[string][]TierInfo {
	d.Lock()
	defer d.Unlock()

	return copyPolOrder(d.endpointToUntrackedPolicyOrder)
}
func (d *MockDataplane) EndpointToPreDNATPolicyOrder() map[string][]TierInfo {
	d.Lock()
	defer d.Unlock()

	return copyPolOrder(d.endpointToPreDNATPolicyOrder)
}

func copyPolOrder(in map[string][]TierInfo) map[string][]TierInfo {
	copy := map[string][]TierInfo{}
	for k, v := range in {
		if v == nil {
			copy[k] = nil
		}
		vCopy := make([]TierInfo, len(v))
		for i := range v {
			vCopy[i] = v[i]
		}
		copy[k] = vCopy
	}
	return copy
}

func (d *MockDataplane) Config() map[string]string {
	d.Lock()
	defer d.Unlock()

	if d.config == nil {
		return nil
	}
	copy := map[string]string{}
	for k, v := range d.config {
		copy[k] = v
	}
	return copy
}

func NewMockDataplane() *MockDataplane {
	s := &MockDataplane{
		ipSets:                         make(map[string]set.Set),
		activePolicies:                 set.New(),
		activeProfiles:                 set.New(),
		activeUntrackedPolicies:        set.New(),
		activePreDNATPolicies:          set.New(),
		endpointToPolicyOrder:          make(map[string][]TierInfo),
		endpointToUntrackedPolicyOrder: make(map[string][]TierInfo),
		endpointToPreDNATPolicyOrder:   make(map[string][]TierInfo),
	}
	return s
}

func (d *MockDataplane) OnEvent(event interface{}) {
	d.Lock()
	defer d.Unlock()

	evType := reflect.TypeOf(event).String()
	fmt.Fprintf(GinkgoWriter, "       <- Event: %v %v\n", evType, event)
	Expect(event).NotTo(BeNil())
	Expect(reflect.TypeOf(event).Kind()).To(Equal(reflect.Ptr))
	switch event := event.(type) {
	case *proto.InSync:
		d.inSync = true
	case *proto.IPSetUpdate:
		newMembers := set.New()
		for _, ip := range event.Members {
			newMembers.Add(ip)
		}
		d.ipSets[event.Id] = newMembers
	case *proto.IPSetDeltaUpdate:
		members, ok := d.ipSets[event.Id]
		if !ok {
			Fail(fmt.Sprintf("IP set delta to missing ipset %v", event.Id))
			return
		}

		for _, ip := range event.AddedMembers {
			Expect(members.Contains(ip)).To(BeFalse(),
				fmt.Sprintf("IP Set %v already contained added IP %v",
					event.Id, ip))
			members.Add(ip)
		}
		for _, ip := range event.RemovedMembers {
			Expect(members.Contains(ip)).To(BeTrue(),
				fmt.Sprintf("IP Set %v did not contain removed IP %v",
					event.Id, ip))
			members.Discard(ip)
		}
	case *proto.IPSetRemove:
		_, ok := d.ipSets[event.Id]
		if !ok {
			Fail(fmt.Sprintf("IP set remove for unknown ipset %v", event.Id))
			return
		}
		delete(d.ipSets, event.Id)
	case *proto.ActivePolicyUpdate:
		// TODO: check rules against expected rules
		policyID := *event.Id
		d.activePolicies.Add(policyID)
		if event.Policy.Untracked {
			d.activeUntrackedPolicies.Add(policyID)
		} else {
			d.activeUntrackedPolicies.Discard(policyID)
		}
		if event.Policy.PreDnat {
			d.activePreDNATPolicies.Add(policyID)
		} else {
			d.activePreDNATPolicies.Discard(policyID)
		}
	case *proto.ActivePolicyRemove:
		policyID := *event.Id
		d.activePolicies.Discard(policyID)
		d.activeUntrackedPolicies.Discard(policyID)
		d.activePreDNATPolicies.Discard(policyID)
	case *proto.ActiveProfileUpdate:
		// TODO: check rules against expected rules
		d.activeProfiles.Add(*event.Id)
	case *proto.ActiveProfileRemove:
		d.activeProfiles.Discard(*event.Id)
	case *proto.WorkloadEndpointUpdate:
		tiers := event.Endpoint.Tiers
		tierInfos := make([]TierInfo, len(tiers))
		for i, tier := range event.Endpoint.Tiers {
			tierInfos[i].Name = tier.Name
			tierInfos[i].IngressPolicyNames = tier.IngressPolicies
			tierInfos[i].EgressPolicyNames = tier.EgressPolicies
		}
		id := workloadId(*event.Id)
		d.endpointToPolicyOrder[id.String()] = tierInfos
		d.endpointToUntrackedPolicyOrder[id.String()] = []TierInfo{}
		d.endpointToPreDNATPolicyOrder[id.String()] = []TierInfo{}
	case *proto.WorkloadEndpointRemove:
		id := workloadId(*event.Id)
		delete(d.endpointToPolicyOrder, id.String())
		delete(d.endpointToUntrackedPolicyOrder, id.String())
		delete(d.endpointToPreDNATPolicyOrder, id.String())
	case *proto.HostEndpointUpdate:
		tiers := event.Endpoint.Tiers
		tierInfos := make([]TierInfo, len(tiers))
		for i, tier := range tiers {
			tierInfos[i].Name = tier.Name
			tierInfos[i].IngressPolicyNames = tier.IngressPolicies
			tierInfos[i].EgressPolicyNames = tier.EgressPolicies
		}
		id := hostEpId(*event.Id)
		d.endpointToPolicyOrder[id.String()] = tierInfos

		uTiers := event.Endpoint.UntrackedTiers
		uTierInfos := make([]TierInfo, len(uTiers))
		for i, tier := range uTiers {
			uTierInfos[i].Name = tier.Name
			uTierInfos[i].IngressPolicyNames = tier.IngressPolicies
			uTierInfos[i].EgressPolicyNames = tier.EgressPolicies
		}
		d.endpointToUntrackedPolicyOrder[id.String()] = uTierInfos

		pTiers := event.Endpoint.PreDnatTiers
		pTierInfos := make([]TierInfo, len(pTiers))
		for i, tier := range pTiers {
			pTierInfos[i].Name = tier.Name
			pTierInfos[i].IngressPolicyNames = tier.IngressPolicies
			pTierInfos[i].EgressPolicyNames = tier.EgressPolicies
		}
		d.endpointToPreDNATPolicyOrder[id.String()] = pTierInfos
	case *proto.HostEndpointRemove:
		id := hostEpId(*event.Id)
		delete(d.endpointToPolicyOrder, id.String())
		delete(d.endpointToUntrackedPolicyOrder, id.String())
		delete(d.endpointToPreDNATPolicyOrder, id.String())
	}
}

func (d *MockDataplane) UpdateFrom(map[string]string, config.Source) (changed bool, err error) {
	return
}

func (d *MockDataplane) RawValues() map[string]string {
	return d.Config()
}

type TierInfo struct {
	Name               string
	IngressPolicyNames []string
	EgressPolicyNames  []string
}

type workloadId proto.WorkloadEndpointID

func (w *workloadId) String() string {
	return fmt.Sprintf("%v/%v/%v",
		w.OrchestratorId, w.WorkloadId, w.EndpointId)
}

type hostEpId proto.HostEndpointID

func (i *hostEpId) String() string {
	return i.EndpointId
}
<commit_msg>Improve mock dataplane: catch out-of-order updates.<commit_after>// Copyright (c) 2018 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mock

import (
	"fmt"
	"reflect"
	"sync"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"github.com/projectcalico/felix/config"
	"github.com/projectcalico/felix/proto"
	"github.com/projectcalico/libcalico-go/lib/set"
)

type MockDataplane struct {
	sync.Mutex

	inSync                         bool
	ipSets                         map[string]set.Set
	activePolicies                 set.Set
	activeUntrackedPolicies        set.Set
	activePreDNATPolicies          set.Set
	activeProfiles                 set.Set
	endpointToPolicyOrder          map[string][]TierInfo
	endpointToUntrackedPolicyOrder map[string][]TierInfo
	endpointToPreDNATPolicyOrder   map[string][]TierInfo
	endpointToAllPolicyIDs         map[string][]proto.PolicyID
	endpointToProfiles             map[string][]string
	config                         map[string]string
}

func (d *MockDataplane) InSync() bool {
	d.Lock()
	defer d.Unlock()

	return d.inSync
}

func (d *MockDataplane) IPSets() map[string]set.Set {
	d.Lock()
	defer d.Unlock()

	copy := map[string]set.Set{}
	for k, v := range d.ipSets {
		copy[k] = v.Copy()
	}
	return copy
}

func (d *MockDataplane) ActivePolicies() set.Set {
	d.Lock()
	defer d.Unlock()

	return d.activePolicies.Copy()
}
func (d *MockDataplane) ActiveUntrackedPolicies() set.Set {
	d.Lock()
	defer d.Unlock()

	return d.activeUntrackedPolicies.Copy()
}
func (d *MockDataplane) ActivePreDNATPolicies() set.Set {
	d.Lock()
	defer d.Unlock()

	return d.activePreDNATPolicies.Copy()
}
func (d *MockDataplane) ActiveProfiles() set.Set {
	d.Lock()
	defer d.Unlock()

	return d.activeProfiles.Copy()
}
func (d *MockDataplane) EndpointToPolicyOrder() map[string][]TierInfo {
	d.Lock()
	defer d.Unlock()

	return copyPolOrder(d.endpointToPolicyOrder)
}
func (d *MockDataplane) EndpointToUntrackedPolicyOrder() map[string][]TierInfo {
	d.Lock()
	defer d.Unlock()

	return copyPolOrder(d.endpointToUntrackedPolicyOrder)
}
func (d *MockDataplane) EndpointToPreDNATPolicyOrder() map[string][]TierInfo {
	d.Lock()
	defer d.Unlock()

	return copyPolOrder(d.endpointToPreDNATPolicyOrder)
}

func copyPolOrder(in map[string][]TierInfo) map[string][]TierInfo {
	copy := map[string][]TierInfo{}
	for k, v := range in {
		if v == nil {
			copy[k] = nil
		}
		vCopy := make([]TierInfo, len(v))
		for i := range v {
			vCopy[i] = v[i]
		}
		copy[k] = vCopy
	}
	return copy
}

func (d *MockDataplane) Config() map[string]string {
	d.Lock()
	defer d.Unlock()

	if d.config == nil {
		return nil
	}
	copy := map[string]string{}
	for k, v := range d.config {
		copy[k] = v
	}
	return copy
}

func NewMockDataplane() *MockDataplane {
	s := &MockDataplane{
		ipSets:                         make(map[string]set.Set),
		activePolicies:                 set.New(),
		activeProfiles:                 set.New(),
		activeUntrackedPolicies:        set.New(),
		activePreDNATPolicies:          set.New(),
		endpointToPolicyOrder:          make(map[string][]TierInfo),
		endpointToUntrackedPolicyOrder: make(map[string][]TierInfo),
		endpointToPreDNATPolicyOrder:   make(map[string][]TierInfo),
		endpointToProfiles:             make(map[string][]string),
		endpointToAllPolicyIDs:         make(map[string][]proto.PolicyID),
	}
	return s
}

func (d *MockDataplane) OnEvent(event interface{}) {
	d.Lock()
	defer d.Unlock()

	evType := reflect.TypeOf(event).String()
	fmt.Fprintf(GinkgoWriter, "       <- Event: %v %v\n", evType, event)
	Expect(event).NotTo(BeNil())
	Expect(reflect.TypeOf(event).Kind()).To(Equal(reflect.Ptr))
	switch event := event.(type) {
	case *proto.InSync:
		d.inSync = true
	case *proto.IPSetUpdate:
		newMembers := set.New()
		for _, ip := range event.Members {
			newMembers.Add(ip)
		}
		d.ipSets[event.Id] = newMembers
	case *proto.IPSetDeltaUpdate:
		members, ok := d.ipSets[event.Id]
		if !ok {
			Fail(fmt.Sprintf("IP set delta to missing ipset %v", event.Id))
			return
		}

		for _, ip := range event.AddedMembers {
			Expect(members.Contains(ip)).To(BeFalse(),
				fmt.Sprintf("IP Set %v already contained added IP %v",
					event.Id, ip))
			members.Add(ip)
		}
		for _, ip := range event.RemovedMembers {
			Expect(members.Contains(ip)).To(BeTrue(),
				fmt.Sprintf("IP Set %v did not contain removed IP %v",
					event.Id, ip))
			members.Discard(ip)
		}
	case *proto.IPSetRemove:
		_, ok := d.ipSets[event.Id]
		if !ok {
			Fail(fmt.Sprintf("IP set remove for unknown ipset %v", event.Id))
			return
		}
		delete(d.ipSets, event.Id)
	case *proto.ActivePolicyUpdate:
		// TODO: check rules against expected rules
		policyID := *event.Id
		d.activePolicies.Add(policyID)
		if event.Policy.Untracked {
			d.activeUntrackedPolicies.Add(policyID)
		} else {
			d.activeUntrackedPolicies.Discard(policyID)
		}
		if event.Policy.PreDnat {
			d.activePreDNATPolicies.Add(policyID)
		} else {
			d.activePreDNATPolicies.Discard(policyID)
		}
	case *proto.ActivePolicyRemove:
		policyID := *event.Id
		for ep, allPols := range d.endpointToAllPolicyIDs {
			for _, p := range allPols {
				if policyID == p {
					Fail(fmt.Sprintf("Policy %s removed while still in use by endpoint %s", p, ep))
				}
			}
		}
		d.activePolicies.Discard(policyID)
		d.activeUntrackedPolicies.Discard(policyID)
		d.activePreDNATPolicies.Discard(policyID)
	case *proto.ActiveProfileUpdate:
		// TODO: check rules against expected rules
		d.activeProfiles.Add(*event.Id)
	case *proto.ActiveProfileRemove:
		for ep, profs := range d.endpointToProfiles {
			for _, p := range profs {
				if p == event.Id.Name {
					Fail(fmt.Sprintf("Profile %s removed while still in use by endpoint %s", p, ep))
				}
			}
		}
		d.activeProfiles.Discard(*event.Id)
	case *proto.WorkloadEndpointUpdate:
		tiers := event.Endpoint.Tiers
		tierInfos := make([]TierInfo, len(tiers))
		var allPolsIDs []proto.PolicyID
		for i, tier := range event.Endpoint.Tiers {
			tierInfos[i].Name = tier.Name
			tierInfos[i].IngressPolicyNames = tier.IngressPolicies
			tierInfos[i].EgressPolicyNames = tier.EgressPolicies

			// Check that all the policies referenced by the endpoint are already present, which
			// is one of the guarantees provided by the EventSequencer.
			var combinedPolNames []string
			combinedPolNames = append(combinedPolNames, tier.IngressPolicies...)
			combinedPolNames = append(combinedPolNames, tier.EgressPolicies...)
			for _, polName := range combinedPolNames {
				polID := proto.PolicyID{Tier: tier.Name, Name: polName}
				allPolsIDs = append(allPolsIDs, polID)
				Expect(d.activePolicies.Contains(polID)).To(BeTrue(),
					fmt.Sprintf("Expected policy %v referenced by workload endpoint "+
						"update %v to be active", polID, event))
			}
		}
		id := workloadId(*event.Id)
		d.endpointToPolicyOrder[id.String()] = tierInfos
		d.endpointToUntrackedPolicyOrder[id.String()] = []TierInfo{}
		d.endpointToPreDNATPolicyOrder[id.String()] = []TierInfo{}
		d.endpointToAllPolicyIDs[id.String()] = allPolsIDs

		// Check that all the profiles referenced by the endpoint are already present, which
		// is one of the guarantees provided by the EventSequencer.
		for _, profName := range event.Endpoint.ProfileIds {
			profID := proto.ProfileID{Name: profName}
			Expect(d.activeProfiles.Contains(profID)).To(BeTrue(),
				fmt.Sprintf("Expected profile %v referenced by workload endpoint "+
					"update %v to be active", profID, event))
		}
		d.endpointToProfiles[id.String()] = event.Endpoint.ProfileIds
	case *proto.WorkloadEndpointRemove:
		id := workloadId(*event.Id)
		delete(d.endpointToPolicyOrder, id.String())
		delete(d.endpointToUntrackedPolicyOrder, id.String())
		delete(d.endpointToPreDNATPolicyOrder, id.String())
		delete(d.endpointToProfiles, id.String())
		delete(d.endpointToAllPolicyIDs, id.String())
	case *proto.HostEndpointUpdate:
		tiers := event.Endpoint.Tiers
		tierInfos := make([]TierInfo, len(tiers))
		for i, tier := range tiers {
			tierInfos[i].Name = tier.Name
			tierInfos[i].IngressPolicyNames = tier.IngressPolicies
			tierInfos[i].EgressPolicyNames = tier.EgressPolicies
		}
		id := hostEpId(*event.Id)
		d.endpointToPolicyOrder[id.String()] = tierInfos

		uTiers := event.Endpoint.UntrackedTiers
		uTierInfos := make([]TierInfo, len(uTiers))
		for i, tier := range uTiers {
			uTierInfos[i].Name = tier.Name
			uTierInfos[i].IngressPolicyNames = tier.IngressPolicies
			uTierInfos[i].EgressPolicyNames = tier.EgressPolicies
		}
		d.endpointToUntrackedPolicyOrder[id.String()] = uTierInfos

		pTiers := event.Endpoint.PreDnatTiers
		pTierInfos := make([]TierInfo, len(pTiers))
		for i, tier := range pTiers {
			pTierInfos[i].Name = tier.Name
			pTierInfos[i].IngressPolicyNames = tier.IngressPolicies
			pTierInfos[i].EgressPolicyNames = tier.EgressPolicies
		}
		d.endpointToPreDNATPolicyOrder[id.String()] = pTierInfos
	case *proto.HostEndpointRemove:
		id := hostEpId(*event.Id)
		delete(d.endpointToPolicyOrder, id.String())
		delete(d.endpointToUntrackedPolicyOrder, id.String())
		delete(d.endpointToPreDNATPolicyOrder, id.String())
	}
}

func (d *MockDataplane) UpdateFrom(map[string]string, config.Source) (changed bool, err error) {
	return
}

func (d *MockDataplane) RawValues() map[string]string {
	return d.Config()
}

type TierInfo struct {
	Name               string
	IngressPolicyNames []string
	EgressPolicyNames  []string
}

type workloadId proto.WorkloadEndpointID

func (w *workloadId) String() string {
	return fmt.Sprintf("%v/%v/%v",
		w.OrchestratorId, w.WorkloadId, w.EndpointId)
}

type hostEpId proto.HostEndpointID

func (i *hostEpId) String() string {
	return i.EndpointId
}
<|endoftext|>


<commit_before>// HDFS Config Details: https://github.com/zyxar/hdfs

package main

import (
	"bufio"
	"crypto/md5"
	"encoding/hex"
	"encoding/json"
	"flag"
	"fmt"
	//"github.com/zyxar/hdfs"
	"io/ioutil"
	"launchpad.net/goamz/aws"
	"launchpad.net/goamz/s3"
	"os"
	"os/user"
	"path"
	"runtime"
	"strings"
	"time"
)

type continueOn struct {
	nextMarker string
	truncated  bool
	finished   bool
}

type ConfigSettingsType struct {
	AWSCredentials AWSCredentialsType
	System         SystemType
}

type AWSCredentialsType struct {
	AccessKey string
	SecretKey string
}

type SystemType struct {
	javaHome     string
	ldPath       string
	hadoopServer string
	hadoopPort   uint16
}

type fileInfo struct {
	FileName      string
	WriteLocation string
	MD5Sum        string
	File          []byte
	Size          int
}

// Global Constants

var workers int = runtime.NumCPU()

const hadoopServer string = "default"
const hadoopPort int = 0

func getandsave(b s3.Bucket, prefix string, left, right chan continueOn, done chan<- struct{}, threadNum int, directory string, maxMarker int, hadoop bool, hashCheck bool) {
	grab := <-right

	if grab.finished == true {
		left <- grab
		done <- struct{}{}
		return
	}
	//	if grab.truncated == false {
	//		left <- continueOn{truncated: false, nextMarker: ""}
	//		done <- struct{}{}
	//		return
	//	}

	resp, err := b.List(prefix, "", grab.nextMarker, maxMarker)
	if err != nil {
		fmt.Println(err)
		for e := 0; e <= 5 && err != nil; e++ {
			time.Sleep(3 * time.Second)
			resp, err = b.List(prefix, "", grab.nextMarker, maxMarker)
			if e == 5 {
				panic("Directory List Failed")
			}
		}
	}

	onward := continueOn{resp.Contents[len(resp.Contents)-1].Key, resp.IsTruncated, false}

	switch onward.truncated {
	case true:
		left <- onward
	case false:
		left <- continueOn{nextMarker: "", truncated: false, finished: true}
	}

	//	if len(resp.Contents) == 1 {
	//		fmt.Println("Not Downloading")
	//		done <- struct{}{}
	//		return
	//
	startIndex := 0
	if grab.nextMarker == "" {
		startIndex = 1
	}
	var finished string = ""
	if hadoop == true {
		fmt.Println("Remove me")
		//finished = hadoopWrite(*resp, b, directory, startIndex)
	} else {
		finished = standardWrite(*resp, b, directory, startIndex, hashCheck)
	}

	//	for j := startIndex; j <= len(resp.Contents)-1; j++ {
	//
	//		fileData := s3Get(j, *resp, b, directory)
	//
	//
	//
	//
	//
	//
	//	}

	fmt.Println(finished)

	done <- struct{}{}

}

func awaitCompletion(done <-chan struct{}) {
	for f := 0; f < workers; f++ {
		<-done
	}
	//	close(leftmost)
}

func s3Get(j int, resp s3.ListResp, b s3.Bucket, directory string) fileInfo {

	FileGet := resp.Contents[j].Key

	f, err := b.Get(FileGet)
	if err != nil {
		fmt.Println(err)
		for e := 0; e <= 5 && err != nil; e++ {
			time.Sleep(3 * time.Second)
			f, err = b.Get(FileGet)
			if e == 5 {
				panic("File Download Failed")
			}
		}
	}

	dir, filename := path.Split(FileGet)

	return fileInfo{
		FileName:      filename,
		WriteLocation: path.Join(directory, dir),
		MD5Sum:        resp.Contents[j].ETag,
		File:          f,
		Size:          len(f),
	}

}

//func hadoopWrite(resp s3.ListResp, b s3.Bucket, directory string, startIndex int) string {
//
//	currentUser, err := user.Current()
//	if err != nil {
//		panic(err)
//	}
//
//	fs, err := hdfs.ConnectAsUser(hadoopServer, 0, currentUser.Name)
//	if err != nil {
//		panic(err)
//	}
//
//	fmt.Println("Running files: ", len(resp.Contents)-1)
//
//	for j := startIndex; j <= len(resp.Contents)-1; j++ {
//		//
//		//		FileGet := resp.Contents[j].Key
//		//
//		//		f, err := b.Get(FileGet)
//		//		if err != nil {
//		//			fmt.Println(err)
//		//			for e := 0; e <= 5 && err != nil; e++ {
//		//				time.Sleep(3 * time.Second)
//		//				f, err = b.Get(FileGet)
//		//				if e == 5 {
//		//					panic("File Download Failed")
//		//				}
//		//			}
//		//		}
//
//		fileData := s3Get(j, resp, b, directory)
//
//		fmt.Println("Writing File: ", fileData.FileName)
//		fmt.Println("To Directory: ", fileData.WriteLocation)
//
//		file, err := fs.OpenFile(fileData.WriteLocation, 01|0100, 0, 0, 0)
//		if err != nil {
//			panic(err)
//		}
//		fs.Write(file, fileData.File, fileData.Size)
//		fs.Flush(file)
//		fs.CloseFile(file)
//
//	}
//
//	fs.Disconnect()
//
//	return "done"
//}

func standardWrite(resp s3.ListResp, b s3.Bucket, directory string, startIndex int, hashCheck bool) string {

	fmt.Println("Running files: ", len(resp.Contents)-1)

	for j := startIndex; j <= len(resp.Contents)-1; j++ {

		fileData := s3Get(j, resp, b, directory)

		// Check for Directory
		err := os.MkdirAll(fileData.WriteLocation, 0777)
		if err != nil {
			panic(err)
		}

		fmt.Println("Writing File: ", path.Join(fileData.WriteLocation, fileData.FileName))

		file, _ := os.Create(path.Join(fileData.WriteLocation, fileData.FileName))
		writer := bufio.NewWriter(file)
		writer.Write(fileData.File)
		writer.Flush()
		file.Close()

		if hashCheck == true {

			file, _ := ioutil.ReadFile(fileData.WriteLocation)

			s3Hash := strings.Replace(fileData.MD5Sum, "\"", "", -1)
			hash := md5.New()
			hash.Write(file)
			hashString := hex.EncodeToString(hash.Sum([]byte{}))

			if hashString != s3Hash {
				fmt.Println("File MD5 Hash Unmatched")
				j--
			}
		}

	}

	return "done"
}

//func configureHadoop(javaHome, ldPath string) {
//
//	//Set Java Home
//	os.Setenv("JAVA_HOME", javaHome)
//	os.Setenv("LD_LIBRARY_PATH", ldPath)
//
//	//Build CLASSPATH
//
//	//classpath := "/home/hadoop/conf:" + javaHome + "lib/tools.jar"
//	classpath := "/etc/hadoop/conf:" + javaHome + "lib/tools.jar"
//
//	//Grab JAR Files from Hadoop Root Dir
//	//rootList, err := ioutil.ReadDir("/home/hadoop/")
//	rootList, err := ioutil.ReadDir("/usr/lib/hadoop/")
//	if err != nil {
//		panic(err)
//	}
//
//	for _, fi := range rootList {
//		//classpath = classpath + ":/home/hadoop/" + fi.Name()
//		classpath = classpath + ":/usr/lib/hadoop/" + fi.Name()
//	}
//
//	// Grab JAR Files from Hadoop Lib Dir
//	//libList, err := ioutil.ReadDir("/home/hadoop/lib/")
//	libList, err := ioutil.ReadDir("/usr/lib/hadoop/lib/")
//	if err != nil {
//		panic(err)
//	}
//
//	for _, fi := range libList {
//		//classpath = classpath + ":/home/hadoop/lib/" + fi.Name()
//		classpath = classpath + ":/usr/lib/hadoop/lib/" + fi.Name()
//	}
//
//	os.Setenv("CLASSPATH", classpath)
//
//	// Connect to Hadoop
//
//}

func main() {

	// Define flags
	srcbucket := flag.String("bucket", "bmi-weather-test", "Bucket from which to retrieve files")
	prefix := flag.String("prefix", "TestCSV/", "Prefix from which to retrieve files")
	directory := flag.String("dir", "/Users/nrobison/Developer/git/ParS3/", "Directory to store files")
	maxMarker := flag.Int("max", 1000, "Max Markers to Retrieve per Worker")
	hadoopWrite := flag.Bool("hadoop", false, "Write Files to Hadoop Destination")
	hashCheck := flag.Bool("hash", false, "Check MD5 Hashes of Downloaded Files")
	flag.Parse()

	hadoop := *hadoopWrite
	//
	//	hadoopfill := hdfs.FileInfo {
	//		Name: "test",
	//		}
	//	fmt.Println(hadoopfill.Name)

	// Get User HomeDir
	currentUser, err := user.Current()
	if err != nil {
		panic(err)
	}

	// Read in Config File
	//TODO Add Error Checking for Config File
	configDir := currentUser.HomeDir + "/.ParS3"

	file, err := ioutil.ReadFile(configDir)
	if err != nil {
		panic(err)
	}

	//var cfgFile AWSCredentialsType
	var cfgFile ConfigSettingsType
	json.Unmarshal(file, &cfgFile)

	auth := aws.Auth{
		AccessKey: cfgFile.AWSCredentials.AccessKey,
		SecretKey: cfgFile.AWSCredentials.SecretKey,
	}

//	if hadoop == true {
//		configureHadoop(cfgFile.System.javaHome, cfgFile.System.ldPath)
//	}

	e := s3.New(auth, aws.USEast)

	b := s3.Bucket{
		S3:   e,
		Name: *srcbucket,
	}

	// Check for Directory
	errdir := os.MkdirAll(*directory, 0777)
	if errdir != nil {
		panic(errdir)
	}

	i := s3.Bucket(b)
	//fmt.Println("Number of workers: ", workers)
	nextLoop := continueOn{truncated: true, nextMarker: "", finished: false}
	done := make(chan struct{}, workers)
	leftmost := make(chan continueOn)
	right := leftmost
	left := leftmost

	for nextLoop.truncated == true && nextLoop.finished == false {
		leftmost = make(chan continueOn)
		right = leftmost
		left = leftmost
		for z := 0; z < workers; z++ {
			//		nextLoop = getandsave(i, "Singles/", passit.nextMarker, passit)
			//nextLoop = <-passit
			//		fmt.Println("mainLoop")
			right = make(chan continueOn)
			go getandsave(i, *prefix, left, right, done, z, *directory, *maxMarker, hadoop, *hashCheck)
			//nextLoop = <-right
			left = right
			//		fmt.Println("Running next on: ", nextLoop.nextMarker)
			//fmt.Println(nextLoop.markersReturned)
			//		left = make(chan continueOn)
		}
		go func(c chan continueOn) { c <- nextLoop }(right)
		nextLoop = <-leftmost
		awaitCompletion(done)
	}

}
<commit_msg>Added working S3 upload<commit_after>// HDFS Config Details: https://github.com/zyxar/hdfs

package main

import (
	"bufio"
	"crypto/md5"
	"encoding/hex"
	"encoding/json"
	"flag"
	"fmt"
	//"github.com/zyxar/hdfs"
	"io/ioutil"
	"launchpad.net/goamz/aws"
	"launchpad.net/goamz/s3"
	"os"
	"os/user"
	"path"
	"runtime"
	"strings"
	"time"
)

type continueOn struct {
	nextMarker string
	truncated  bool
	finished   bool
}

type ConfigSettingsType struct {
	AWSCredentials AWSCredentialsType
	System         SystemType
}

type AWSCredentialsType struct {
	AccessKey string
	SecretKey string
}

type SystemType struct {
	javaHome     string
	ldPath       string
	hadoopServer string
	hadoopPort   uint16
}

type fileInfo struct {
	FileName      string
	WriteLocation string
	MD5Sum        string
	File          []byte
	Size          int
}

type Job struct {
	filename string
	results chan<- Result
}

type Result struct {
	filename string
}

// Global Constants

var workers int = runtime.NumCPU()

const hadoopServer string = "default"
const hadoopPort int = 0

func getandsave(b s3.Bucket, prefix string, left, right chan continueOn, done chan<- struct{}, threadNum int, directory string, maxMarker int, hadoop bool, hashCheck bool) {
	grab := <-right

	if grab.finished == true {
		left <- grab
		done <- struct{}{}
		return
	}
	//	if grab.truncated == false {
	//		left <- continueOn{truncated: false, nextMarker: ""}
	//		done <- struct{}{}
	//		return
	//	}

	resp, err := b.List(prefix, "", grab.nextMarker, maxMarker)
	if err != nil {
		fmt.Println(err)
		for e := 0; e <= 5 && err != nil; e++ {
			time.Sleep(3 * time.Second)
			resp, err = b.List(prefix, "", grab.nextMarker, maxMarker)
			if e == 5 {
				panic("Directory List Failed")
			}
		}
	}

	onward := continueOn{resp.Contents[len(resp.Contents)-1].Key, resp.IsTruncated, false}

	switch onward.truncated {
	case true:
		left <- onward
	case false:
		left <- continueOn{nextMarker: "", truncated: false, finished: true}
	}

	//	if len(resp.Contents) == 1 {
	//		fmt.Println("Not Downloading")
	//		done <- struct{}{}
	//		return
	//
	startIndex := 0
	if grab.nextMarker == "" {
		startIndex = 1
	}
	var finished string = ""
	if hadoop == true {
		fmt.Println("Remove me")
		//finished = hadoopWrite(*resp, b, directory, startIndex)
	} else {
		finished = standardWrite(*resp, b, directory, startIndex, hashCheck)
	}

	//	for j := startIndex; j <= len(resp.Contents)-1; j++ {
	//
	//		fileData := s3Get(j, *resp, b, directory)
	//
	//
	//
	//
	//
	//
	//	}

	fmt.Println(finished)

	done <- struct{}{}

}

func awaitCompletion(done <-chan struct{}) {
	for f := 0; f < workers; f++ {
		<-done
	}
	//	close(leftmost)
}

func s3Get(j int, resp s3.ListResp, b s3.Bucket, directory string) fileInfo {

	FileGet := resp.Contents[j].Key

	f, err := b.Get(FileGet)
	if err != nil {
		fmt.Println(err)
		for e := 0; e <= 5 && err != nil; e++ {
			time.Sleep(3 * time.Second)
			f, err = b.Get(FileGet)
			if e == 5 {
				panic("File Download Failed")
			}
		}
	}

	dir, filename := path.Split(FileGet)

	return fileInfo{
		FileName:      filename,
		WriteLocation: path.Join(directory, dir),
		MD5Sum:        resp.Contents[j].ETag,
		File:          f,
		Size:          len(f),
	}

}

//func hadoopWrite(resp s3.ListResp, b s3.Bucket, directory string, startIndex int) string {
//
//	currentUser, err := user.Current()
//	if err != nil {
//		panic(err)
//	}
//
//	fs, err := hdfs.ConnectAsUser(hadoopServer, 0, currentUser.Name)
//	if err != nil {
//		panic(err)
//	}
//
//	fmt.Println("Running files: ", len(resp.Contents)-1)
//
//	for j := startIndex; j <= len(resp.Contents)-1; j++ {
//		//
//		//		FileGet := resp.Contents[j].Key
//		//
//		//		f, err := b.Get(FileGet)
//		//		if err != nil {
//		//			fmt.Println(err)
//		//			for e := 0; e <= 5 && err != nil; e++ {
//		//				time.Sleep(3 * time.Second)
//		//				f, err = b.Get(FileGet)
//		//				if e == 5 {
//		//					panic("File Download Failed")
//		//				}
//		//			}
//		//		}
//
//		fileData := s3Get(j, resp, b, directory)
//
//		fmt.Println("Writing File: ", fileData.FileName)
//		fmt.Println("To Directory: ", fileData.WriteLocation)
//
//		file, err := fs.OpenFile(fileData.WriteLocation, 01|0100, 0, 0, 0)
//		if err != nil {
//			panic(err)
//		}
//		fs.Write(file, fileData.File, fileData.Size)
//		fs.Flush(file)
//		fs.CloseFile(file)
//
//	}
//
//	fs.Disconnect()
//
//	return "done"
//}

func standardWrite(resp s3.ListResp, b s3.Bucket, directory string, startIndex int, hashCheck bool) string {

	fmt.Println("Running files: ", len(resp.Contents)-1)

	for j := startIndex; j <= len(resp.Contents)-1; j++ {

		fileData := s3Get(j, resp, b, directory)

		// Check for Directory
		err := os.MkdirAll(fileData.WriteLocation, 0777)
		if err != nil {
			panic(err)
		}

		fmt.Println("Writing File: ", path.Join(fileData.WriteLocation, fileData.FileName))

		file, _ := os.Create(path.Join(fileData.WriteLocation, fileData.FileName))
		writer := bufio.NewWriter(file)
		writer.Write(fileData.File)
		writer.Flush()
		file.Close()

		if hashCheck == true {

			file, _ := ioutil.ReadFile(fileData.WriteLocation)

			s3Hash := strings.Replace(fileData.MD5Sum, "\"", "", -1)
			hash := md5.New()
			hash.Write(file)
			hashString := hex.EncodeToString(hash.Sum([]byte{}))

			if hashString != s3Hash {
				fmt.Println("File MD5 Hash Unmatched")
				j--
			}
		}

	}

	return "done"
}

//func configureHadoop(javaHome, ldPath string) {
//
//	//Set Java Home
//	os.Setenv("JAVA_HOME", javaHome)
//	os.Setenv("LD_LIBRARY_PATH", ldPath)
//
//	//Build CLASSPATH
//
//	//classpath := "/home/hadoop/conf:" + javaHome + "lib/tools.jar"
//	classpath := "/etc/hadoop/conf:" + javaHome + "lib/tools.jar"
//
//	//Grab JAR Files from Hadoop Root Dir
//	//rootList, err := ioutil.ReadDir("/home/hadoop/")
//	rootList, err := ioutil.ReadDir("/usr/lib/hadoop/")
//	if err != nil {
//		panic(err)
//	}
//
//	for _, fi := range rootList {
//		//classpath = classpath + ":/home/hadoop/" + fi.Name()
//		classpath = classpath + ":/usr/lib/hadoop/" + fi.Name()
//	}
//
//	// Grab JAR Files from Hadoop Lib Dir
//	//libList, err := ioutil.ReadDir("/home/hadoop/lib/")
//	libList, err := ioutil.ReadDir("/usr/lib/hadoop/lib/")
//	if err != nil {
//		panic(err)
//	}
//
//	for _, fi := range libList {
//		//classpath = classpath + ":/home/hadoop/lib/" + fi.Name()
//		classpath = classpath + ":/usr/lib/hadoop/lib/" + fi.Name()
//	}
//
//	os.Setenv("CLASSPATH", classpath)
//
//	// Connect to Hadoop
//
//}

func addJobs(jobs chan<- Job, files []os.FileInfo, results chan<- Result) {
	for _, file := range files {
		jobs <- Job{file.Name(), results}
	}
	close(jobs)
}

func doJobs(done chan<- struct{}, b s3.Bucket, dir string, prefix string, jobs <-chan Job) {
	for job := range jobs {
	s3Put(b, job.filename, dir, prefix)
	}
	done <- struct{}{}

}

func s3Put(b s3.Bucket, filename string, dir string, prefix string) {
	readLocation := path.Join(dir, filename)
	s3Path := path.Join(prefix, filename)
	fmt.Println("Reading ", readLocation)
	file, err := ioutil.ReadFile(readLocation)
	if err != nil {
		panic(err)
	}

	fmt.Println("Putting ", filename)
	b.Put(s3Path, file, "binary/octet-stream", s3.Private)
}

func main() {

	// Define flags
	srcbucket := flag.String("bucket", "bmi-weather-test", "Bucket from which to retrieve files")
	prefix := flag.String("prefix", "Temp/", "Prefix from which to retrieve files")
	directory := flag.String("dir", "/Users/nrobison/Developer/git/ParS3/", "Directory to store files")
	maxMarker := flag.Int("max", 1000, "Max Markers to Retrieve per Worker")
	hadoopWrite := flag.Bool("hadoop", false, "Write Files to Hadoop Destination")
	hashCheck := flag.Bool("hash", false, "Check MD5 Hashes of Downloaded Files")
	gets := flag.Bool("gets", false, "Get Files from S3")
	flag.Parse()

	hadoop := *hadoopWrite
	//
	//	hadoopfill := hdfs.FileInfo {
	//		Name: "test",
	//		}
	//	fmt.Println(hadoopfill.Name)

	// Get User HomeDir
	currentUser, err := user.Current()
	if err != nil {
		panic(err)
	}

	// Read in Config File
	//TODO Add Error Checking for Config File
	configDir := currentUser.HomeDir + "/.ParS3"

	file, err := ioutil.ReadFile(configDir)
	if err != nil {
		panic(err)
	}

	//var cfgFile AWSCredentialsType
	var cfgFile ConfigSettingsType
	json.Unmarshal(file, &cfgFile)

	auth := aws.Auth{
		AccessKey: cfgFile.AWSCredentials.AccessKey,
		SecretKey: cfgFile.AWSCredentials.SecretKey,
	}

//	if hadoop == true {
//		configureHadoop(cfgFile.System.javaHome, cfgFile.System.ldPath)
//	}

	e := s3.New(auth, aws.USEast)

	b := s3.Bucket{
		S3:   e,
		Name: *srcbucket,
	}

	// Check for Directory
	errdir := os.MkdirAll(*directory, 0777)
	if errdir != nil {
		panic(errdir)
	}

	i := s3.Bucket(b)
	//fmt.Println("Number of workers: ", workers)

	if *gets == true {
	nextLoop := continueOn{truncated: true, nextMarker: "", finished: false}
	done := make(chan struct{}, workers)
	leftmost := make(chan continueOn)
	right := leftmost
	left := leftmost

	for nextLoop.truncated == true && nextLoop.finished == false {
		leftmost = make(chan continueOn)
		right = leftmost
		left = leftmost
		for z := 0; z < workers; z++ {
			//		nextLoop = getandsave(i, "Singles/", passit.nextMarker, passit)
			//nextLoop = <-passit
			//		fmt.Println("mainLoop")
			right = make(chan continueOn)
			go getandsave(i, *prefix, left, right, done, z, *directory, *maxMarker, hadoop, *hashCheck)
			//nextLoop = <-right
			left = right
			//		fmt.Println("Running next on: ", nextLoop.nextMarker)
			//fmt.Println(nextLoop.markersReturned)
			//		left = make(chan continueOn)
		}
		go func(c chan continueOn) { c <- nextLoop }(right)
		nextLoop = <-leftmost
		awaitCompletion(done)
	}
	}

	if *gets != true {

		// Get Directory List
		files, _ := ioutil.ReadDir(*directory)
		fmt.Println("Uploading %s files", len(files))

		jobs := make(chan Job, workers)
		results := make(chan Result, len(files))
		done := make(chan struct{}, workers)

		go addJobs(jobs, files, results)
		for j := 0; j < workers; j++ {
			go doJobs(done, i, *directory, *prefix, jobs)
		}
		awaitCompletion(done)
		close(results)

	}

}

<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package flushfs

import (
	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseutil"
)

// Create a file system containing a single file named "foo".
//
// The file may be opened for reading and/or writing. Its initial contents are
// empty. Whenever a flush or fsync is received, the supplied function will be
// called with the current contents of the file.
func NewFileSystem(
	reportFlush func(string),
	reportFsync func(string)) (fs fuse.FileSystem, err error) {
	fs = &flushFS{}
	return
}

type flushFS struct {
	fuseutil.NotImplementedFileSystem
}
<commit_msg>flushFS.Init<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package flushfs

import (
	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseutil"
	"golang.org/x/net/context"
)

// Create a file system containing a single file named "foo".
//
// The file may be opened for reading and/or writing. Its initial contents are
// empty. Whenever a flush or fsync is received, the supplied function will be
// called with the current contents of the file.
func NewFileSystem(
	reportFlush func(string),
	reportFsync func(string)) (fs fuse.FileSystem, err error) {
	fs = &flushFS{}
	return
}

type flushFS struct {
	fuseutil.NotImplementedFileSystem
}

func (fs *flushFS) Init(
	ctx context.Context,
	req *fuse.InitRequest) (
	resp *fuse.InitResponse, err error) {
	resp = &fuse.InitResponse{}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"time"

	"github.com/kid0m4n/go-rpi/sensor/keypad/matrix4x3"
)

func main() {
	rowPins := []int{4, 17, 27, 22}
	colPins := []int{23, 24, 25}

	keypad := matrix4x3.New(rowPins, colPins)

	for {
		if key, err := keypad.PressedKey(); err == nil {
			fmt.Printf("Key Pressed = %v\n", key)
		}

		time.Sleep(500 * time.Millisecond)
	}
}
<commit_msg>Changed keypad sample code to import the package from interface<commit_after>package main

import (
	"fmt"
	"time"

	"github.com/kid0m4n/go-rpi/interface/keypad/matrix4x3"
)

func main() {
	rowPins := []int{4, 17, 27, 22}
	colPins := []int{23, 24, 25}

	keypad := matrix4x3.New(rowPins, colPins)

	for {
		if key, err := keypad.PressedKey(); err == nil {
			fmt.Printf("Key Pressed = %v\n", key)
		}

		time.Sleep(500 * time.Millisecond)
	}
}
<|endoftext|>


<commit_before>package registry

import "gopkg.in/mgo.v2"
import "gopkg.in/mgo.v2/bson"

func NewMongoStore(server string,
                    database string,
                    collection string) (*mgo.Collection, error) {
    session, error := mgo.Dial(server);
    if (error != nil) {
        return nil, error
    }
    defer session.Close();

    connection := session.DB(database).C(collection)
    return connection, nil
}

func (store *mgo.Collection) GetTemplate(name string) (Template, error) {
    result := Template{}
    error := store.Find(bson.M{"name": name}).One(&result)
    if (error != nil) {
       return Template{}, UnavailableTemplateError
    }
    return result, nil
}
<commit_msg>Added MongoStore struct<commit_after>package registry

import "gopkg.in/mgo.v2"
import "gopkg.in/mgo.v2/bson"

type MongoStore struct {
    connection *mgo.Collection
    err error
}

func NewMongoStore(server string,
                    database string,
                    collection string) MongoStore {
    session, error := mgo.Dial(server);

    if (error != nil) {
        return MongoStore{connection: nil, err: error}
    }
    defer session.Close();

    connection := session.DB(database).C(collection)
    return MongoStore{connection: connection, err: nil}
}

func (store MongoStore) GetTemplate(name string) (Template, error) {
    result := Template{}
    error := store.connection.Find(bson.M{"name": name}).One(&result)
    if (error != nil) {
       return Template{}, UnavailableTemplateError
    }
    return result, nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"
)

const (
	Version                = "0.9"
	SprocketsRequireClause = "//= require "
)

//FIXME: add test cases

var (
	outdir         = flag.String("outdir", filepath.Join("public", "assets"), "folder where to put packaged files")
	verbose        = flag.Bool("verbose", false, "turn on verbose logging")
	version        = flag.Bool("version", false, "print version and exit")
	digest         = flag.String("digest", "", "inject digest into output file names")
	generateDigest = flag.Bool("generatedigest", false, "generate digest from the output source")
	assetPath      = flag.String("assetpath", "", "set assetpath if your assets are not on the path specified in manifest")
	jsCompressor   = flag.String("jscompressor", "closure", "javascript compiler: closure or uglifyjs")
	// Platform specific stuff, will be configured in main
	shellForCommands      = ""
	shasumResultSeparator = ""
)

func main() {
	flag.Parse()

	if *version {
		fmt.Printf("%v\n", Version)
		os.Exit(0)
	}

	// Parse manifest files
	manifestFiles := flag.Args()
	if len(manifestFiles) == 0 {
		fmt.Printf("Specify the manifest file(s) to process\n")
		os.Exit(1)
	}
	if *verbose {
		fmt.Printf("Manifest files: %v (%d)\n", manifestFiles, len(manifestFiles))
	}

	if runtime.GOOS == "windows" {
		shellForCommands = "sh"
		shasumResultSeparator = " *"
	} else {
		shellForCommands = "/bin/sh"
		shasumResultSeparator = "  "
	}

	runtime.GOMAXPROCS(runtime.NumCPU())

	// Create out dir (public/assets)
	if err := os.MkdirAll(*outdir, 0777); err != nil {
		panic(err)
	}

	// Validate js compiler
	if *jsCompressor != "closure" && *jsCompressor != "uglifyjs" {
		fmt.Printf("Invalid Javascript compiler: '%s'", jsCompressor)
		os.Exit(1)
	}

	// Create cache dir
	user, err := user.Current()
	if err != nil {
		panic(err)
	}
	appDir := filepath.Join(user.HomeDir, ".wortels")
	cacheDir := filepath.Join(appDir, "cache", *jsCompressor)
	if err := os.MkdirAll(cacheDir, 0777); err != nil {
		panic(err)
	}

	// Read in file list(s)
	files := make(map[string][]string)
	for _, manifest := range manifestFiles {
		b, err := ioutil.ReadFile(filepath.Join(*assetPath, manifest))
		if err != nil {
			panic(err)
		}
		for _, file := range strings.Split(string(b), "\n") {
			file = strings.TrimSpace(file)
			if file == "" {
				continue
			}
			// Convert Sprockets
			if strings.HasPrefix(file, SprocketsRequireClause) {
				file = strings.Replace(file, SprocketsRequireClause, "", 1)
				if !strings.HasSuffix(file, ".js") {
					file = file + ".js"
				}
			}
			// Ignore comments
			if strings.HasPrefix(file, "//") {
				continue
			}
			// Sprockets manifest files have relative paths
			file = filepath.Join(*assetPath, file)
			// Save the file under asset file list
			files[manifest] = append(files[manifest], file)
		}
	}
	if *verbose {
		fmt.Printf("File list: %v (%d)\n", files, len(files))
	}

	// Populate shasums dictionary
	shasums := make(map[string]string)
	dirs := make(map[string]bool)
	for manifest := range files {
		for _, file := range files[manifest] {
			dirs[filepath.Dir(file)] = true
		}
	}
	for dir, _ := range dirs {
		shasum(filepath.Join(dir, "*"), &shasums)
	}
	if *verbose {
		fmt.Printf("SHA database: %v (%d)\n", shasums, len(shasums))
	}

	// Find out which files need to be recompiled
	uniqueCompilationList := make(map[string]bool)
	for manifest := range files {
		for _, file := range files[manifest] {
			sha, knownFile := shasums[file]
			if !knownFile {
				panic(fmt.Sprintf("File not found in SHA1 database: '%v'\n", file))
			}
			cached := filepath.Join(cacheDir, sha)
			if !fileExists(cached) {
				uniqueCompilationList[file] = true
			}
		}
	}
	var compilationList []string
	for file := range uniqueCompilationList {
		compilationList = append(compilationList, file)
	}
	if *verbose {
		fmt.Printf("Files to compile: %v (%d)\n", compilationList, len(compilationList))
	}

	// Compile
	// http://closure-compiler.googlecode.com/files/compiler-latest.zip
	if len(compilationList) > 0 {
		compilationStart := time.Now()
		portableCompilationList := make([]string, len(compilationList))
		for i, path := range compilationList {
			portableCompilationList[i] = filepath.ToSlash(path)
		}
		cmd := jsCompileCommand(portableCompilationList, appDir)
		if *verbose {
			fmt.Printf("%v\n", cmd)
		}
		b, err := exec.Command(shellForCommands, "-c", cmd).CombinedOutput()
		if err != nil {
			fmt.Printf("%v\n", string(b))
			os.Exit(1)
		}

		// Split compiler output into separate cached files
		var currentFile *os.File
		lines := strings.Split(string(b), "\n")
		lastLine := len(lines) - 1
		for i, line := range lines {
			if strings.HasPrefix(line, "// Input ") {
				if currentFile != nil {
					currentFile.Close()
				}
				i, err := strconv.Atoi(strings.Split(line, "// Input ")[1])
				if err != nil {
					panic(err)
				}
				file := compilationList[i]
				sha := shasums[file]
				cached := filepath.Join(cacheDir, sha)
				if *verbose {
					fmt.Printf("%v (%s)\n", file, sha)
				}
				currentFile, err = os.Create(cached)
				if err != nil {
					panic(err)
				}
				continue
			}
			if currentFile == nil {
				panic(fmt.Sprintf("No file to write to! Line: %s", line))
			}
			currentFile.Write([]byte(line))
			if i != lastLine {
				currentFile.Write([]byte("\n"))
			}
		}
		if currentFile != nil {
			currentFile.Close()
		}
		if *verbose {
			fmt.Printf("JS compiled in %v\n", time.Since(compilationStart))
		}
	}

	// Assemble asset file from compiled files
	var outputFiles []string
	for manifest := range files {
		catList := make([]string, len(files[manifest]))
		for i, file := range files[manifest] {
			catFile := filepath.Join(cacheDir, shasums[file])
			portableCatFile := filepath.ToSlash(catFile)
			catList[i] = portableCatFile
		}
		inputFiles := strings.Join(catList, " ")
		outputFile := filepath.Join(*outdir, filepath.Base(manifest))
		if *digest != "" {
			outputFile = injectDigest(outputFile, *digest)
		}
		portableOutputFile := filepath.ToSlash(outputFile)
		cmd := fmt.Sprintf("cat %s > %s", inputFiles, portableOutputFile)
		if *verbose {
			fmt.Printf("%v\n", cmd)
		}
		_, err = exec.Command(shellForCommands, "-c", cmd).CombinedOutput()
		if err != nil {
			panic(err)
		}
		outputFiles = append(outputFiles, outputFile)
	}
	if *verbose {
		fmt.Printf("Output files: %v\n", outputFiles)
	}

	// Inject digests into output filenames, if no digest is given
	if *generateDigest {
		// Shasum the output files
		outputDigests := make(map[string]string)
		shasum(strings.Join(outputFiles, " "), &outputDigests)

		// Rename files to include the shasum
		for outputFile, sha1 := range outputDigests {
			renamedFile := injectDigest(outputFile, sha1)
			cmd := fmt.Sprintf("mv %s %s", outputFile, renamedFile)
			if *verbose {
				fmt.Printf("%v\n", cmd)
			}
			_, err = exec.Command(shellForCommands, "-c", cmd).CombinedOutput()
			if err != nil {
				panic(err)
			}
		}
	}
}

func jsCompileCommand(portableCompilationList []string, appDir string) string {
	if *jsCompressor == "closure" {
		portableCompilerPath := filepath.ToSlash(filepath.Join(appDir, "compiler-latest", "compiler.jar"))
		return fmt.Sprintf("java -jar %s --warning_level QUIET --compilation_level SIMPLE_OPTIMIZATIONS --formatting print_input_delimiter --js %s",
			portableCompilerPath,
			strings.Join(portableCompilationList, " "))
	} else if *jsCompressor == "uglifyjs" {
		return fmt.Sprintf("uglifyjs %s", strings.Join(portableCompilationList, " "))
	}
	return ""
}

func injectDigest(outputFile, digest string) string {
	currentExt := filepath.Ext(outputFile)
	newExt := "-" + digest + currentExt
	return strings.Replace(outputFile, currentExt, newExt, -1)
}

func shasum(path string, shasums *map[string]string) {
	portablePath := filepath.ToSlash(path)
	cmd := fmt.Sprintf("shasum %s", portablePath)
	if *verbose {
		fmt.Printf("%v\n", cmd)
	}
	b, err := exec.Command(shellForCommands, "-c", cmd).CombinedOutput()
	if err != nil {
		// Cygwin shasum seems to exit status 1 even if there's no error?
		if !((err.Error() == "exit status 1") && (runtime.GOOS == "windows")) {
			panic(err)
		}
	}
	for _, shasumResult := range strings.Split(string(b), "\n") {
		if shasumResult != "" {
			// Ignore shasum messages a la
			// shasum: foo/bar: Is a directory
			if !strings.HasPrefix(shasumResult, "shasum: ") && !strings.HasSuffix(shasumResult, "Is a directory") {
				fields := strings.Split(shasumResult, shasumResultSeparator)

				if len(fields) < 2 {
					panic(fmt.Sprintf("Unexpected shasum result with separator '%s': '%v'\n", shasumResultSeparator, shasumResult))
				}
				sha := fields[0]
				file := filepath.FromSlash(fields[1])
				(*shasums)[file] = sha
			}
		}
	}
}

func fileExists(path string) bool {
	if _, err := os.Stat(path); err != nil {
		if os.IsNotExist(err) {
			return false
		} else {
			panic(err)
		}
	}
	return true
}
<commit_msg>Newer shasum versions return exit code 1 even if there's no error<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"
)

const (
	Version                = "0.9"
	SprocketsRequireClause = "//= require "
)

//FIXME: add test cases

var (
	outdir         = flag.String("outdir", filepath.Join("public", "assets"), "folder where to put packaged files")
	verbose        = flag.Bool("verbose", false, "turn on verbose logging")
	version        = flag.Bool("version", false, "print version and exit")
	digest         = flag.String("digest", "", "inject digest into output file names")
	generateDigest = flag.Bool("generatedigest", false, "generate digest from the output source")
	assetPath      = flag.String("assetpath", "", "set assetpath if your assets are not on the path specified in manifest")
	jsCompressor   = flag.String("jscompressor", "closure", "javascript compiler: closure or uglifyjs")
	// Platform specific stuff, will be configured in main
	shellForCommands      = ""
	shasumResultSeparator = ""
)

func main() {
	flag.Parse()

	if *version {
		fmt.Printf("%v\n", Version)
		os.Exit(0)
	}

	// Parse manifest files
	manifestFiles := flag.Args()
	if len(manifestFiles) == 0 {
		fmt.Printf("Specify the manifest file(s) to process\n")
		os.Exit(1)
	}
	if *verbose {
		fmt.Printf("Manifest files: %v (%d)\n", manifestFiles, len(manifestFiles))
	}

	if runtime.GOOS == "windows" {
		shellForCommands = "sh"
		shasumResultSeparator = " *"
	} else {
		shellForCommands = "/bin/sh"
		shasumResultSeparator = "  "
	}

	runtime.GOMAXPROCS(runtime.NumCPU())

	// Create out dir (public/assets)
	if err := os.MkdirAll(*outdir, 0777); err != nil {
		panic(err)
	}

	// Validate js compiler
	if *jsCompressor != "closure" && *jsCompressor != "uglifyjs" {
		fmt.Printf("Invalid Javascript compiler: '%s'", jsCompressor)
		os.Exit(1)
	}

	// Create cache dir
	user, err := user.Current()
	if err != nil {
		panic(err)
	}
	appDir := filepath.Join(user.HomeDir, ".wortels")
	cacheDir := filepath.Join(appDir, "cache", *jsCompressor)
	if err := os.MkdirAll(cacheDir, 0777); err != nil {
		panic(err)
	}

	// Read in file list(s)
	files := make(map[string][]string)
	for _, manifest := range manifestFiles {
		b, err := ioutil.ReadFile(filepath.Join(*assetPath, manifest))
		if err != nil {
			panic(err)
		}
		for _, file := range strings.Split(string(b), "\n") {
			file = strings.TrimSpace(file)
			if file == "" {
				continue
			}
			// Convert Sprockets
			if strings.HasPrefix(file, SprocketsRequireClause) {
				file = strings.Replace(file, SprocketsRequireClause, "", 1)
				if !strings.HasSuffix(file, ".js") {
					file = file + ".js"
				}
			}
			// Ignore comments
			if strings.HasPrefix(file, "//") {
				continue
			}
			// Sprockets manifest files have relative paths
			file = filepath.Join(*assetPath, file)
			// Save the file under asset file list
			files[manifest] = append(files[manifest], file)
		}
	}
	if *verbose {
		fmt.Printf("File list: %v (%d)\n", files, len(files))
	}

	// Populate shasums dictionary
	shasums := make(map[string]string)
	dirs := make(map[string]bool)
	for manifest := range files {
		for _, file := range files[manifest] {
			dirs[filepath.Dir(file)] = true
		}
	}
	for dir, _ := range dirs {
		shasum(filepath.Join(dir, "*"), &shasums)
	}
	if *verbose {
		fmt.Printf("SHA database: %v (%d)\n", shasums, len(shasums))
	}

	// Find out which files need to be recompiled
	uniqueCompilationList := make(map[string]bool)
	for manifest := range files {
		for _, file := range files[manifest] {
			sha, knownFile := shasums[file]
			if !knownFile {
				panic(fmt.Sprintf("File not found in SHA1 database: '%v'\n", file))
			}
			cached := filepath.Join(cacheDir, sha)
			if !fileExists(cached) {
				uniqueCompilationList[file] = true
			}
		}
	}
	var compilationList []string
	for file := range uniqueCompilationList {
		compilationList = append(compilationList, file)
	}
	if *verbose {
		fmt.Printf("Files to compile: %v (%d)\n", compilationList, len(compilationList))
	}

	// Compile
	// http://closure-compiler.googlecode.com/files/compiler-latest.zip
	if len(compilationList) > 0 {
		compilationStart := time.Now()
		portableCompilationList := make([]string, len(compilationList))
		for i, path := range compilationList {
			portableCompilationList[i] = filepath.ToSlash(path)
		}
		cmd := jsCompileCommand(portableCompilationList, appDir)
		if *verbose {
			fmt.Printf("%v\n", cmd)
		}
		b, err := exec.Command(shellForCommands, "-c", cmd).CombinedOutput()
		if err != nil {
			fmt.Printf("%v\n", string(b))
			os.Exit(1)
		}

		// Split compiler output into separate cached files
		var currentFile *os.File
		lines := strings.Split(string(b), "\n")
		lastLine := len(lines) - 1
		for i, line := range lines {
			if strings.HasPrefix(line, "// Input ") {
				if currentFile != nil {
					currentFile.Close()
				}
				i, err := strconv.Atoi(strings.Split(line, "// Input ")[1])
				if err != nil {
					panic(err)
				}
				file := compilationList[i]
				sha := shasums[file]
				cached := filepath.Join(cacheDir, sha)
				if *verbose {
					fmt.Printf("%v (%s)\n", file, sha)
				}
				currentFile, err = os.Create(cached)
				if err != nil {
					panic(err)
				}
				continue
			}
			if currentFile == nil {
				panic(fmt.Sprintf("No file to write to! Line: %s", line))
			}
			currentFile.Write([]byte(line))
			if i != lastLine {
				currentFile.Write([]byte("\n"))
			}
		}
		if currentFile != nil {
			currentFile.Close()
		}
		if *verbose {
			fmt.Printf("JS compiled in %v\n", time.Since(compilationStart))
		}
	}

	// Assemble asset file from compiled files
	var outputFiles []string
	for manifest := range files {
		catList := make([]string, len(files[manifest]))
		for i, file := range files[manifest] {
			catFile := filepath.Join(cacheDir, shasums[file])
			portableCatFile := filepath.ToSlash(catFile)
			catList[i] = portableCatFile
		}
		inputFiles := strings.Join(catList, " ")
		outputFile := filepath.Join(*outdir, filepath.Base(manifest))
		if *digest != "" {
			outputFile = injectDigest(outputFile, *digest)
		}
		portableOutputFile := filepath.ToSlash(outputFile)
		cmd := fmt.Sprintf("cat %s > %s", inputFiles, portableOutputFile)
		if *verbose {
			fmt.Printf("%v\n", cmd)
		}
		_, err = exec.Command(shellForCommands, "-c", cmd).CombinedOutput()
		if err != nil {
			panic(err)
		}
		outputFiles = append(outputFiles, outputFile)
	}
	if *verbose {
		fmt.Printf("Output files: %v\n", outputFiles)
	}

	// Inject digests into output filenames, if no digest is given
	if *generateDigest {
		// Shasum the output files
		outputDigests := make(map[string]string)
		shasum(strings.Join(outputFiles, " "), &outputDigests)

		// Rename files to include the shasum
		for outputFile, sha1 := range outputDigests {
			renamedFile := injectDigest(outputFile, sha1)
			cmd := fmt.Sprintf("mv %s %s", outputFile, renamedFile)
			if *verbose {
				fmt.Printf("%v\n", cmd)
			}
			_, err = exec.Command(shellForCommands, "-c", cmd).CombinedOutput()
			if err != nil {
				panic(err)
			}
		}
	}
}

func jsCompileCommand(portableCompilationList []string, appDir string) string {
	if *jsCompressor == "closure" {
		portableCompilerPath := filepath.ToSlash(filepath.Join(appDir, "compiler-latest", "compiler.jar"))
		return fmt.Sprintf("java -jar %s --warning_level QUIET --compilation_level SIMPLE_OPTIMIZATIONS --formatting print_input_delimiter --js %s",
			portableCompilerPath,
			strings.Join(portableCompilationList, " "))
	} else if *jsCompressor == "uglifyjs" {
		return fmt.Sprintf("uglifyjs %s", strings.Join(portableCompilationList, " "))
	}
	return ""
}

func injectDigest(outputFile, digest string) string {
	currentExt := filepath.Ext(outputFile)
	newExt := "-" + digest + currentExt
	return strings.Replace(outputFile, currentExt, newExt, -1)
}

func shasum(path string, shasums *map[string]string) {
	portablePath := filepath.ToSlash(path)
	cmd := fmt.Sprintf("shasum %s", portablePath)
	if *verbose {
		fmt.Printf("%v\n", cmd)
	}
	b, err := exec.Command(shellForCommands, "-c", cmd).CombinedOutput()
	if err != nil {
		// newer shasum seems to exit status 1 even if there's no error?
		if err.Error() != "exit status 1" {
			panic(err)
		}
	}
	for _, shasumResult := range strings.Split(string(b), "\n") {
		if shasumResult != "" {
			// Ignore shasum messages a la
			// shasum: foo/bar: Is a directory
			if !strings.HasPrefix(shasumResult, "shasum: ") && !strings.HasSuffix(shasumResult, "Is a directory") {
				fields := strings.Split(shasumResult, shasumResultSeparator)

				if len(fields) < 2 {
					panic(fmt.Sprintf("Unexpected shasum result with separator '%s': '%v'\n", shasumResultSeparator, shasumResult))
				}
				sha := fields[0]
				file := filepath.FromSlash(fields[1])
				(*shasums)[file] = sha
			}
		}
	}
}

func fileExists(path string) bool {
	if _, err := os.Stat(path); err != nil {
		if os.IsNotExist(err) {
			return false
		} else {
			panic(err)
		}
	}
	return true
}
<|endoftext|>