<commit_before>package stns

import (
	"net/http"
	"reflect"
	"regexp"

	"github.com/STNS/STNS/settings"
	"github.com/ant0ine/go-json-rest/rest"
)

type Handler struct {
	config *Config
}

type MetaData struct {
	ApiVersion float64 `json:"api_version"`
	Salt       bool    `json:"salt_enable"`
	Stretching int     `json:"stretching_count"`
	Result     string  `json:"result""`
}

type ResponseFormat struct {
	MetaData *MetaData   `json:"metadata"`
	Items    *Attributes `json:"items"`
}

func (h *Handler) getQuery(r *rest.Request) *Query {
	value := r.PathParam("value")
	column := r.PathParam("column")
	resource_name := r.PathParam("resource_name")
	return &Query{h.config, resource_name, column, value}
}

func (h *Handler) getListQuery(r *rest.Request) *Query {
	resource_name := r.PathParam("resource_name")
	return &Query{h.config, resource_name, "list", ""}
}

func (h *Handler) Get(w rest.ResponseWriter, r *rest.Request) {
	query := h.getQuery(r)
	h.Response(query, w, r)
}

func (h *Handler) Auth(w rest.ResponseWriter, r *rest.Request) {
	query := h.getQuery(r)
	h.AuthResponse(query, w, r)
}

func (h *Handler) GetList(w rest.ResponseWriter, r *rest.Request) {
	query := h.getListQuery(r)
	h.Response(query, w, r)
}

func (h *Handler) Response(q *Query, w rest.ResponseWriter, r *rest.Request) {
	attr := q.Get()
	if attr == nil || reflect.ValueOf(attr).IsNil() {
		rest.NotFound(w, r)
		return
	}

	v2 := regexp.MustCompile(`^/v2`)
	if v2.MatchString(r.URL.Path) {
		response := ResponseFormat{
			MetaData: &MetaData{
				ApiVersion: settings.API_VERSION,
				Salt:       h.config.Salt,
				Stretching: h.config.Stretching,
				Result:     settings.SUCCESS,
			},
			Items: &attr,
		}
		w.WriteJson(response)
		return
	} else {
		w.WriteJson(attr)
		return
	}
	rest.NotFound(w, r)
}

func (h *Handler) AuthResponse(q *Query, w rest.ResponseWriter, r *rest.Request) {
	attr := q.Get()
	if attr == nil || reflect.ValueOf(attr).IsNil() {
		rest.NotFound(w, r)
		return
	}

	for _, params := range attr {
		if params.Password == r.PathParam("hash") {
			response := ResponseFormat{
				MetaData: &MetaData{
					ApiVersion: settings.API_VERSION,
					Salt:       h.config.Salt,
					Stretching: h.config.Stretching,
					Result:     settings.SUCCESS,
				},
				Items: &Attributes{},
			}
			w.WriteJson(response)
			return
		} else {
			rest.Error(w, settings.AUTH_ERROR, http.StatusUnauthorized)
			return
		}
	}
	rest.NotFound(w, r)
}
<commit_msg>char matching before lower<commit_after>package stns

import (
	"net/http"
	"reflect"
	"regexp"
	"strings"

	"github.com/STNS/STNS/settings"
	"github.com/ant0ine/go-json-rest/rest"
)

type Handler struct {
	config *Config
}

type MetaData struct {
	ApiVersion float64 `json:"api_version"`
	Salt       bool    `json:"salt_enable"`
	Stretching int     `json:"stretching_count"`
	Result     string  `json:"result""`
}

type ResponseFormat struct {
	MetaData *MetaData   `json:"metadata"`
	Items    *Attributes `json:"items"`
}

func (h *Handler) getQuery(r *rest.Request) *Query {
	value := r.PathParam("value")
	column := r.PathParam("column")
	resource_name := r.PathParam("resource_name")
	return &Query{h.config, resource_name, column, value}
}

func (h *Handler) getListQuery(r *rest.Request) *Query {
	resource_name := r.PathParam("resource_name")
	return &Query{h.config, resource_name, "list", ""}
}

func (h *Handler) Get(w rest.ResponseWriter, r *rest.Request) {
	query := h.getQuery(r)
	h.Response(query, w, r)
}

func (h *Handler) Auth(w rest.ResponseWriter, r *rest.Request) {
	query := h.getQuery(r)
	h.AuthResponse(query, w, r)
}

func (h *Handler) GetList(w rest.ResponseWriter, r *rest.Request) {
	query := h.getListQuery(r)
	h.Response(query, w, r)
}

func (h *Handler) Response(q *Query, w rest.ResponseWriter, r *rest.Request) {
	attr := q.Get()
	if attr == nil || reflect.ValueOf(attr).IsNil() {
		rest.NotFound(w, r)
		return
	}

	v2 := regexp.MustCompile(`^/v2`)
	if v2.MatchString(r.URL.Path) {
		response := ResponseFormat{
			MetaData: &MetaData{
				ApiVersion: settings.API_VERSION,
				Salt:       h.config.Salt,
				Stretching: h.config.Stretching,
				Result:     settings.SUCCESS,
			},
			Items: &attr,
		}
		w.WriteJson(response)
		return
	} else {
		w.WriteJson(attr)
		return
	}
	rest.NotFound(w, r)
}

func (h *Handler) AuthResponse(q *Query, w rest.ResponseWriter, r *rest.Request) {
	attr := q.Get()
	if attr == nil || reflect.ValueOf(attr).IsNil() {
		rest.NotFound(w, r)
		return
	}

	for _, params := range attr {
		if strings.ToLower(params.Password) == r.PathParams("hash") {
			response := ResponseFormat{
				MetaData: &MetaData{
					ApiVersion: settings.API_VERSION,
					Salt:       h.config.Salt,
					Stretching: h.config.Stretching,
					Result:     settings.SUCCESS,
				},
				Items: &Attributes{},
			}
			w.WriteJson(response)
			return
		} else {
			rest.Error(w, settings.AUTH_ERROR, http.StatusUnauthorized)
			return
		}
	}
	rest.NotFound(w, r)
}
<|endoftext|>


<commit_before>package hsts

import (
	"reflect"
	"testing"
	"time"
)

func TestMemStorage_Contains(t *testing.T) {
	m := NewMemStorage()
	m.Add(&Domain{
		Host:       "example.org",
		Subdomains: false,
		Permanent:  false,
		Created:    time.Now().Unix(),
		MaxAge:     100,
	})
	m.Add(&Domain{
		Host:       "a.example.org",
		Subdomains: true,
		Permanent:  false,
		Created:    time.Now().Unix(),
		MaxAge:     100,
	})

	m.Add(&Domain{
		Host:       "a.example.com",
		Subdomains: true,
		Permanent:  false,
		Created:    time.Now().Unix(),
		MaxAge:     100,
	})
	m.Add(&Domain{
		Host:       "b.example.com",
		Subdomains: false,
		Permanent:  false,
		Created:    time.Now().Unix(),
		MaxAge:     100,
	})

	done := make(chan bool)
	go func() {
		orgTest(m, t)
		// try to make a data race
		m.Add(&Domain{
			Host:       "example.org",
			Subdomains: false,
			Permanent:  false,
			Created:    time.Now().Unix(),
			MaxAge:     100,
		})
		done <- true
	}()

	go func() {
		comTest(m, t)
		// try to make a data race
		m.Add(&Domain{
			Host:       "a.example.com",
			Subdomains: true,
			Permanent:  false,
			Created:    time.Now().Unix(),
			MaxAge:     100,
		})
		done <- true
	}()

	// wait for tests to finish
	<-done
	<-done
}

func orgTest(m *MemStorage, t *testing.T) {
	tests := []struct {
		name     string
		host     string
		expected bool
	}{
		{
			name:     "root match org",
			host:     "example.org",
			expected: true,
		},
		{
			name:     "subdomain match org",
			host:     "a.example.org",
			expected: true,
		},
		{
			name:     "subdomain miss-match org",
			host:     "b.example.org",
			expected: false,
		},
	}

	for _, test := range tests {
		out := m.Contains(test.host)
		if out != test.expected {
			t.Logf("host: %s", test.host)
			t.Logf("want:%v", test.expected)
			t.Logf("got:%v", out)
			t.Fatalf("test case failed: %s", test.name)
		}
	}
}

func comTest(m *MemStorage, t *testing.T) {
	tests := []struct {
		name     string
		host     string
		expected bool
	}{
		{
			name:     "subdomain enabled",
			host:     "z.a.example.com",
			expected: true,
		},
		{
			name:     "sub-subdomain",
			host:     "z.y.a.example.com",
			expected: true,
		},
		{
			name:     "subdomain disabled",
			host:     "z.b.example.com",
			expected: false,
		},
		{
			name:     "exact match",
			host:     "b.example.com",
			expected: true,
		},
		{
			name:     "complete missmatch",
			host:     "z.example.com",
			expected: false,
		},
	}

	for _, test := range tests {
		out := m.Contains(test.host)
		if out != test.expected {
			t.Logf("host: %s", test.host)
			t.Logf("want:%v", test.expected)
			t.Logf("got:%v", out)
			t.Fatalf("test case failed: %s", test.name)
		}
	}
}

func TestMemStorage_Add(t *testing.T) {
	m := &MemStorage{}

	permanentDomain := Domain{
		Host:       "example.org",
		Subdomains: false,
		Permanent:  true,
		Created:    time.Now().Unix(),
		MaxAge:     0,
	}

	normalDomain := Domain{
		Host:       "a.example.org",
		Subdomains: false,
		Permanent:  false,
		Created:    time.Now().Unix(),
		MaxAge:     100,
	}

	removeNormalDomain := Domain{
		Host:       "a.example.org",
		Subdomains: false,
		Permanent:  false,
		Created:    time.Now().Unix(),
		MaxAge:     0,
	}

	removePermanetDomain := Domain{
		Host:       "example.org",
		Subdomains: false,
		Permanent:  false,
		Created:    time.Now().Unix(),
		MaxAge:     0,
	}

	// permanent
	m.Add(&permanentDomain)

	expected := map[string]Domain{
		"example.org": permanentDomain,
	}

	if !reflect.DeepEqual(m.domains, expected) {
		t.Logf("want:%v", expected)
		t.Logf("got:%v", m.domains)
		t.Fatal("Add failed after permanent")
	}

	// normal
	m.Add(&normalDomain)

	expected = map[string]Domain{
		"example.org":   permanentDomain,
		"a.example.org": normalDomain,
	}

	if !reflect.DeepEqual(m.domains, expected) {
		t.Logf("want:%v", expected)
		t.Logf("got:%v", m.domains)
		t.Fatal("Add failed after adding normal")
	}

	// remove normal
	m.Add(&removeNormalDomain)

	expected = map[string]Domain{
		"example.org": permanentDomain,
	}

	if !reflect.DeepEqual(m.domains, expected) {
		t.Logf("want:%v", expected)
		t.Logf("got:%v", m.domains)
		t.Fatal("Add failed after removing normal")
	}

	// attempt to remove the permanent
	m.Add(&removePermanetDomain)

	expected = map[string]Domain{
		"example.org": permanentDomain,
	}

	if !reflect.DeepEqual(m.domains, expected) {
		t.Logf("want:%v", expected)
		t.Logf("got:%v", m.domains)
		t.Fatal("Add failed after attempting to remove the permanent domain")
	}
}
<commit_msg>storage_test: fix issue #2, t.Fatalf from main thread<commit_after>package hsts

import (
	"fmt"
	"reflect"
	"testing"
	"time"
)

func TestMemStorage_Contains(t *testing.T) {
	m := NewMemStorage()
	m.Add(&Domain{
		Host:       "example.org",
		Subdomains: false,
		Permanent:  false,
		Created:    time.Now().Unix(),
		MaxAge:     100,
	})
	m.Add(&Domain{
		Host:       "a.example.org",
		Subdomains: true,
		Permanent:  false,
		Created:    time.Now().Unix(),
		MaxAge:     100,
	})

	m.Add(&Domain{
		Host:       "a.example.com",
		Subdomains: true,
		Permanent:  false,
		Created:    time.Now().Unix(),
		MaxAge:     100,
	})
	m.Add(&Domain{
		Host:       "b.example.com",
		Subdomains: false,
		Permanent:  false,
		Created:    time.Now().Unix(),
		MaxAge:     100,
	})

	done := make(chan bool)

	var orgErr error
	go func() {
		orgErr = orgTest(m, t)
		// try to make a data race
		m.Add(&Domain{
			Host:       "example.org",
			Subdomains: false,
			Permanent:  false,
			Created:    time.Now().Unix(),
			MaxAge:     100,
		})
		done <- true
	}()

	var comErr error
	go func() {
		comErr = comTest(m, t)
		// try to make a data race
		m.Add(&Domain{
			Host:       "a.example.com",
			Subdomains: true,
			Permanent:  false,
			Created:    time.Now().Unix(),
			MaxAge:     100,
		})
		done <- true
	}()

	// wait for tests to finish
	<-done
	<-done

	if orgErr != nil {
		t.Fatal(orgErr)
	}

	if comErr != nil {
		t.Fatal(comErr)
	}
}

func orgTest(m *MemStorage, t *testing.T) error {
	tests := []struct {
		name     string
		host     string
		expected bool
	}{
		{
			name:     "root match org",
			host:     "example.org",
			expected: true,
		},
		{
			name:     "subdomain match org",
			host:     "a.example.org",
			expected: true,
		},
		{
			name:     "subdomain miss-match org",
			host:     "b.example.org",
			expected: false,
		},
	}

	for _, test := range tests {
		out := m.Contains(test.host)
		if out != test.expected {
			t.Logf("host: %s", test.host)
			t.Logf("want:%v", test.expected)
			t.Logf("got:%v", out)
			return fmt.Errorf("test case failed: %s", test.name)
		}
	}

	return nil
}

func comTest(m *MemStorage, t *testing.T) error {
	tests := []struct {
		name     string
		host     string
		expected bool
	}{
		{
			name:     "subdomain enabled",
			host:     "z.a.example.com",
			expected: true,
		},
		{
			name:     "sub-subdomain",
			host:     "z.y.a.example.com",
			expected: true,
		},
		{
			name:     "subdomain disabled",
			host:     "z.b.example.com",
			expected: false,
		},
		{
			name:     "exact match",
			host:     "b.example.com",
			expected: true,
		},
		{
			name:     "complete missmatch",
			host:     "z.example.com",
			expected: false,
		},
	}

	for _, test := range tests {
		out := m.Contains(test.host)
		if out != test.expected {
			t.Logf("host: %s", test.host)
			t.Logf("want:%v", test.expected)
			t.Logf("got:%v", out)
			return fmt.Errorf("test case failed: %s", test.name)
		}
	}

	return nil
}

func TestMemStorage_Add(t *testing.T) {
	m := &MemStorage{}

	permanentDomain := Domain{
		Host:       "example.org",
		Subdomains: false,
		Permanent:  true,
		Created:    time.Now().Unix(),
		MaxAge:     0,
	}

	normalDomain := Domain{
		Host:       "a.example.org",
		Subdomains: false,
		Permanent:  false,
		Created:    time.Now().Unix(),
		MaxAge:     100,
	}

	removeNormalDomain := Domain{
		Host:       "a.example.org",
		Subdomains: false,
		Permanent:  false,
		Created:    time.Now().Unix(),
		MaxAge:     0,
	}

	removePermanetDomain := Domain{
		Host:       "example.org",
		Subdomains: false,
		Permanent:  false,
		Created:    time.Now().Unix(),
		MaxAge:     0,
	}

	// permanent
	m.Add(&permanentDomain)

	expected := map[string]Domain{
		"example.org": permanentDomain,
	}

	if !reflect.DeepEqual(m.domains, expected) {
		t.Logf("want:%v", expected)
		t.Logf("got:%v", m.domains)
		t.Fatal("Add failed after permanent")
	}

	// normal
	m.Add(&normalDomain)

	expected = map[string]Domain{
		"example.org":   permanentDomain,
		"a.example.org": normalDomain,
	}

	if !reflect.DeepEqual(m.domains, expected) {
		t.Logf("want:%v", expected)
		t.Logf("got:%v", m.domains)
		t.Fatal("Add failed after adding normal")
	}

	// remove normal
	m.Add(&removeNormalDomain)

	expected = map[string]Domain{
		"example.org": permanentDomain,
	}

	if !reflect.DeepEqual(m.domains, expected) {
		t.Logf("want:%v", expected)
		t.Logf("got:%v", m.domains)
		t.Fatal("Add failed after removing normal")
	}

	// attempt to remove the permanent
	m.Add(&removePermanetDomain)

	expected = map[string]Domain{
		"example.org": permanentDomain,
	}

	if !reflect.DeepEqual(m.domains, expected) {
		t.Logf("want:%v", expected)
		t.Logf("got:%v", m.domains)
		t.Fatal("Add failed after attempting to remove the permanent domain")
	}
}
<|endoftext|>


<commit_before>package stream

import (
	"github.com/cevian/go-stream/util/slog"
)

type Chain interface {
	Operators() []Operator
	Run() error
	Stop() error
	Add(o Operator) Chain
	SetName(string) Chain
	//NewSubChain creates a new empty chain inheriting the properties of the parent chain
	//Usefull for distribute/fanout building functions
	NewSubChain() Chain
	//async functions
	Start() error
	Wait() error
}

/* A SimpleChain implements the operator interface too! */
type SimpleChain struct {
	runner *Runner
	//	Ops         []Operator
	//	wg          *sync.WaitGroup
	//	closenotify chan bool
	//	closeerror  chan error
	sentstop bool
	Name     string
}

func NewChain() *SimpleChain {
	return NewSimpleChain()
}

func NewSimpleChain() *SimpleChain {
	return &SimpleChain{runner: NewRunner()}
}

func (c *SimpleChain) Operators() []Operator {
	return c.runner.Operators()
}

func (c *SimpleChain) SetName(name string) Chain {
	c.Name = name
	return c
}

func (c *SimpleChain) NewSubChain() Chain {
	return NewSimpleChain()
}

func (c *SimpleChain) Add(o Operator) Chain {
	ops := c.runner.Operators()
	opIn, isIn := o.(In)
	if isIn && len(ops) > 0 {
		slog.Infof("Setting input channel of %s", Name(o))
		last := ops[len(ops)-1]

		lastOutCh := last.(Out).Out()

		opIn.SetIn(lastOutCh)
	}

	out, ok := o.(Out)
	if ok {
		slog.Infof("Setting output channel of %s", Name(o))
		ch := make(chan Object, CHAN_SLACK)
		out.SetOut(ch)
	}

	c.runner.Add(o)
	return c
}

func (c *SimpleChain) Start() error {
	c.runner.AsyncRunAll()
	return nil
}

func (c *SimpleChain) SoftStop() error {
	if !c.sentstop {
		c.sentstop = true
		slog.Warnf("In soft close")
		ops := c.runner.Operators()
		ops[0].Stop()
	}
	return nil
}

/* A stop is a hard stop as per the Operator interface */
func (c *SimpleChain) Stop() error {
	if !c.sentstop {
		c.sentstop = true
		slog.Warnf("In hard close")
		c.runner.HardStop()
	}
	return nil
}

func (c *SimpleChain) Wait() error {
	//slog.Logf(logger.Levels.Info, "Waiting for closenotify %s", c.Name)
	//<-c.runner.CloseNotifier()
	err, errOk := <-c.runner.ErrorChannel()
	if errOk {
		slog.Warnf("Hard Close in SimpleChain %s %v", c.Name, err)
		c.Stop()
	} else {
		slog.Infof("Soft Close in SimpleChain %s", c.Name)
		c.SoftStop()
	}
	slog.Infof("Waiting for runner to finish")
	c.runner.Wait()
	slog.Infof("Exiting SimpleChain")

	return nil
}

/* Operator compatibility */
func (c *SimpleChain) Run() error {
	if err := c.Start(); err != nil {
		return err
	}
	return c.Wait()
}

type OrderedChain struct {
	*SimpleChain
}

func NewOrderedChain() *OrderedChain {
	return &OrderedChain{NewChain()}
}

func (c *OrderedChain) Add(o Operator) Chain {
	parallel, ok := o.(ParallelizableOperator)
	if ok {
		if !parallel.IsOrdered() {
			parallel = parallel.MakeOrdered()
			if !parallel.IsOrdered() {
				slog.Fatalf("%s", "Couldn't make parallel operator ordered")
			}
		}
		c.SimpleChain.Add(parallel)
	} else {
		c.SimpleChain.Add(o)
	}
	return c
}

func (c *OrderedChain) NewSubChain() Chain {
	return NewOrderedChain()
}

type InChain interface {
	Chain
	In
}

type inChain struct {
	Chain
}

func NewInChainWrapper(c Chain) InChain {
	return &inChain{c}
}

func (c *inChain) In() chan Object {
	ops := c.Operators()
	return ops[0].(In).In()
}

func (c *inChain) GetInDepth() int {
	ops := c.Operators()
	return ops[0].(In).GetInDepth()
}

func (c *inChain) SetIn(ch chan Object) {
	ops := c.Operators()
	ops[0].(In).SetIn(ch)
}
<commit_msg>Update chain.go<commit_after>package stream

import (
	"github.com/cevian/go-stream/util/slog"
)

type Chain interface {
	Operators() []Operator
	Run() error
	Stop() error
	Add(o Operator) Chain
	SetName(string) Chain
	//NewSubChain creates a new empty
	//chain inheriting the properties of the parent chain
	//Usefull for distribute/fanout building functions
	NewSubChain() Chain
	//async functions
	Start() error
	Wait() error
}

/* A SimpleChain implements the operator interface too! */
type SimpleChain struct {
	runner *Runner
	//	Ops         []Operator
	//	wg          *sync.WaitGroup
	//	closenotify chan bool
	//	closeerror  chan error
	sentstop bool
	Name     string
}

func NewChain() *SimpleChain {
	return NewSimpleChain()
}

func NewSimpleChain() *SimpleChain {
	return &SimpleChain{runner: NewRunner()}
}

func (c *SimpleChain) Operators() []Operator {
	return c.runner.Operators()
}

func (c *SimpleChain) SetName(name string) Chain {
	c.Name = name
	return c
}

func (c *SimpleChain) NewSubChain() Chain {
	return NewSimpleChain()
}

func (c *SimpleChain) Add(o Operator) Chain {
	ops := c.runner.Operators()
	opIn, isIn := o.(In)
	if isIn && len(ops) > 0 {
		slog.Infof("Setting input channel of %s", Name(o))
		last := ops[len(ops)-1]

		lastOutCh := last.(Out).Out()

		opIn.SetIn(lastOutCh)
	}

	out, ok := o.(Out)
	if ok {
		slog.Infof("Setting output channel of %s", Name(o))
		ch := make(chan Object, CHAN_SLACK)
		out.SetOut(ch)
	}

	c.runner.Add(o)
	return c
}

func (c *SimpleChain) Start() error {
	c.runner.AsyncRunAll()
	return nil
}

func (c *SimpleChain) SoftStop() error {
	if !c.sentstop {
		c.sentstop = true
		slog.Warnf("In soft close")
		ops := c.runner.Operators()
		ops[0].Stop()
	}
	return nil
}

/* A stop is a hard stop as per the Operator interface */
func (c *SimpleChain) Stop() error {
	if !c.sentstop {
		c.sentstop = true
		slog.Warnf("In hard close")
		c.runner.HardStop()
	}
	return nil
}

func (c *SimpleChain) Wait() error {
	//slog.Logf(logger.Levels.Info, "Waiting for closenotify %s", c.Name)
	//<-c.runner.CloseNotifier()
	err, errOk := <-c.runner.ErrorChannel()
	if errOk {
		slog.Warnf("Hard Close in SimpleChain %s %v", c.Name, err)
		c.Stop()
	} else {
		slog.Infof("Soft Close in SimpleChain %s", c.Name)
		c.SoftStop()
	}
	slog.Infof("Waiting for runner to finish")
	c.runner.Wait()
	slog.Infof("Exiting SimpleChain")

	return nil
}

/* Operator compatibility */
func (c *SimpleChain) Run() error {
	if err := c.Start(); err != nil {
		return err
	}
	return c.Wait()
}

type OrderedChain struct {
	*SimpleChain
}

func NewOrderedChain() *OrderedChain {
	return &OrderedChain{NewChain()}
}

func (c *OrderedChain) Add(o Operator) Chain {
	parallel, ok := o.(ParallelizableOperator)
	if ok {
		if !parallel.IsOrdered() {
			parallel = parallel.MakeOrdered()
			if !parallel.IsOrdered() {
				slog.Fatalf("%s", "Couldn't make parallel operator ordered")
			}
		}
		c.SimpleChain.Add(parallel)
	} else {
		c.SimpleChain.Add(o)
	}
	return c
}

func (c *OrderedChain) NewSubChain() Chain {
	return NewOrderedChain()
}

type InChain interface {
	Chain
	In
}

type inChain struct {
	Chain
}

func NewInChainWrapper(c Chain) InChain {
	return &inChain{c}
}

func (c *inChain) In() chan Object {
	ops := c.Operators()
	return ops[0].(In).In()
}

func (c *inChain) GetInDepth() int {
	ops := c.Operators()
	return ops[0].(In).GetInDepth()
}

func (c *inChain) SetIn(ch chan Object) {
	ops := c.Operators()
	ops[0].(In).SetIn(ch)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/binary"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"strconv"
	"strings"

	"github.com/phil-mansfield/gotetra/los"
	"github.com/phil-mansfield/gotetra/los/analyze"
	"github.com/phil-mansfield/gotetra/render/io"
)

type Params struct {
	// HaloProfiles params
	RBins, Spokes, Rings int
	MaxMult, MinMult float64

	// Splashback params
	Order, Window, Levels int
	Cutoff float64
}

func main() {
	// Parse.
	p := parseCmd()
	ids, snaps, err := parseStdin()
	if err != nil { log.Fatal(err.Error()) }

	if len(ids) == 0 { return }

	// Compute coefficients.
	coeffs := make([][]float64, len(ids))
	snapBins, idxBins := binBySnap(snaps, ids)
	buf := make([]analyze.RingBuffer, p.Rings)
	for i := range buf { buf[i].Init(p.Spokes, p.RBins) }

	var losBuf *los.Buffers
	for snap, snapIDs := range snapBins {
		idxs := idxBins[snap]

		if snap == -1 { continue }

		// Bin halos
		hds, files, err := readHeaders(snap)
		if err != nil { err.Error() }
		if losBuf == nil { losBuf = los.NewBuffers(files[0], &hds[0]) }
		halos := createHalos(&hds[0], snapIDs, p)
		intrBins := binIntersections(hds, halos)

		// Add densities. Done header by header to limit I/O time.
		for i := range hds {
			hdContainer := []io.SheetHeader{hds[i]}
			fileContainer := []string{files[i]}
			los.LoadPtrDensities(
				intrBins[i], hdContainer, fileContainer, losBuf,
			)
		}

		for i := range halos {
			coeffs[idxs[i]] = calcCoeffs(&halos[i], buf, p)
		}
	}

	printCoeffs(ids, snaps, coeffs)
}

func parseCmd() *Params {
	// Parse command line.
	p := &Params{}
	flag.IntVar(&p.RBins, "RBins", 256,
		"Number of radial bins used per LoS.")
	flag.IntVar(&p.Spokes, "Spokes", 1024,
		"Number of LoS's used per ring.")
	flag.IntVar(&p.Rings, "Rings", 10,
		"Number of rings used per halo. 3, 4, 6, and 10 rings are\n" +
			"guaranteed to be uniformly spaced.")
	flag.Float64Var(&p.MaxMult, "MaxMult", 3,
		"Ending radius of LoSs as a multiple of R_200m.")
	flag.Float64Var(&p.MinMult, "MinMult", 0.5,
		"Starting radius of LoSs as a multiple of R_200m.")
	flag.IntVar(&p.Order, "Order", 5,
		"Order of the shell fitting function.")
	flag.IntVar(&p.Window, "Window", 121,
		"Number of bins within smoothign window. Must be odd.")
	flag.IntVar(&p.Levels, "Levels", 4,
		"The number of recurve max-finding levels used by the 2D edge finder.")
	flag.Float64Var(&p.Cutoff, "Cutoff", 0.0,
		"The shallowest slope that can be considered a splashback point.")
	flag.Parse()
	return p
}

func parseStdin() (ids, snaps []int, err error) {
	ids, snaps = []int{}, []int{}
	lines, err := stdinLines()
	if err != nil { return nil, nil, err }
	for i, line := range lines {
		tokens := strings.Split(line, " ")

		var (
			id, snap int
			err error
		)
		switch len(tokens) {
		case 0:
			continue
		case 2:
			id, err = strconv.Atoi(tokens[0])
			if err != nil {
				return nil, nil, fmt.Errorf(
					"One line %d of stdin, %s does not parse as an int.",
					i + 1, tokens[0],
				)
			}
			snap, err = strconv.Atoi(tokens[1])
			if err != nil {
				return nil, nil, fmt.Errorf(
					"One line %d of stdin, %s does not parse as an int.",
					i + 1, tokens[1],
				)
			}
		case 1:
			if tokens[0] == "" { continue }
			fallthrough
		default:
			return nil, nil, fmt.Errorf(
				"Line %d of stdin has %d tokens, but 2 are required.",
				i + 1, len(tokens),
			)
		}

		ids = append(ids, id)
		snaps = append(snaps, snap)
	}

	return ids, snaps, err
}

func stdinLines() ([]string, error) {
		bs, err := ioutil.ReadAll(os.Stdin)
	if err != nil {
		return nil, fmt.Errorf(
			"Error reading stdin: %s.", err.Error(),
		)
	}

	text := string(bs)
	return strings.Split(text, "\n"), nil
}

func readHeaders(snap int) ([]io.SheetHeader, []string, error) {
	memoDir := os.Getenv("GTET_MEMO_DIR")
	if memoDir == "" {
		// You don't want to memoize? Fine. Deal with the consequences.
		return readHeadersFromSheet(snap)
	}
	if _, err := os.Stat(memoDir); err != nil {
		return nil, nil, err
	}

	memoFile := path.Join(memoDir, fmt.Sprintf("hd_snap%d.dat", snap))

	if _, err := os.Stat(memoFile); err != nil {
		// File not written yet.
		hds, files, err := readHeadersFromSheet(snap)
		if err != nil { return nil, nil, err }

        f, err := os.Create(memoFile)
        if err != nil { return nil, nil, err }
        defer f.Close()
        binary.Write(f, binary.LittleEndian, hds)

		return hds, files, nil
	} else {
		// File exists: read from it instead.

		f, err := os.Open(memoFile)
        if err != nil { return nil, nil, err }
        defer f.Close()

		n, err := sheetNum(snap)
		if err != nil { return nil, nil, err }
		hds := make([]io.SheetHeader, n)
        binary.Read(f, binary.LittleEndian, hds)

		gtetFmt := os.Getenv("GTET_FMT")
		dir := fmt.Sprintf(gtetFmt, snap)
		files, err := dirContents(dir)
		if err != nil { return nil, nil, err }

		return hds, files, nil
	}

}

func dirContents(dir string) ([]string, error) {
	infos, err := ioutil.ReadDir(dir)
	if err != nil { return nil, err }

	files := make([]string, len(infos))
	for i := range infos {
		files[i] = path.Join(dir, infos[i].Name())
	}

	return files, nil
}

func sheetNum(snap int) (int, error) {
	gtetFmt := os.Getenv("GTET_FMT")
	dir := fmt.Sprintf(gtetFmt, snap)
	files, err := dirContents(dir)
	if err != nil { return 0, err }
	return len(files), nil
}

func readHeadersFromSheet(snap int) ([]io.SheetHeader, []string, error) {
	gtetFmt := os.Getenv("GTET_FMT")
	dir := fmt.Sprintf(gtetFmt, snap)
	files, err := dirContents(dir)
	if err != nil { return nil, nil, err }

	hds := make([]io.SheetHeader, len(files))
	for i := range files {
		err = io.ReadSheetHeaderAt(files[i], &hds[i])
		if err != nil { return nil, nil, err }
	}
	return hds, files, nil
}

func createHalos(hd *io.SheetHeader, ids []int, p *Params) []los.HaloProfiles {
	return nil
}

func binIntersections(
	hds []io.SheetHeader, halos []los.HaloProfiles,
) [][]*los.HaloProfiles {

	bins := make([][]*los.HaloProfiles, len(hds))
	for i := range hds {
		for hi := range halos {
			if (&halos[hi]).SheetIntersect(&hds[i]) {
				bins[i] = append(bins[i], &halos[hi])
			}
		}
	}
	return bins
}

func calcCoeffs(
	halo *los.HaloProfiles, buf []analyze.RingBuffer, p *Params,
) []float64 {
	for i := range buf {
		buf[i].Clear()
		buf[i].Splashback(halo, i, p.Window, p.Cutoff)
	}
	pxs, pys := analyze.FilterPoints(buf, p.Levels)
	cs, _ := analyze.PennaVolumeFit(pxs, pys, halo, p.Order, p.Order)
	return cs
}

func printCoeffs(ids, snaps []int, coeffs [][]float64) {
	idWidth, snapWidth := 0, 0
	coeffWidths := make([]int, len(coeffs[0]))
	for i := range ids {
		iWidth := len(fmt.Sprintf("%d", ids[i]))
		sWidth := len(fmt.Sprintf("%d", snaps[i]))
		if iWidth > idWidth { idWidth = iWidth }
		if sWidth > snapWidth { snapWidth = sWidth }
		for j := range coeffs[i] {
			width := len(fmt.Sprintf("%.5g", coeffs[i][j]))
			if width > coeffWidths[j] { coeffWidths[j] = width }
		}
	}

	rowFmt := fmt.Sprintf("%%%dd %%%dd", idWidth, snapWidth)
	coefFmts := make([]string, len(coeffs[0]))
	for i := range coefFmts {
		rowFmt += fmt.Sprintf(" %%%d.5g", coeffWidths[i])
	}
	rowFmt += "\n"
	for i := range ids {
		args := append([]interface{}{ids[i], snaps[i]}, intr(coeffs[i])...)
		fmt.Printf(rowFmt, args...)
	}
}

func intr(xs []float64) []interface{} {
	is := make([]interface{}, len(xs))
	for i, x := range xs { is[i] = x }
	return is
}

func binBySnap(snaps, ids []int) (snapBins, idxBins map[int][]int) {
	snapBins = make(map[int][]int)
	idxBins = make(map[int][]int)
	for i, snap := range snaps {
		id := ids[i]
		snapBins[snap] = append(snapBins[snap], id)
		idxBins[snap] = append(idxBins[snap], i)
	}
	return snapBins, idxBins
}
<commit_msg>Added halo creation functions to gtet_shell.go<commit_after>package main

import (
	"encoding/binary"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math"
	"math/rand"
	"os"
	"path"
	"sort"
	"strconv"
	"strings"

	"github.com/phil-mansfield/gotetra/los"
	"github.com/phil-mansfield/gotetra/los/geom"
	"github.com/phil-mansfield/gotetra/los/analyze"
	"github.com/phil-mansfield/gotetra/render/io"
	"github.com/phil-mansfield/gotetra/render/halo"
)

type Params struct {
	// HaloProfiles params
	RBins, Spokes, Rings int
	MaxMult, MinMult float64

	// Splashback params
	Order, Window, Levels int
	Cutoff float64
}

func main() {
	// Parse.
	p := parseCmd()
	ids, snaps, err := parseStdin()
	if err != nil { log.Fatal(err.Error()) }

	if len(ids) == 0 { return }

	// Compute coefficients.
	coeffs := make([][]float64, len(ids))
	snapBins, idxBins := binBySnap(snaps, ids)
	buf := make([]analyze.RingBuffer, p.Rings)
	for i := range buf { buf[i].Init(p.Spokes, p.RBins) }

	var losBuf *los.Buffers
	for snap, snapIDs := range snapBins {
		idxs := idxBins[snap]

		if snap == -1 { continue }

		// Bin halos
		hds, files, err := readHeaders(snap)
		if err != nil { err.Error() }
		if losBuf == nil { losBuf = los.NewBuffers(files[0], &hds[0]) }
		halos, err := createHalos(snap, &hds[0], snapIDs, p)
		if err != nil { log.Fatal(err.Error()) }
		intrBins := binIntersections(hds, halos)

		// Add densities. Done header by header to limit I/O time.
		for i := range hds {
			hdContainer := []io.SheetHeader{hds[i]}
			fileContainer := []string{files[i]}
			los.LoadPtrDensities(
				intrBins[i], hdContainer, fileContainer, losBuf,
			)
		}

		// Calculate Penna coefficients.
		for i := range halos {
			coeffs[idxs[i]] = calcCoeffs(&halos[i], buf, p)
		}
	}

	printCoeffs(ids, snaps, coeffs)
}

func parseCmd() *Params {
	// Parse command line.
	p := &Params{}
	flag.IntVar(&p.RBins, "RBins", 256,
		"Number of radial bins used per LoS.")
	flag.IntVar(&p.Spokes, "Spokes", 1024,
		"Number of LoS's used per ring.")
	flag.IntVar(&p.Rings, "Rings", 10,
		"Number of rings used per halo. 3, 4, 6, and 10 rings are\n" +
			"guaranteed to be uniformly spaced.")
	flag.Float64Var(&p.MaxMult, "MaxMult", 3,
		"Ending radius of LoSs as a multiple of R_200m.")
	flag.Float64Var(&p.MinMult, "MinMult", 0.5,
		"Starting radius of LoSs as a multiple of R_200m.")
	flag.IntVar(&p.Order, "Order", 5,
		"Order of the shell fitting function.")
	flag.IntVar(&p.Window, "Window", 121,
		"Number of bins within smoothign window. Must be odd.")
	flag.IntVar(&p.Levels, "Levels", 4,
		"The number of recurve max-finding levels used by the 2D edge finder.")
	flag.Float64Var(&p.Cutoff, "Cutoff", 0.0,
		"The shallowest slope that can be considered a splashback point.")
	flag.Parse()
	return p
}

func parseStdin() (ids, snaps []int, err error) {
	ids, snaps = []int{}, []int{}
	lines, err := stdinLines()
	if err != nil { return nil, nil, err }
	for i, line := range lines {
		tokens := strings.Split(line, " ")

		var (
			id, snap int
			err error
		)
		switch len(tokens) {
		case 0:
			continue
		case 2:
			id, err = strconv.Atoi(tokens[0])
			if err != nil {
				return nil, nil, fmt.Errorf(
					"One line %d of stdin, %s does not parse as an int.",
					i + 1, tokens[0],
				)
			}
			snap, err = strconv.Atoi(tokens[1])
			if err != nil {
				return nil, nil, fmt.Errorf(
					"One line %d of stdin, %s does not parse as an int.",
					i + 1, tokens[1],
				)
			}
		case 1:
			if tokens[0] == "" { continue }
			fallthrough
		default:
			return nil, nil, fmt.Errorf(
				"Line %d of stdin has %d tokens, but 2 are required.",
				i + 1, len(tokens),
			)
		}

		ids = append(ids, id)
		snaps = append(snaps, snap)
	}

	return ids, snaps, err
}

func stdinLines() ([]string, error) {
		bs, err := ioutil.ReadAll(os.Stdin)
	if err != nil {
		return nil, fmt.Errorf(
			"Error reading stdin: %s.", err.Error(),
		)
	}

	text := string(bs)
	return strings.Split(text, "\n"), nil
}

func readHeaders(snap int) ([]io.SheetHeader, []string, error) {
	memoDir := os.Getenv("GTET_MEMO_DIR")
	if memoDir == "" {
		// You don't want to memoize? Fine. Deal with the consequences.
		return readHeadersFromSheet(snap)
	}
	if _, err := os.Stat(memoDir); err != nil {
		return nil, nil, err
	}

	memoFile := path.Join(memoDir, fmt.Sprintf("hd_snap%d.dat", snap))

	if _, err := os.Stat(memoFile); err != nil {
		// File not written yet.
		hds, files, err := readHeadersFromSheet(snap)
		if err != nil { return nil, nil, err }

        f, err := os.Create(memoFile)
        if err != nil { return nil, nil, err }
        defer f.Close()
        binary.Write(f, binary.LittleEndian, hds)

		return hds, files, nil
	} else {
		// File exists: read from it instead.

		f, err := os.Open(memoFile)
        if err != nil { return nil, nil, err }
        defer f.Close()

		n, err := sheetNum(snap)
		if err != nil { return nil, nil, err }
		hds := make([]io.SheetHeader, n)
        binary.Read(f, binary.LittleEndian, hds)

		gtetFmt := os.Getenv("GTET_FMT")
		dir := fmt.Sprintf(gtetFmt, snap)
		files, err := dirContents(dir)
		if err != nil { return nil, nil, err }

		return hds, files, nil
	}

}

func dirContents(dir string) ([]string, error) {
	infos, err := ioutil.ReadDir(dir)
	if err != nil { return nil, err }

	files := make([]string, len(infos))
	for i := range infos {
		files[i] = path.Join(dir, infos[i].Name())
	}

	return files, nil
}

func sheetNum(snap int) (int, error) {
	gtetFmt := os.Getenv("GTET_FMT")
	dir := fmt.Sprintf(gtetFmt, snap)
	files, err := dirContents(dir)
	if err != nil { return 0, err }
	return len(files), nil
}

func readHeadersFromSheet(snap int) ([]io.SheetHeader, []string, error) {
	gtetFmt := os.Getenv("GTET_FMT")
	dir := fmt.Sprintf(gtetFmt, snap)
	files, err := dirContents(dir)
	if err != nil { return nil, nil, err }

	hds := make([]io.SheetHeader, len(files))
	for i := range files {
		err = io.ReadSheetHeaderAt(files[i], &hds[i])
		if err != nil { return nil, nil, err }
	}
	return hds, files, nil
}

func createHalos(
	snap int, hd *io.SheetHeader, ids []int, p *Params,
) ([]los.HaloProfiles, error) {
	// Read coordinates, radii and IDs of halos.
	rockstarDir := os.Getenv("GTET_ROCKSTAR_DIR")
	if rockstarDir == "" {
		return nil, fmt.Errorf("$GTET_ROCKSTAR_DIR not set.")
	}

	files, err := dirContents(rockstarDir)
	if err != nil { return nil, err }
	file := files[snap]

	ids, vals, err := halo.ReadRockstarVals(
		file, &hd.Cosmo, halo.X, halo.Y, halo.Z, halo.M200b,
	)
	if err != nil { return nil, err }

	xs, ys, zs, ms := vals[1], vals[2], vals[3], vals[4]
	halo.R200m.Radius(&hd.Cosmo, ms, ms)
	rs := ms

	// Initialize halos.
	halos := make([]los.HaloProfiles, len(ids))
	seenIDs := make(map[int]bool)
	for i, id := range ids {
		idx := sort.SearchInts(ids, id)
		if ids[idx] !=  id {
			return nil, fmt.Errorf("Halo ID %d not in halo catalog.", id)
		}

		origin := &geom.Vec{
			float32(xs[idx]), float32(ys[idx]), float32(zs[idx]),
		}

		// If we've already seen a halo once, randomize its orientation.
		if seenIDs[id] {
			halos[i].Init(
				id, p.Rings, origin, rs[idx] * p.MinMult, rs[idx] * p.MaxMult,
				p.RBins, p.Spokes, hd.TotalWidth, los.Log(true),
				los.Rotate(float32(2 * math.Pi * rand.Float64()),
                    float32(2 * math.Pi * rand.Float64()),
                    float32(2 * math.Pi * rand.Float64())),
			)
		} else {
			seenIDs[id] = true
			halos[i].Init(
				id, p.Rings, origin, rs[idx] * p.MinMult, rs[idx] * p.MaxMult,
				p.RBins, p.Spokes, hd.TotalWidth, los.Log(true),
			)
		}
	}

	panic("NYI")
}

func binIntersections(
	hds []io.SheetHeader, halos []los.HaloProfiles,
) [][]*los.HaloProfiles {

	bins := make([][]*los.HaloProfiles, len(hds))
	for i := range hds {
		for hi := range halos {
			if (&halos[hi]).SheetIntersect(&hds[i]) {
				bins[i] = append(bins[i], &halos[hi])
			}
		}
	}
	return bins
}

func calcCoeffs(
	halo *los.HaloProfiles, buf []analyze.RingBuffer, p *Params,
) []float64 {
	for i := range buf {
		buf[i].Clear()
		buf[i].Splashback(halo, i, p.Window, p.Cutoff)
	}
	pxs, pys := analyze.FilterPoints(buf, p.Levels)
	cs, _ := analyze.PennaVolumeFit(pxs, pys, halo, p.Order, p.Order)
	return cs
}

func printCoeffs(ids, snaps []int, coeffs [][]float64) {
	idWidth, snapWidth := 0, 0
	coeffWidths := make([]int, len(coeffs[0]))
	for i := range ids {
		iWidth := len(fmt.Sprintf("%d", ids[i]))
		sWidth := len(fmt.Sprintf("%d", snaps[i]))
		if iWidth > idWidth { idWidth = iWidth }
		if sWidth > snapWidth { snapWidth = sWidth }
		for j := range coeffs[i] {
			width := len(fmt.Sprintf("%.5g", coeffs[i][j]))
			if width > coeffWidths[j] { coeffWidths[j] = width }
		}
	}

	rowFmt := fmt.Sprintf("%%%dd %%%dd", idWidth, snapWidth)
	coefFmts := make([]string, len(coeffs[0]))
	for i := range coefFmts {
		rowFmt += fmt.Sprintf(" %%%d.5g", coeffWidths[i])
	}
	rowFmt += "\n"
	for i := range ids {
		args := append([]interface{}{ids[i], snaps[i]}, intr(coeffs[i])...)
		fmt.Printf(rowFmt, args...)
	}
}

func intr(xs []float64) []interface{} {
	is := make([]interface{}, len(xs))
	for i, x := range xs { is[i] = x }
	return is
}

func binBySnap(snaps, ids []int) (snapBins, idxBins map[int][]int) {
	snapBins = make(map[int][]int)
	idxBins = make(map[int][]int)
	for i, snap := range snaps {
		id := ids[i]
		snapBins[snap] = append(snapBins[snap], id)
		idxBins[snap] = append(idxBins[snap], i)
	}
	return snapBins, idxBins
}
<|endoftext|>


<commit_before>// This is the main initialization of the Frala Tool

package main

import (
	"fmt"
	"github.com/JoshStrobl/frala"
	"github.com/JoshStrobl/nflag"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

// OriginalLanguage is the original value of frala
var OriginalLanguage string

// TargetDirectory is the directory we should save parsed files or Po content to
var TargetDirectory string

// Initialization
func init() {
	nflag.Configure(nflag.ConfigOptions{ShowHelpIfNoArgs: true, ProgramDescription: "Frala CLI tool for file parsing and Po conversion."})

	nflag.Set("convert-terms", nflag.Flag{ // Create the convert-terms flag
		Type:         "bool",
		Descriptor:   "Use convert-terms to declare the action of Terms to Po conversion. Don't pass for Po to Terms conversion.",
		AllowNothing: true,
	})

	nflag.Set("lang", nflag.Flag{ // Create the lang Flag
		Descriptor:   "Language to parse files, Terms to Po conversion, or Po to Terms conversion.",
		Type:         "string",
		DefaultValue: frala.Config.DefaultLanguage, // Set DefaultValue to whatever is already set in frala
		AllowNothing: true,                         // Allow nothing, since we'll just default to DefaultValue
	})

	nflag.Set("parse", nflag.Flag{ // Create the parse flag
		Descriptor: "Files to parse. Accepts comma-separated values.",
		Type:       "string",
	})

	nflag.Set("po", nflag.Flag{ // Create the po flag
		Descriptor: "Po file you wish to convert to Terms or save Terms to.",
		Type:       "string",
	})

	currentWorkingDirectory, getWdErr := os.Getwd()

	if getWdErr != nil { // If there was an error getting the working directory
		absolutePathToDir, absolutePathErr := filepath.Abs(".") // Attempt to get the absolute path of the directory

		if absolutePathErr == nil { // If there was no issue getting the absolute path of the directory
			currentWorkingDirectory = absolutePathToDir // Change currentWorkingDirectory to absolutePathToDir
		} else { // If there was an issue getting the the current working directory
			fmt.Println("Unable to determine the current working directory. Exiting.")
			os.Exit(1)
		}
	}

	nflag.Set("target-dir", nflag.Flag{ // Create the target-dir flag
		Descriptor:   "Target directory to save parsed files or Po content to.",
		DefaultValue: currentWorkingDirectory, // Current working directory
		Type:         "string",
	})
}

func main() {
	nflag.Parse() // Parse nflag

	OriginalLanguage = frala.Config.DefaultLanguage                     // Set OriginalLanguage before doing any parsing or conversion
	frala.Config.DefaultLanguage, _ = nflag.GetAsString("lang")         // Change DefaultLanguage to whatever may be set as lang. This may not change if no value is passed to lang
	frala.Config.Direction = frala.GetDirection(frala.Config.Direction) // Ensure we have an accurate Direction

	parseFiles, parseFilesErr := nflag.GetAsString("parse")
	poFile, poFileErr := nflag.GetAsString("po")

	TargetDirectory, _ = nflag.GetAsString("target-dir")
	TargetDirectory += "/" // Ensure / is appending to end of path

	if (parseFilesErr == nil) && (parseFiles != "") { // If we are parsing files
		ParseFiles(parseFiles) // Call ParseFiles
	} else if (poFileErr == nil) && (poFile != "") { // If we are doing Po conversion
		convertTermsVal, convertTermsGetErr := nflag.GetAsBool("convert-terms") // Get the boolean value of convert-terms

		if convertTermsGetErr != nil { // If there was an error getting convertTermsVal
			convertTermsVal = false // Don't convert from Terms to Po
		}

		PoConversion(convertTermsVal, poFile) // Do Po conversion
	} else { // If we are neither parsing nor doing Po conversion
		nflag.PrintFlags() // Print the nflag flags
	}
}

// ParseFiles will parse the files provided and export parsed to target directory
func ParseFiles(parseFiles string) {
	filesToParse := strings.Split(parseFiles, ",")   // Split the comma-separated parseFiles to filesToParse
	parseResponses := frala.MultiParse(filesToParse) // Parse each file

	for fileName, parseResponse := range parseResponses { // For each parseResponse
		if parseResponse.Error == nil { // If there was no issue parsing this file
			if parseResponse.Content != "" { // If the content is not empty
				fileNameNoPath := filepath.Base(fileName) // Get the base filename
				fmt.Println("Writing content to: " + TargetDirectory + fileNameNoPath)
				os.MkdirAll(TargetDirectory, 0755) // Ensure the directory exists
				ioutil.WriteFile(TargetDirectory+fileNameNoPath, []byte(parseResponse.Content), 0755)
			} else { // If there was no content in the parseResponse.Content
				fmt.Println("No content provided via parsing: " + fileName)
			}
		} else { // If there was an issue parsing this file
			fmt.Println(parseResponse.Error) // Print the error
		}
	}
}

// PoConversion will handle conversion to/from Po
func PoConversion(convertTerms bool, poFile string) {
	if strings.HasSuffix(poFile, ".po") { // If the file is a .po file
		if convertTerms { // If we are converting Terms to Po
			os.MkdirAll(TargetDirectory, 0755)                                    // Ensure the target directory exists
			poFileContent := frala.ConvertToPo(frala.Config.DefaultLanguage)      // Convert the Terms of the language defined (or default language)
			ioutil.WriteFile(TargetDirectory+poFile, []byte(poFileContent), 0755) // Save the Po contents to the poFile in the target directory
		} else { // If we are converting Po to Terms
			conversionError := frala.ConvertFromPo(poFile) // Convert the poFile to Frala Terms

			if conversionError == nil { // If there was no conversion error
				frala.Config.DefaultLanguage = OriginalLanguage // Ensure the default language is maintained after importing from Po
				frala.SaveConfig()                              // Save the config
			} else { // If there was a conversion error
				fmt.Println(conversionError)
			}
		}
	} else {
		fmt.Println(poFile + " does not appear to be a Po file. Please ensure the extension is .po")
	}
}
<commit_msg>Update frala-tool to updated MultiParse response and ParseResponse struct.<commit_after>// This is the main initialization of the Frala Tool

package main

import (
	"fmt"
	"github.com/JoshStrobl/frala"
	"github.com/JoshStrobl/nflag"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

// OriginalLanguage is the original value of frala
var OriginalLanguage string

// TargetDirectory is the directory we should save parsed files or Po content to
var TargetDirectory string

// Initialization
func init() {
	nflag.Configure(nflag.ConfigOptions{ShowHelpIfNoArgs: true, ProgramDescription: "Frala CLI tool for file parsing and Po conversion."})

	nflag.Set("convert-terms", nflag.Flag{ // Create the convert-terms flag
		Type:         "bool",
		Descriptor:   "Use convert-terms to declare the action of Terms to Po conversion. Don't pass for Po to Terms conversion.",
		AllowNothing: true,
	})

	nflag.Set("lang", nflag.Flag{ // Create the lang Flag
		Descriptor:   "Language to parse files, Terms to Po conversion, or Po to Terms conversion.",
		Type:         "string",
		DefaultValue: frala.Config.DefaultLanguage, // Set DefaultValue to whatever is already set in frala
		AllowNothing: true,                         // Allow nothing, since we'll just default to DefaultValue
	})

	nflag.Set("parse", nflag.Flag{ // Create the parse flag
		Descriptor: "Files to parse. Accepts comma-separated values.",
		Type:       "string",
	})

	nflag.Set("po", nflag.Flag{ // Create the po flag
		Descriptor: "Po file you wish to convert to Terms or save Terms to.",
		Type:       "string",
	})

	currentWorkingDirectory, getWdErr := os.Getwd()

	if getWdErr != nil { // If there was an error getting the working directory
		absolutePathToDir, absolutePathErr := filepath.Abs(".") // Attempt to get the absolute path of the directory

		if absolutePathErr == nil { // If there was no issue getting the absolute path of the directory
			currentWorkingDirectory = absolutePathToDir // Change currentWorkingDirectory to absolutePathToDir
		} else { // If there was an issue getting the the current working directory
			fmt.Println("Unable to determine the current working directory. Exiting.")
			os.Exit(1)
		}
	}

	nflag.Set("target-dir", nflag.Flag{ // Create the target-dir flag
		Descriptor:   "Target directory to save parsed files or Po content to.",
		DefaultValue: currentWorkingDirectory, // Current working directory
		Type:         "string",
	})
}

func main() {
	nflag.Parse() // Parse nflag

	OriginalLanguage = frala.Config.DefaultLanguage                     // Set OriginalLanguage before doing any parsing or conversion
	frala.Config.DefaultLanguage, _ = nflag.GetAsString("lang")         // Change DefaultLanguage to whatever may be set as lang. This may not change if no value is passed to lang
	frala.Config.Direction = frala.GetDirection(frala.Config.Direction) // Ensure we have an accurate Direction

	parseFiles, parseFilesErr := nflag.GetAsString("parse")
	poFile, poFileErr := nflag.GetAsString("po")

	TargetDirectory, _ = nflag.GetAsString("target-dir")
	TargetDirectory += "/" // Ensure / is appending to end of path

	if (parseFilesErr == nil) && (parseFiles != "") { // If we are parsing files
		ParseFiles(parseFiles) // Call ParseFiles
	} else if (poFileErr == nil) && (poFile != "") { // If we are doing Po conversion
		convertTermsVal, convertTermsGetErr := nflag.GetAsBool("convert-terms") // Get the boolean value of convert-terms

		if convertTermsGetErr != nil { // If there was an error getting convertTermsVal
			convertTermsVal = false // Don't convert from Terms to Po
		}

		PoConversion(convertTermsVal, poFile) // Do Po conversion
	} else { // If we are neither parsing nor doing Po conversion
		nflag.PrintFlags() // Print the nflag flags
	}
}

// ParseFiles will parse the files provided and export parsed to target directory
func ParseFiles(parseFiles string) {
	filesToParse := strings.Split(parseFiles, ",")   // Split the comma-separated parseFiles to filesToParse
	parseResponses := frala.MultiParse(filesToParse) // Parse each file

	for _, parseResponse := range parseResponses { // For each parseResponse
		if parseResponse.Error == nil { // If there was no issue parsing this file
			if parseResponse.Content != "" { // If the content is not empty
				fileNameNoPath := filepath.Base(parseResponse.Name) // Get the base filename
				fmt.Println("Writing content to: " + TargetDirectory + fileNameNoPath)
				os.MkdirAll(TargetDirectory, 0755) // Ensure the directory exists
				ioutil.WriteFile(TargetDirectory+fileNameNoPath, []byte(parseResponse.Content), 0755)
			} else { // If there was no content in the parseResponse.Content
				fmt.Println("No content provided via parsing: " + parseResponse.Name)
			}
		} else { // If there was an issue parsing this file
			fmt.Println(parseResponse.Error) // Print the error
		}
	}
}

// PoConversion will handle conversion to/from Po
func PoConversion(convertTerms bool, poFile string) {
	if strings.HasSuffix(poFile, ".po") { // If the file is a .po file
		if convertTerms { // If we are converting Terms to Po
			os.MkdirAll(TargetDirectory, 0755)                                    // Ensure the target directory exists
			poFileContent := frala.ConvertToPo(frala.Config.DefaultLanguage)      // Convert the Terms of the language defined (or default language)
			ioutil.WriteFile(TargetDirectory+poFile, []byte(poFileContent), 0755) // Save the Po contents to the poFile in the target directory
		} else { // If we are converting Po to Terms
			conversionError := frala.ConvertFromPo(poFile) // Convert the poFile to Frala Terms

			if conversionError == nil { // If there was no conversion error
				frala.Config.DefaultLanguage = OriginalLanguage // Ensure the default language is maintained after importing from Po
				frala.SaveConfig()                              // Save the config
			} else { // If there was a conversion error
				fmt.Println(conversionError)
			}
		}
	} else {
		fmt.Println(poFile + " does not appear to be a Po file. Please ensure the extension is .po")
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.

package server

import (
	"fmt"
	"time"

	"github.com/kyokomi/emoji"
	apns "github.com/sideshow/apns2"
	"github.com/sideshow/apns2/certificate"
	"github.com/sideshow/apns2/payload"
)

type AppleNotificationServer struct {
	ApplePushSettings ApplePushSettings
	AppleClient       *apns.Client
}

func NewAppleNotificationServer(settings ApplePushSettings) NotificationServer {
	return &AppleNotificationServer{ApplePushSettings: settings}
}

func (me *AppleNotificationServer) Initialize() bool {
	LogInfo(fmt.Sprintf("Initializing apple notification server for type=%v", me.ApplePushSettings.Type))

	if len(me.ApplePushSettings.ApplePushCertPrivate) > 0 {
		appleCert, appleCertErr := certificate.FromPemFile(me.ApplePushSettings.ApplePushCertPrivate, me.ApplePushSettings.ApplePushCertPassword)
		if appleCertErr != nil {
			LogCritical(fmt.Sprintf("Failed to load the apple pem cert err=%v for type=%v", appleCertErr, me.ApplePushSettings.Type))
			return false
		}

		if me.ApplePushSettings.ApplePushUseDevelopment {
			me.AppleClient = apns.NewClient(appleCert).Development()
		} else {
			me.AppleClient = apns.NewClient(appleCert).Production()
		}

		return true
	} else {
		LogError(fmt.Sprintf("Apple push notifications not configured.  Missing ApplePushCertPrivate. for type=%v", me.ApplePushSettings.Type))
		return false
	}
}

func (me *AppleNotificationServer) SendNotification(msg *PushNotification) PushResponse {
	notification := &apns.Notification{}
	notification.DeviceToken = msg.DeviceId
	payload := payload.NewPayload()
	notification.Payload = payload
	notification.Topic = me.ApplePushSettings.ApplePushTopic

	payload.Badge(msg.Badge)

	if msg.Type != PUSH_TYPE_CLEAR {
		payload.Category(msg.Category)
		payload.Sound("default")
		payload.Custom("version", msg.Version)

		if len(msg.ChannelName) > 0 && msg.Version == "v2" {
			payload.AlertTitle(msg.ChannelName)
			payload.AlertBody(emoji.Sprint(msg.Message))
			payload.Custom("channel_name", msg.ChannelName)
		} else {
			payload.Alert(emoji.Sprint(msg.Message))

			if len(msg.ChannelName) > 0 {
				payload.Custom("channel_name", msg.ChannelName)
			}
		}
	} else {
		payload.Alert("")
	}

	payload.Custom("type", msg.Type)

	if len(msg.ChannelId) > 0 {
		payload.Custom("channel_id", msg.ChannelId)
	}

	if len(msg.TeamId) > 0 {
		payload.Custom("team_id", msg.TeamId)
	}

	if len(msg.SenderId) > 0 {
		payload.Custom("sender_id", msg.SenderId)
	}

	if len(msg.PostId) > 0 {
		payload.Custom("post_id", msg.PostId)
	}

	if len(msg.RootId) > 0 {
		payload.Custom("root_id", msg.RootId)
	}

	if len(msg.OverrideUsername) > 0 {
		payload.Custom("override_username", msg.OverrideUsername)
	}

	if len(msg.OverrideIconUrl) > 0 {
		payload.Custom("override_icon_url", msg.OverrideIconUrl)
	}

	if len(msg.FromWebhook) > 0 {
		payload.Custom("from_webhook", msg.FromWebhook)
	}

	if me.AppleClient != nil {
		LogInfo(fmt.Sprintf("Sending apple push notification type=%v", me.ApplePushSettings.Type))
		start := time.Now()
		res, err := me.AppleClient.Push(notification)
		observeAPNSResponse(time.Since(start).Seconds())
		if err != nil {
			LogError(fmt.Sprintf("Failed to send apple push sid=%v did=%v err=%v type=%v", msg.ServerId, msg.DeviceId, err, me.ApplePushSettings.Type))
			incrementFailure(me.ApplePushSettings.Type)
			return NewErrorPushResponse("unknown transport error")
		}

		if !res.Sent() {
			if res.Reason == "BadDeviceToken" || res.Reason == "Unregistered" || res.Reason == "MissingDeviceToken" || res.Reason == "DeviceTokenNotForTopic" {
				LogInfo(fmt.Sprintf("Failed to send apple push sending remove code res ApnsID=%v reason=%v code=%v type=%v", res.ApnsID, res.Reason, res.StatusCode, me.ApplePushSettings.Type))
				incrementRemoval(me.ApplePushSettings.Type)
				return NewRemovePushResponse()
			}

			LogError(fmt.Sprintf("Failed to send apple push with res ApnsID=%v reason=%v code=%v type=%v", res.ApnsID, res.Reason, res.StatusCode, me.ApplePushSettings.Type))
			incrementFailure(me.ApplePushSettings.Type)
			return NewErrorPushResponse("unknown send response error")
		}
	}

	incrementSuccess(me.ApplePushSettings.Type)
	return NewOkPushResponse()
}
<commit_msg>Use aps thread-id to group notifications from the same channel in iOS12<commit_after>// Copyright (c) 2015 Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.

package server

import (
	"fmt"
	"time"

	"github.com/kyokomi/emoji"
	apns "github.com/sideshow/apns2"
	"github.com/sideshow/apns2/certificate"
	"github.com/sideshow/apns2/payload"
)

type AppleNotificationServer struct {
	ApplePushSettings ApplePushSettings
	AppleClient       *apns.Client
}

func NewAppleNotificationServer(settings ApplePushSettings) NotificationServer {
	return &AppleNotificationServer{ApplePushSettings: settings}
}

func (me *AppleNotificationServer) Initialize() bool {
	LogInfo(fmt.Sprintf("Initializing apple notification server for type=%v", me.ApplePushSettings.Type))

	if len(me.ApplePushSettings.ApplePushCertPrivate) > 0 {
		appleCert, appleCertErr := certificate.FromPemFile(me.ApplePushSettings.ApplePushCertPrivate, me.ApplePushSettings.ApplePushCertPassword)
		if appleCertErr != nil {
			LogCritical(fmt.Sprintf("Failed to load the apple pem cert err=%v for type=%v", appleCertErr, me.ApplePushSettings.Type))
			return false
		}

		if me.ApplePushSettings.ApplePushUseDevelopment {
			me.AppleClient = apns.NewClient(appleCert).Development()
		} else {
			me.AppleClient = apns.NewClient(appleCert).Production()
		}

		return true
	} else {
		LogError(fmt.Sprintf("Apple push notifications not configured.  Missing ApplePushCertPrivate. for type=%v", me.ApplePushSettings.Type))
		return false
	}
}

func (me *AppleNotificationServer) SendNotification(msg *PushNotification) PushResponse {
	notification := &apns.Notification{}
	notification.DeviceToken = msg.DeviceId
	payload := payload.NewPayload()
	notification.Payload = payload
	notification.Topic = me.ApplePushSettings.ApplePushTopic

	payload.Badge(msg.Badge)

	if msg.Type != PUSH_TYPE_CLEAR {
		payload.Category(msg.Category)
		payload.Sound("default")
		payload.Custom("version", msg.Version)

		if len(msg.ChannelName) > 0 && msg.Version == "v2" {
			payload.AlertTitle(msg.ChannelName)
			payload.AlertBody(emoji.Sprint(msg.Message))
			payload.Custom("channel_name", msg.ChannelName)
		} else {
			payload.Alert(emoji.Sprint(msg.Message))

			if len(msg.ChannelName) > 0 {
				payload.Custom("channel_name", msg.ChannelName)
			}
		}
	} else {
		payload.Alert("")
	}

	payload.Custom("type", msg.Type)

	if len(msg.ChannelId) > 0 {
		payload.Custom("channel_id", msg.ChannelId)
		payload.ThreadID(msg.ChannelId)
	}

	if len(msg.TeamId) > 0 {
		payload.Custom("team_id", msg.TeamId)
	}

	if len(msg.SenderId) > 0 {
		payload.Custom("sender_id", msg.SenderId)
	}

	if len(msg.PostId) > 0 {
		payload.Custom("post_id", msg.PostId)
	}

	if len(msg.RootId) > 0 {
		payload.Custom("root_id", msg.RootId)
	}

	if len(msg.OverrideUsername) > 0 {
		payload.Custom("override_username", msg.OverrideUsername)
	}

	if len(msg.OverrideIconUrl) > 0 {
		payload.Custom("override_icon_url", msg.OverrideIconUrl)
	}

	if len(msg.FromWebhook) > 0 {
		payload.Custom("from_webhook", msg.FromWebhook)
	}

	if me.AppleClient != nil {
		LogInfo(fmt.Sprintf("Sending apple push notification type=%v", me.ApplePushSettings.Type))
		start := time.Now()
		res, err := me.AppleClient.Push(notification)
		observeAPNSResponse(time.Since(start).Seconds())
		if err != nil {
			LogError(fmt.Sprintf("Failed to send apple push sid=%v did=%v err=%v type=%v", msg.ServerId, msg.DeviceId, err, me.ApplePushSettings.Type))
			incrementFailure(me.ApplePushSettings.Type)
			return NewErrorPushResponse("unknown transport error")
		}

		if !res.Sent() {
			if res.Reason == "BadDeviceToken" || res.Reason == "Unregistered" || res.Reason == "MissingDeviceToken" || res.Reason == "DeviceTokenNotForTopic" {
				LogInfo(fmt.Sprintf("Failed to send apple push sending remove code res ApnsID=%v reason=%v code=%v type=%v", res.ApnsID, res.Reason, res.StatusCode, me.ApplePushSettings.Type))
				incrementRemoval(me.ApplePushSettings.Type)
				return NewRemovePushResponse()
			}

			LogError(fmt.Sprintf("Failed to send apple push with res ApnsID=%v reason=%v code=%v type=%v", res.ApnsID, res.Reason, res.StatusCode, me.ApplePushSettings.Type))
			incrementFailure(me.ApplePushSettings.Type)
			return NewErrorPushResponse("unknown send response error")
		}
	}

	incrementSuccess(me.ApplePushSettings.Type)
	return NewOkPushResponse()
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"os"
	"unicode"
)

const BUFSIZE int = 70

var (
	minspan = flag.Int("m", 6, "Defines the minimum span size for a series "+
		"of runes to be considered a string.")
	offsets = flag.Bool("o", false, "Print the file offsets for the "+
		"beginning of strings.")
)

func usage() {
	fmt.Fprintf(os.Stderr, "Usage: strings [-o] [-m min] [file ...]\n")
	flag.PrintDefaults()
	os.Exit(2)
}

func error(s string) {
	fmt.Fprint(os.Stderr, s, "\n")
	os.Exit(1)
}

func prntstr(s string, start int64) {
	if *offsets {
		fmt.Printf("%8d: %s\n", start, s)
	} else {
		fmt.Printf("%s\n", s)
	}
}

func stringit(f *os.File) {
	buf := make([]rune, BUFSIZE)
	var start, posn int64

	b := bufio.NewReader(f)

	posn = 0
	start = 0
	for {
		c, size, err := b.ReadRune()
		if err != nil {
			break
		}
		posn += int64(size)

		if unicode.IsGraphic(c) {
			if start == 0 {
				start = posn
			}
			buf = append(buf, c)
			if len(buf) == BUFSIZE {
				prntstr(string(buf)+" ...", start)
				buf = buf[0:0]
				start = 0
			}
		} else {
			if len(buf) >= *minspan {
				prntstr(string(buf), start)
				buf = buf[0:0]
				start = 0
			}
		}
	}

	if len(buf) >= *minspan {
		prntstr(string(buf), start)
	}
}

func main() {
	flag.Usage = usage
	flag.Parse()
	args := flag.Args()

	if len(args) == 0 {
		stringit(os.Stdin)
	} else {
		for _, path := range args {
			f, err := os.Open(path)
			if err != nil {
				fmt.Fprintf(os.Stderr, "open %s: %s", path, err)
				continue
			}
			stringit(f)
			f.Close()
		}
	}

	os.Exit(0)
}
<commit_msg>cleaned up code and added "..." when len(string) > BUFSIZE<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"os"
	"unicode"
)

const BUFSIZE int = 70

var (
	minSpan = flag.Int("m", 6, "Defines the minimum span size for a series "+
		"of runes to be considered a string.")
	verbose = flag.Bool("v", false, "Print the file offsets for the "+
		"beginning of strings.")
)

func usage() {
	fmt.Fprintf(os.Stderr, "Usage: strings [-o] [-m min] [file ...]\n")
	flag.PrintDefaults()
	os.Exit(2)
}

func error(s string) {
	fmt.Fprint(os.Stderr, s, "\n")
	os.Exit(1)
}

func makeString(f *os.File) {
	buf := make([]rune, BUFSIZE)
	b := bufio.NewReader(f)

	var position, start int64
	printString := func(buf []rune, start int64) {
		if *verbose {
			fmt.Printf("%8d: %s\n", start, string(buf))
		} else {
			fmt.Printf("%s\n", string(buf))
		}
	}

	for {
		c, size, err := b.ReadRune()
		if err != nil {
			break
		}
		position += int64(size)

		if unicode.IsGraphic(c) {
			if start == 0 {
				start = position
			}
			buf = append(buf, c)
			if len(buf) == BUFSIZE {
				buf = append(buf, []rune("...")...)
				printString(buf, start)
				buf = buf[0:0]
				start = 0
			}
		} else {
			if len(buf) >= *minSpan {
				printString(buf, start)
				buf = buf[0:0]
				start = 0
			}
		}
	}

	if len(buf) >= *minSpan {
		printString(buf, start)
	}
}

func main() {
	flag.Usage = usage
	flag.Parse()
	args := flag.Args()

	if len(args) == 0 {
		makeString(os.Stdin)
	} else {
		for _, path := range args {
			f, err := os.Open(path)
			if err != nil {
				fmt.Fprintf(os.Stderr, "open %s: %s", path, err)
				continue
			}
			makeString(f)
			f.Close()
		}
	}

	os.Exit(0)
}
<|endoftext|>


<commit_before>package main

import (
	"net/http"

	"github.com/privacybydesign/irmago/server/myirmaserver"
)

func main() {
	/*db := &myirmaserver.MyirmaMemoryDB{
		UserData: map[string]myirmaserver.MemoryUserData{
			"rgBpfxdwfE": myirmaserver.MemoryUserData{
				ID:    1,
				Email: []string{"test@test.com"},
				LogEntries: []myirmaserver.LogEntry{
					myirmaserver.LogEntry{
						Event:     "IRMA_SESSION",
						Timestamp: time.Now().Unix(),
					},
				},
			},
			"blabla": myirmaserver.MemoryUserData{
				ID:    2,
				Email: []string{"test@test.com", "test2@test.com"},
			},
		},
		LoginEmailTokens: map[string]string{},
		VerifyEmailTokens: map[string]int64{
			"blablabla": 2,
		},
	}*/
	db, err := myirmaserver.NewPostgresDatabase("postgresql://localhost:5432/test")
	s, err := myirmaserver.New(&myirmaserver.Configuration{
		URL:                    "http://127.0.0.1:8000",
		SchemesPath:            "../../../testdata/irma_configuration",
		StaticPath:             "irma_keyshare_webclient/build",
		StaticPrefix:           "/test/",
		DB:                     db,
		KeyshareAttributeNames: []string{"test.test.mijnirma.email"},
		EmailAttributeNames:    []string{"test.test.mijnirma.email"},
		EmailServer:            "localhost:1025",
		EmailFrom:              "test@example.com",
		DefaultLanguage:        "en",
		LoginEmailFiles:        map[string]string{"en": "testtemplate.html"},
		LoginEmailSubject:      map[string]string{"en": "Login MyIRMA"},
		LoginEmailBaseURL:      map[string]string{"en": "http://127.0.0.1:8000/test/#token="},
	})
	if err != nil {
		panic(err)
	}
	http.ListenAndServe("localhost:8000", s.Handler())
}
<commit_msg>Removed temporary scaffolding for myirmaserver.<commit_after><|endoftext|>


<commit_before>package keymgr

import (
	"bufio"
	"bytes"
	"encoding/hex"
	"fmt"
	"os"
	"strconv"
	"strings"
	"time"
	//"crypto/rand"
	//"github.com/codahale/blake2"
)

const (
	date_format     string = "2006-01-02"
	generatedFormat string = "Mon 02 Jan 2006 15:04:05 GMT"
)

type Remailer struct {
	name    string    // Remailer Shortname
	Address string    // Remailer Address
	Keyid   []byte    // 16 Byte Mixmaster KeyID
	version string    // Mixmaster version
	caps    string    // Remailer capstring
	PK      []byte    // Curve25519 Public Key
	from    time.Time // Valid-from date
	until   time.Time // Valid until date
	latent  int       // Latency (minutes)
	uptime  int       // Uptime (10ths of a %)
}

type Pubring struct {
	pubringFile    string // Pubring filename
	statsFile      string // mlist type file
	pub            map[string]Remailer
	xref           map[string]string // A cross-reference of shortnames to addresses
	Stats          bool              // Have current reliability stats been imported?
	useExpired     bool              // Consider expired keys as candidates (Echolot wants this)
	advertised     string            // The keyid a local server is currently advertising
	keysImported   time.Time         // Timestamp on most recently read pubring.mix file
	statsImported  time.Time         // Timestamp on most recently read mlist2.txt file
	statsGenerated time.Time         // Generated timestamp on mlist2.txt file
}

func NewPubring(pubfile, statfile string, useExpired bool) *Pubring {
	return &Pubring{
		pubringFile: pubfile,
		statsFile:   statfile,
		pub:         make(map[string]Remailer),
		xref:        make(map[string]string),
		Stats:       false,
		useExpired:  useExpired,
	}
}

// StatsStale returns true if stats are over h hours old
func (p *Pubring) StatsStale(h int) bool {
	return p.Stats && int((time.Since(p.statsGenerated).Hours())) > h
}

// KeyRefresh returns True if the Pubring file has been modified
func (p *Pubring) KeyRefresh() bool {
	stat, err := os.Stat(p.pubringFile)
	if err != nil {
		panic(err)
	}
	return stat.ModTime().After(p.keysImported)
}

// StatRefresh returns True if the mlist2.txt file has been modified
func (p *Pubring) StatRefresh() (refresh bool, err error) {
	stat, err := os.Stat(p.statsFile)
	if err != nil {
		// Cannot read the stats file
		err = fmt.Errorf("%s: File not found", p.statsFile)
		return
	}
	refresh = stat.ModTime().After(p.statsImported)
	return
}

// Candidates provides a list of remailer addresses that match the specified criteria
func (p Pubring) Candidates(minlat, maxlat int, minrel float32, exit bool) (c []string) {
	for addy := range p.pub {
		stats := p.pub[addy]
		if exit {
			if strings.Contains(stats.caps, "M") {
				// Exits are required and this is a Middle
				continue
			}
		}
		if stats.latent < minlat || stats.latent > maxlat {
			continue
		}
		if stats.uptime < int(minrel*10) {
			continue
		}
		c = append(c, addy)
	}
	return
}

// Count returns the number of known Public keys
func (p Pubring) Count() int {
	return len(p.pub)
}

// Produces a list of public key headers
func (p Pubring) KeyList() (addresses []string) {
	for addy := range p.pub {
		key := p.pub[addy]
		header := key.name + " "
		header += key.Address + " "
		header += hex.EncodeToString(key.Keyid) + " "
		header += key.version + " "
		header += key.caps + " "
		header += key.from.UTC().Format(date_format) + " "
		header += key.until.UTC().Format(date_format)
		addresses = append(addresses, header)
	}
	return
}

// Put inserts a new remailer struct into the Keyring
func (p Pubring) Put(r Remailer) {
	p.pub[r.Address] = r
	p.xref[r.name] = r.Address
}

// Get returns a remailer's public info when requested by name or address
func (p Pubring) Get(ref string) (r Remailer, err error) {
	var exists bool
	if strings.Contains(ref, "@") {
		r, exists = p.pub[ref]
		if !exists {
			err = fmt.Errorf("%s: Remailer address not found in public keyring", ref)
			return
		}
	} else {
		var addy string
		addy, exists = p.xref[ref]
		if !exists {
			err = fmt.Errorf("%s: Remailer name not found in public keyring", ref)
			return
		}
		r = p.pub[addy]
	}
	return
}

func (p *Pubring) ImportStats() (err error) {
	f, err := os.Open(p.statsFile)
	if err != nil {
		return
	}
	scanner := bufio.NewScanner(f)
	var elements []string
	var line string     //Each line in mlist2.txt
	var rem_name string //Remailer name in stats
	var rem_addy string //Remailer address from xref
	var lat []string    //Latency hours:minutes
	var lathrs int      //Latent Hours
	var latmin int      //Latent Minutes
	var exists bool     //Test for presence of remailer in xref
	stat_phase := 0
	/* Stat phases are:
	0 Want Generated timestamp
	1 Expecting long string of dashes
	2 Expecting stats lines
	*/
	for scanner.Scan() {
		line = scanner.Text()
		switch stat_phase {
		case 0:
			// Generated: Tue 07 Oct 2014 10:50:01 GMT
			if strings.HasPrefix(line, "Generated: ") {
				p.statsGenerated, err = time.Parse(generatedFormat, line[11:])
				if err != nil {
					err = fmt.Errorf("Failed to parse Generated date: %s", err)
					return
				}
				stat_phase++
			}
		case 1:
			// Expecting dashes
			if strings.HasPrefix(line, "----------") {
				stat_phase++
			}
		case 2:
			// Expecting stats
			line = strings.Split(line, "%")[0]
			elements = strings.Fields(line)
			if len(elements) == 5 {
				rem_name = elements[0]
				_, exists = p.xref[rem_name]
				if exists {
					rem_addy = p.xref[rem_name]
					// Element 2 is Latency in the format (hrs:mins)
					lat = strings.Split(elements[2], ":")
					if lat[0] == "" {
						lathrs = 0
					} else {
						lathrs, err = strconv.Atoi(lat[0])
						if err != nil {
							fmt.Fprintf(os.Stderr, "%s: Invalid latent hours\n", rem_name)
							continue
						}
						if lathrs < 0 || lathrs > 99 {
							fmt.Fprintf(os.Stderr, "%s: Latent hours out of range\n", rem_name)
							continue
						}
					}
					latmin, err = strconv.Atoi(lat[1])
					if err != nil {
						fmt.Fprintf(os.Stderr, "%s: Invalid latent minutes\n", rem_name)
						continue
					}
					if latmin < 0 || latmin > 59 {
						fmt.Fprintf(os.Stderr, "%s: Latent minutes out of range\n", rem_name)
						continue
					}
					// Element 4 is Uptime in format (xxx.xx)
					uptmp, err := strconv.ParseFloat(elements[4], 32)
					if err != nil {
						fmt.Fprintf(os.Stderr, "%s: Invalid uptime\n", rem_name)
						continue
					}
					if uptmp < 0 || uptmp > 100 {
						fmt.Fprintf(os.Stderr, "%s: Uptime out of range\n", rem_name)
						continue
					}
					tmp := p.pub[rem_addy]
					tmp.latent = (lathrs * 60) + latmin
					tmp.uptime = int(uptmp * 10)
					p.pub[rem_addy] = tmp
				} else {
					fmt.Fprintf(os.Stderr, "%s: Stats for unknown remailer\n", rem_name)
				}
			} else {
				stat_phase++
			}
		case 3:
			// Reserved for future mlist2.txt processing
			break
		}
	}
	// Update last-imported timestamp for stats
	stat, err := os.Stat(p.statsFile)
	if err != nil {
		panic(err)
	}
	p.statsImported = stat.ModTime()
	p.Stats = true
	return
}

func Headers(filename string) (headers []string, err error) {
	var f *os.File
	f, err = os.Open(filename)
	if err != nil {
		return
	}
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		line := scanner.Text()
		if len(strings.Split(line, " ")) == 7 {
			headers = append(headers, line)
		}
	}
	return
}

// ImportPubring reads a YAMN Pubring.mix file
func (p *Pubring) ImportPubring() (err error) {
	var f *os.File
	f, err = os.Open(p.pubringFile)
	if err != nil {
		return
	}
	scanner := bufio.NewScanner(f)
	var elements []string
	var num_elements int
	var line string //Each line within Pubring.mix
	var rem *Remailer
	var pkdata []byte // Decoded Public key
	now := time.Now() // Current time for key validity testing
	key_phase := 0
	/* Key phases are:
	0	Expecting header line
	1 Expecting Begin cutmark
	2 Expecting Keyid line
	3	Expecting public key
	4 Got End cutmark
	*/

	for scanner.Scan() {
		line = scanner.Text()
		switch key_phase {
		case 0:
			// Expecting key header line
			elements = strings.Split(line, " ")
			num_elements = len(elements)
			// 7 elements indicates a remailer header line in Pubring.mix
			if num_elements == 7 {
				//TODO Decide if validity dates should be authenticated here
				from, err := time.Parse(date_format, elements[5])
				if err != nil {
					fmt.Fprintln(os.Stderr, "Malformed valid-from date")
					key_phase = 0
					continue
				}
				if now.Before(from) {
					fmt.Fprintln(os.Stderr, elements[0]+": Key not yet valid")
					key_phase = 0
					continue
				}
				until, err := time.Parse(date_format, elements[6])
				if err != nil {
					fmt.Fprintln(os.Stderr, "Malformed valid-to date")
					key_phase = 0
					continue
				}
				if now.After(until) {
					fmt.Fprintf(
						os.Stderr,
						"Key expired: Name=%s, Key=%s, Date=%s\n",
						elements[0],
						elements[2],
						elements[6],
					)
					// Echolot likes to ping expired keys.
					// This option dictates if expired keys
					// should be included as condidates.
					if !p.useExpired {
						key_phase = 0
						continue
					}
				}
				rem = new(Remailer)
				rem.name = elements[0]
				rem.Keyid, err = hex.DecodeString(elements[2])
				if err != nil {
					// keyid is not valid hex
					fmt.Fprintln(os.Stderr, "Keyid in header is not hex")
					key_phase = 0
					continue
				}
				rem.version = elements[3]
				rem.caps = elements[4]
				rem.Address = elements[1]
				key_phase = 1
			}
		case 1:
			// Expecting Begin cutmark
			if line == "-----Begin Mix Key-----" {
				key_phase = 2
			}
		case 2:
			// Expecting Keyid line
			keyid, err := hex.DecodeString(line)
			if err != nil {
				// keyid is not valid hex
				fmt.Fprintln(os.Stderr, "Keyid in pubkey is not hex")
				key_phase = 0
				continue
			}
			if !bytes.Equal(keyid, rem.Keyid) {
				// Corrupt keyblock - header keyid doesn't match keyid in block
				fmt.Fprintln(os.Stderr, "Keyid in header differs from keyid in pubkey")
				key_phase = 0
				continue
			}
			key_phase = 3
		case 3:
			// Expecting Public key
			pkdata, err = hex.DecodeString(line)
			if err != nil {
				// Public key is not valid hex
				fmt.Fprintln(os.Stderr, "Unable to decode Public key")
				key_phase = 0
				continue
			}
			if len(pkdata) != 32 {
				fmt.Fprintln(os.Stderr, "Public key is not 32 bits")
				key_phase = 0
				continue
			}
			rem.PK = pkdata
			key_phase = 4
		case 4:
			// Expecting end cutmark
			if line == "-----End Mix Key-----" {
				p.Put(*rem)
				key_phase = 0
			}
		} // End of phases
	} // End of file scan loop

	// Set key imported timestamp
	stat, err := os.Stat(p.pubringFile)
	if err != nil {
		panic(err)
	}
	p.keysImported = stat.ModTime()
	return
}
<commit_msg>Invert a condition to save an indent level<commit_after>package keymgr

import (
	"bufio"
	"bytes"
	"encoding/hex"
	"fmt"
	"os"
	"strconv"
	"strings"
	"time"
	//"crypto/rand"
	//"github.com/codahale/blake2"
)

const (
	date_format     string = "2006-01-02"
	generatedFormat string = "Mon 02 Jan 2006 15:04:05 GMT"
)

type Remailer struct {
	name    string    // Remailer Shortname
	Address string    // Remailer Address
	Keyid   []byte    // 16 Byte Mixmaster KeyID
	version string    // Mixmaster version
	caps    string    // Remailer capstring
	PK      []byte    // Curve25519 Public Key
	from    time.Time // Valid-from date
	until   time.Time // Valid until date
	latent  int       // Latency (minutes)
	uptime  int       // Uptime (10ths of a %)
}

type Pubring struct {
	pubringFile    string // Pubring filename
	statsFile      string // mlist type file
	pub            map[string]Remailer
	xref           map[string]string // A cross-reference of shortnames to addresses
	Stats          bool              // Have current reliability stats been imported?
	useExpired     bool              // Consider expired keys as candidates (Echolot wants this)
	advertised     string            // The keyid a local server is currently advertising
	keysImported   time.Time         // Timestamp on most recently read pubring.mix file
	statsImported  time.Time         // Timestamp on most recently read mlist2.txt file
	statsGenerated time.Time         // Generated timestamp on mlist2.txt file
}

func NewPubring(pubfile, statfile string, useExpired bool) *Pubring {
	return &Pubring{
		pubringFile: pubfile,
		statsFile:   statfile,
		pub:         make(map[string]Remailer),
		xref:        make(map[string]string),
		Stats:       false,
		useExpired:  useExpired,
	}
}

// StatsStale returns true if stats are over h hours old
func (p *Pubring) StatsStale(h int) bool {
	return p.Stats && int((time.Since(p.statsGenerated).Hours())) > h
}

// KeyRefresh returns True if the Pubring file has been modified
func (p *Pubring) KeyRefresh() bool {
	stat, err := os.Stat(p.pubringFile)
	if err != nil {
		panic(err)
	}
	return stat.ModTime().After(p.keysImported)
}

// StatRefresh returns True if the mlist2.txt file has been modified
func (p *Pubring) StatRefresh() (refresh bool, err error) {
	stat, err := os.Stat(p.statsFile)
	if err != nil {
		// Cannot read the stats file
		err = fmt.Errorf("%s: File not found", p.statsFile)
		return
	}
	refresh = stat.ModTime().After(p.statsImported)
	return
}

// Candidates provides a list of remailer addresses that match the specified criteria
func (p Pubring) Candidates(minlat, maxlat int, minrel float32, exit bool) (c []string) {
	for addy := range p.pub {
		stats := p.pub[addy]
		if exit {
			if strings.Contains(stats.caps, "M") {
				// Exits are required and this is a Middle
				continue
			}
		}
		if stats.latent < minlat || stats.latent > maxlat {
			continue
		}
		if stats.uptime < int(minrel*10) {
			continue
		}
		c = append(c, addy)
	}
	return
}

// Count returns the number of known Public keys
func (p Pubring) Count() int {
	return len(p.pub)
}

// Produces a list of public key headers
func (p Pubring) KeyList() (addresses []string) {
	for addy := range p.pub {
		key := p.pub[addy]
		header := key.name + " "
		header += key.Address + " "
		header += hex.EncodeToString(key.Keyid) + " "
		header += key.version + " "
		header += key.caps + " "
		header += key.from.UTC().Format(date_format) + " "
		header += key.until.UTC().Format(date_format)
		addresses = append(addresses, header)
	}
	return
}

// Put inserts a new remailer struct into the Keyring
func (p Pubring) Put(r Remailer) {
	p.pub[r.Address] = r
	p.xref[r.name] = r.Address
}

// Get returns a remailer's public info when requested by name or address
func (p Pubring) Get(ref string) (r Remailer, err error) {
	var exists bool
	if strings.Contains(ref, "@") {
		r, exists = p.pub[ref]
		if !exists {
			err = fmt.Errorf("%s: Remailer address not found in public keyring", ref)
			return
		}
	} else {
		var addy string
		addy, exists = p.xref[ref]
		if !exists {
			err = fmt.Errorf("%s: Remailer name not found in public keyring", ref)
			return
		}
		r = p.pub[addy]
	}
	return
}

func (p *Pubring) ImportStats() (err error) {
	f, err := os.Open(p.statsFile)
	if err != nil {
		return
	}
	scanner := bufio.NewScanner(f)
	var elements []string
	var line string     //Each line in mlist2.txt
	var rem_name string //Remailer name in stats
	var rem_addy string //Remailer address from xref
	var lat []string    //Latency hours:minutes
	var lathrs int      //Latent Hours
	var latmin int      //Latent Minutes
	var exists bool     //Test for presence of remailer in xref
	stat_phase := 0
	/* Stat phases are:
	0 Want Generated timestamp
	1 Expecting long string of dashes
	2 Expecting stats lines
	*/
	for scanner.Scan() {
		line = scanner.Text()
		switch stat_phase {
		case 0:
			// Generated: Tue 07 Oct 2014 10:50:01 GMT
			if strings.HasPrefix(line, "Generated: ") {
				p.statsGenerated, err = time.Parse(generatedFormat, line[11:])
				if err != nil {
					err = fmt.Errorf("Failed to parse Generated date: %s", err)
					return
				}
				stat_phase++
			}
		case 1:
			// Expecting dashes
			if strings.HasPrefix(line, "----------") {
				stat_phase++
			}
		case 2:
			// Expecting stats
			line = strings.Split(line, "%")[0]
			elements = strings.Fields(line)
			if len(elements) == 5 {
				rem_name = elements[0]
				_, exists = p.xref[rem_name]
				if exists {
					rem_addy = p.xref[rem_name]
					// Element 2 is Latency in the format (hrs:mins)
					lat = strings.Split(elements[2], ":")
					if lat[0] == "" {
						lathrs = 0
					} else {
						lathrs, err = strconv.Atoi(lat[0])
						if err != nil {
							fmt.Fprintf(os.Stderr, "%s: Invalid latent hours\n", rem_name)
							continue
						}
						if lathrs < 0 || lathrs > 99 {
							fmt.Fprintf(os.Stderr, "%s: Latent hours out of range\n", rem_name)
							continue
						}
					}
					latmin, err = strconv.Atoi(lat[1])
					if err != nil {
						fmt.Fprintf(os.Stderr, "%s: Invalid latent minutes\n", rem_name)
						continue
					}
					if latmin < 0 || latmin > 59 {
						fmt.Fprintf(os.Stderr, "%s: Latent minutes out of range\n", rem_name)
						continue
					}
					// Element 4 is Uptime in format (xxx.xx)
					uptmp, err := strconv.ParseFloat(elements[4], 32)
					if err != nil {
						fmt.Fprintf(os.Stderr, "%s: Invalid uptime\n", rem_name)
						continue
					}
					if uptmp < 0 || uptmp > 100 {
						fmt.Fprintf(os.Stderr, "%s: Uptime out of range\n", rem_name)
						continue
					}
					tmp := p.pub[rem_addy]
					tmp.latent = (lathrs * 60) + latmin
					tmp.uptime = int(uptmp * 10)
					p.pub[rem_addy] = tmp
				} else {
					fmt.Fprintf(os.Stderr, "%s: Stats for unknown remailer\n", rem_name)
				}
			} else {
				stat_phase++
			}
		case 3:
			// Reserved for future mlist2.txt processing
			break
		}
	}
	// Update last-imported timestamp for stats
	stat, err := os.Stat(p.statsFile)
	if err != nil {
		panic(err)
	}
	p.statsImported = stat.ModTime()
	p.Stats = true
	return
}

func Headers(filename string) (headers []string, err error) {
	var f *os.File
	f, err = os.Open(filename)
	if err != nil {
		return
	}
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		line := scanner.Text()
		if len(strings.Split(line, " ")) == 7 {
			headers = append(headers, line)
		}
	}
	return
}

// ImportPubring reads a YAMN Pubring.mix file
func (p *Pubring) ImportPubring() (err error) {
	var f *os.File
	f, err = os.Open(p.pubringFile)
	if err != nil {
		return
	}
	scanner := bufio.NewScanner(f)
	var elements []string
	var num_elements int
	var line string //Each line within Pubring.mix
	var rem *Remailer
	var pkdata []byte // Decoded Public key
	now := time.Now() // Current time for key validity testing
	key_phase := 0
	/* Key phases are:
	0	Expecting header line
	1	Expecting Begin cutmark
	2	Expecting Keyid line
	3	Expecting public key
	4	Got End cutmark
	*/

	for scanner.Scan() {
		line = scanner.Text()
		switch key_phase {
		case 0:
			// Expecting key header line
			elements = strings.Split(line, " ")
			num_elements = len(elements)
			// 7 elements indicates a remailer header line in Pubring.mix
			if num_elements != 7 {
				continue
			}
			from, err := time.Parse(date_format, elements[5])
			if err != nil {
				fmt.Fprintln(os.Stderr, "Malformed valid-from date")
				key_phase = 0
				continue
			}
			if now.Before(from) {
				fmt.Fprintln(os.Stderr, elements[0]+": Key not yet valid")
				key_phase = 0
				continue
			}
			until, err := time.Parse(date_format, elements[6])
			if err != nil {
				fmt.Fprintln(os.Stderr, "Malformed valid-to date")
				key_phase = 0
				continue
			}
			// Echolot likes to ping expired keys.  The useExpired
			// option dictates if expired keys should be included
			// as condidates.
			if !p.useExpired && now.After(until) {
				fmt.Fprintf(
					os.Stderr,
					"Key expired: Name=%s, Key=%s, Date=%s\n",
					elements[0],
					elements[2],
					elements[6],
				)
				key_phase = 0
				continue
			}
			rem = new(Remailer)
			rem.name = elements[0]
			rem.Keyid, err = hex.DecodeString(elements[2])
			if err != nil {
				// keyid is not valid hex
				fmt.Fprintln(os.Stderr, "Keyid in header is not hex")
				key_phase = 0
				continue
			}
			rem.version = elements[3]
			rem.caps = elements[4]
			rem.Address = elements[1]
			key_phase = 1
		case 1:
			// Expecting Begin cutmark
			if line == "-----Begin Mix Key-----" {
				key_phase = 2
			}
		case 2:
			// Expecting Keyid line
			keyid, err := hex.DecodeString(line)
			if err != nil {
				// keyid is not valid hex
				fmt.Fprintln(os.Stderr, "Keyid in pubkey is not hex")
				key_phase = 0
				continue
			}
			if !bytes.Equal(keyid, rem.Keyid) {
				// Corrupt keyblock - header keyid doesn't match keyid in block
				fmt.Fprintln(os.Stderr, "Keyid in header differs from keyid in pubkey")
				key_phase = 0
				continue
			}
			key_phase = 3
		case 3:
			// Expecting Public key
			pkdata, err = hex.DecodeString(line)
			if err != nil {
				// Public key is not valid hex
				fmt.Fprintln(os.Stderr, "Unable to decode Public key")
				key_phase = 0
				continue
			}
			if len(pkdata) != 32 {
				fmt.Fprintln(os.Stderr, "Public key is not 32 bits")
				key_phase = 0
				continue
			}
			rem.PK = pkdata
			key_phase = 4
		case 4:
			// Expecting end cutmark
			if line == "-----End Mix Key-----" {
				p.Put(*rem)
				key_phase = 0
			}
		} // End of phases
	} // End of file scan loop

	// Set key imported timestamp
	stat, err := os.Stat(p.pubringFile)
	if err != nil {
		panic(err)
	}
	p.keysImported = stat.ModTime()
	return
}
<|endoftext|>


<commit_before>package device

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/google/uuid"
	"github.com/pkg/errors"

	deviceConfig "github.com/lxc/lxd/lxd/device/config"
	pcidev "github.com/lxc/lxd/lxd/device/pci"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	"github.com/lxc/lxd/lxd/resources"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/logger"
)

type gpuMdev struct {
	deviceCommon
}

// Start is run when the device is added to the container.
func (d *gpuMdev) Start() (*deviceConfig.RunConfig, error) {
	err := d.validateEnvironment()
	if err != nil {
		return nil, err
	}

	err = d.createVirtualGPU()
	if err != nil {
		return nil, err
	}

	return d.startVM()
}

// Stop is run when the device is removed from the instance.
func (d *gpuMdev) Stop() (*deviceConfig.RunConfig, error) {
	runConf := deviceConfig.RunConfig{
		PostHooks: []func() error{d.postStop},
	}

	return &runConf, nil
}

// startVM detects the requested GPU devices and related virtual functions and rebinds them to the vfio-pci driver.
func (d *gpuMdev) startVM() (*deviceConfig.RunConfig, error) {
	runConf := deviceConfig.RunConfig{}
	gpus, err := resources.GetGPU()
	if err != nil {
		return nil, err
	}

	saveData := make(map[string]string)
	var pciAddress string

	for _, gpu := range gpus.Cards {
		// Skip any cards that don't match the vendorid, pci, productid or DRM ID settings (if specified).
		if (d.config["vendorid"] != "" && gpu.VendorID != d.config["vendorid"]) ||
			(d.config["pci"] != "" && gpu.PCIAddress != d.config["pci"]) ||
			(d.config["productid"] != "" && gpu.ProductID != d.config["productid"]) ||
			(d.config["id"] != "" && (gpu.DRM == nil || fmt.Sprintf("%d", gpu.DRM.ID) != d.config["id"])) {
			continue
		}

		if pciAddress != "" {
			return nil, fmt.Errorf("VMs cannot match multiple GPUs per device")
		}

		pciAddress = gpu.PCIAddress
	}

	if pciAddress == "" {
		return nil, fmt.Errorf("Failed to detect requested GPU device")
	}

	// Get PCI information about the GPU device.
	devicePath := filepath.Join("/sys/bus/pci/devices", pciAddress)
	pciDev, err := pcidev.ParseUeventFile(filepath.Join(devicePath, "uevent"))
	if err != nil {
		return nil, errors.Wrapf(err, "Failed to get PCI device info for GPU %q", pciAddress)
	}

	v := d.volatileGet()

	saveData["last_state.pci.slot.name"] = pciDev.SlotName
	saveData["last_state.pci.driver"] = pciDev.Driver
	saveData["vgpu.uuid"] = v["vgpu.uuid"]

	runConf.GPUDevice = append(runConf.GPUDevice,
		[]deviceConfig.RunConfigItem{
			{Key: "devName", Value: d.name},
			{Key: "pciSlotName", Value: saveData["last_state.pci.slot.name"]},
			{Key: "vgpu", Value: v["vgpu.uuid"]},
		}...)

	err = d.volatileSet(saveData)
	if err != nil {
		return nil, err
	}

	return &runConf, nil
}

// postStop is run after the device is removed from the instance.
func (d *gpuMdev) postStop() error {
	defer d.volatileSet(map[string]string{
		"last_state.pci.slot.name": "",
		"last_state.pci.driver":    "",
		"vgpu.uuid":                "",
	})

	v := d.volatileGet()

	if v["vgpu.uuid"] != "" {
		path := fmt.Sprintf("/sys/bus/mdev/devices/%s", v["vgpu.uuid"])

		if shared.PathExists(path) {
			err := ioutil.WriteFile(filepath.Join(path, "remove"), []byte("1\n"), 0200)
			if err != nil {
				logger.Debugf("Failed to remove vgpu %q", v["vgpu.uuid"])
			}
		}
	}

	return nil
}

// validateConfig checks the supplied config for correctness.
func (d *gpuMdev) validateConfig(instConf instance.ConfigReader) error {
	if !instanceSupported(instConf.Type(), instancetype.VM) {
		return ErrUnsupportedDevType
	}

	requiredFields := []string{
		"mdev",
	}

	optionalFields := []string{
		"vendorid",
		"productid",
		"id",
		"pci",
	}

	err := d.config.Validate(gpuValidationRules(requiredFields, optionalFields))
	if err != nil {
		return err
	}

	if d.config["pci"] != "" {
		for _, field := range []string{"id", "productid", "vendorid"} {
			if d.config[field] != "" {
				return fmt.Errorf(`Cannot use %q when when "pci" is set`, field)
			}
		}

		d.config["pci"] = pcidev.NormaliseAddress(d.config["pci"])
	}

	if d.config["id"] != "" {
		for _, field := range []string{"pci", "productid", "vendorid"} {
			if d.config[field] != "" {
				return fmt.Errorf(`Cannot use %q when when "id" is set`, field)
			}
		}
	}

	return nil
}

// validateEnvironment checks the runtime environment for correctness.
func (d *gpuMdev) validateEnvironment() error {
	if d.inst.Type() == instancetype.VM && shared.IsTrue(d.inst.ExpandedConfig()["migration.stateful"]) {
		return fmt.Errorf("GPU devices cannot be used when migration.stateful is enabled")
	}

	return validatePCIDevice(d.config["pci"])
}

func (d *gpuMdev) createVirtualGPU() error {
	gpus, err := resources.GetGPU()
	if err != nil {
		return err
	}

	for _, gpu := range gpus.Cards {
		// Skip any cards that don't match the vendorid, pci or productid settings (if specified).
		if (d.config["vendorid"] != "" && gpu.VendorID != d.config["vendorid"]) ||
			(d.config["pci"] != "" && gpu.PCIAddress != d.config["pci"]) ||
			(d.config["productid"] != "" && gpu.ProductID != d.config["productid"]) {
			continue
		}

		foundMdev := false

		for mdev := range gpu.Mdev {
			if d.config["mdev"] == mdev {
				foundMdev = true
				break
			}
		}

		if !foundMdev {
			return fmt.Errorf("Invalid mdev %q", d.config["mdev"])
		}

		// Check if the vgpu exists before creating it.
		v := d.volatileGet()

		if v["vgpu.uuid"] != "" && shared.PathExists(fmt.Sprintf("/sys/bus/pci/devices/%s/%s", gpu.PCIAddress, v["vgpu.uuid"])) {
			return nil
		}

		// Create the virtual gpu
		devUUID, err := uuid.NewUUID()
		if err != nil {
			return errors.Wrap(err, "Failed to generate UUID")
		}

		err = ioutil.WriteFile(filepath.Join(fmt.Sprintf("/sys/bus/pci/devices/%s/mdev_supported_types/%s/create", gpu.PCIAddress, d.config["mdev"])), []byte(devUUID.String()), 200)
		if err != nil {
			if os.IsNotExist(err) {
				return fmt.Errorf("The requested profile %q does not exist", d.config["mdev"])
			}

			return errors.Wrapf(err, "Failed to create virtual gpu %q", devUUID.String())
		}

		err = d.volatileSet(map[string]string{"vgpu.uuid": devUUID.String()})
		if err != nil {
			return err
		}

		break
	}

	return nil
}
<commit_msg>lxd/device/gpu_mdev: Simplify start logic<commit_after>package device

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/google/uuid"
	"github.com/pkg/errors"

	deviceConfig "github.com/lxc/lxd/lxd/device/config"
	pcidev "github.com/lxc/lxd/lxd/device/pci"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	"github.com/lxc/lxd/lxd/resources"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/logger"
)

type gpuMdev struct {
	deviceCommon
}

// Start is run when the device is added to the container.
func (d *gpuMdev) Start() (*deviceConfig.RunConfig, error) {
	err := d.validateEnvironment()
	if err != nil {
		return nil, err
	}

	return d.startVM()
}

// Stop is run when the device is removed from the instance.
func (d *gpuMdev) Stop() (*deviceConfig.RunConfig, error) {
	runConf := deviceConfig.RunConfig{
		PostHooks: []func() error{d.postStop},
	}

	return &runConf, nil
}

// startVM detects the requested GPU devices and related virtual functions and rebinds them to the vfio-pci driver.
func (d *gpuMdev) startVM() (*deviceConfig.RunConfig, error) {
	runConf := deviceConfig.RunConfig{}

	// Get any existing UUID.
	v := d.volatileGet()
	mdevUUID := v["vgpu.uuid"]

	// Get the local GPUs.
	gpus, err := resources.GetGPU()
	if err != nil {
		return nil, err
	}

	var pciAddress string
	for _, gpu := range gpus.Cards {
		// Skip any cards that don't match the vendorid, pci, productid or DRM ID settings (if specified).
		if (d.config["vendorid"] != "" && gpu.VendorID != d.config["vendorid"]) ||
			(d.config["pci"] != "" && gpu.PCIAddress != d.config["pci"]) ||
			(d.config["productid"] != "" && gpu.ProductID != d.config["productid"]) ||
			(d.config["id"] != "" && (gpu.DRM == nil || fmt.Sprintf("%d", gpu.DRM.ID) != d.config["id"])) {
			continue
		}

		if pciAddress != "" {
			return nil, fmt.Errorf("VMs cannot match multiple GPUs per device")
		}

		pciAddress = gpu.PCIAddress

		// Look for the requested mdev profile.
		foundMdev := false
		for mdev := range gpu.Mdev {
			if d.config["mdev"] == mdev {
				foundMdev = true
				break
			}
		}

		if !foundMdev {
			return nil, fmt.Errorf("Invalid mdev %q", d.config["mdev"])
		}

		// Create the vGPU.
		if mdevUUID == "" || !shared.PathExists(fmt.Sprintf("/sys/bus/pci/devices/%s/%s", gpu.PCIAddress, mdevUUID)) {
			devUUID, err := uuid.NewUUID()
			if err != nil {
				return nil, errors.Wrap(err, "Failed to generate UUID")
			}
			mdevUUID = devUUID.String()

			err = ioutil.WriteFile(filepath.Join(fmt.Sprintf("/sys/bus/pci/devices/%s/mdev_supported_types/%s/create", gpu.PCIAddress, d.config["mdev"])), []byte(mdevUUID), 200)
			if err != nil {
				if os.IsNotExist(err) {
					return nil, fmt.Errorf("The requested profile %q does not exist", d.config["mdev"])
				}

				return nil, errors.Wrapf(err, "Failed to create virtual gpu %q", mdevUUID)
			}
		}
	}

	if pciAddress == "" {
		return nil, fmt.Errorf("Failed to detect requested GPU device")
	}

	// Get PCI information about the GPU device.
	devicePath := filepath.Join("/sys/bus/pci/devices", pciAddress)
	pciDev, err := pcidev.ParseUeventFile(filepath.Join(devicePath, "uevent"))
	if err != nil {
		return nil, errors.Wrapf(err, "Failed to get PCI device info for GPU %q", pciAddress)
	}

	// Prepare the new volatile keys.
	saveData := make(map[string]string)
	saveData["last_state.pci.slot.name"] = pciDev.SlotName
	saveData["last_state.pci.driver"] = pciDev.Driver
	saveData["vgpu.uuid"] = mdevUUID

	runConf.GPUDevice = append(runConf.GPUDevice,
		[]deviceConfig.RunConfigItem{
			{Key: "devName", Value: d.name},
			{Key: "pciSlotName", Value: saveData["last_state.pci.slot.name"]},
			{Key: "vgpu", Value: mdevUUID},
		}...)

	err = d.volatileSet(saveData)
	if err != nil {
		return nil, err
	}

	return &runConf, nil
}

// postStop is run after the device is removed from the instance.
func (d *gpuMdev) postStop() error {
	defer d.volatileSet(map[string]string{
		"last_state.pci.slot.name": "",
		"last_state.pci.driver":    "",
		"vgpu.uuid":                "",
	})

	v := d.volatileGet()

	if v["vgpu.uuid"] != "" {
		path := fmt.Sprintf("/sys/bus/mdev/devices/%s", v["vgpu.uuid"])

		if shared.PathExists(path) {
			err := ioutil.WriteFile(filepath.Join(path, "remove"), []byte("1\n"), 0200)
			if err != nil {
				logger.Debugf("Failed to remove vgpu %q", v["vgpu.uuid"])
			}
		}
	}

	return nil
}

// validateConfig checks the supplied config for correctness.
func (d *gpuMdev) validateConfig(instConf instance.ConfigReader) error {
	if !instanceSupported(instConf.Type(), instancetype.VM) {
		return ErrUnsupportedDevType
	}

	requiredFields := []string{
		"mdev",
	}

	optionalFields := []string{
		"vendorid",
		"productid",
		"id",
		"pci",
	}

	err := d.config.Validate(gpuValidationRules(requiredFields, optionalFields))
	if err != nil {
		return err
	}

	if d.config["pci"] != "" {
		for _, field := range []string{"id", "productid", "vendorid"} {
			if d.config[field] != "" {
				return fmt.Errorf(`Cannot use %q when when "pci" is set`, field)
			}
		}

		d.config["pci"] = pcidev.NormaliseAddress(d.config["pci"])
	}

	if d.config["id"] != "" {
		for _, field := range []string{"pci", "productid", "vendorid"} {
			if d.config[field] != "" {
				return fmt.Errorf(`Cannot use %q when when "id" is set`, field)
			}
		}
	}

	return nil
}

// validateEnvironment checks the runtime environment for correctness.
func (d *gpuMdev) validateEnvironment() error {
	if d.inst.Type() == instancetype.VM && shared.IsTrue(d.inst.ExpandedConfig()["migration.stateful"]) {
		return fmt.Errorf("GPU devices cannot be used when migration.stateful is enabled")
	}

	return validatePCIDevice(d.config["pci"])
}
<|endoftext|>