<commit_before>package collection

import (
	"context"
	"testing"
	"time"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pkg/require"
	"github.com/pachyderm/pachyderm/src/client/pkg/uuid"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/watch"

	etcd "github.com/coreos/etcd/clientv3"
)

var (
	pipelineIndex    Index = Index{"Pipeline", false}
	inputsMultiIndex Index = Index{"Inputs", true}
)

func TestIndex(t *testing.T) {
	etcdClient, err := getEtcdClient()
	require.NoError(t, err)
	uuidPrefix := uuid.NewWithoutDashes()

	jobInfos := NewCollection(etcdClient, uuidPrefix, []Index{pipelineIndex}, &pps.JobInfo{})

	j1 := &pps.JobInfo{
		Job:      &pps.Job{"j1"},
		Pipeline: &pps.Pipeline{"p1"},
	}
	j2 := &pps.JobInfo{
		Job:      &pps.Job{"j2"},
		Pipeline: &pps.Pipeline{"p1"},
	}
	j3 := &pps.JobInfo{
		Job:      &pps.Job{"j3"},
		Pipeline: &pps.Pipeline{"p2"},
	}
	_, err = NewSTM(context.Background(), etcdClient, func(stm STM) error {
		jobInfos := jobInfos.ReadWrite(stm)
		jobInfos.Put(j1.Job.ID, j1)
		jobInfos.Put(j2.Job.ID, j2)
		jobInfos.Put(j3.Job.ID, j3)
		return nil
	})
	require.NoError(t, err)

	jobInfosReadonly := jobInfos.ReadOnly(context.Background())

	iter, err := jobInfosReadonly.GetByIndex(pipelineIndex, j1.Pipeline)
	require.NoError(t, err)
	var ID string
	job := new(pps.JobInfo)
	ok, err := iter.Next(&ID, job)
	require.NoError(t, err)
	require.True(t, ok)
	require.Equal(t, j1.Job.ID, ID)
	require.Equal(t, j1, job)
	ok, err = iter.Next(&ID, job)
	require.NoError(t, err)
	require.True(t, ok)
	require.Equal(t, j2.Job.ID, ID)
	require.Equal(t, j2, job)
	ok, err = iter.Next(&ID, job)
	require.NoError(t, err)
	require.False(t, ok)

	iter, err = jobInfosReadonly.GetByIndex(pipelineIndex, j3.Pipeline)
	require.NoError(t, err)
	ok, err = iter.Next(&ID, job)
	require.NoError(t, err)
	require.True(t, ok)
	require.Equal(t, j3.Job.ID, ID)
	require.Equal(t, j3, job)
	ok, err = iter.Next(&ID, job)
	require.NoError(t, err)
	require.False(t, ok)
}

func TestIndexWatch(t *testing.T) {
	etcdClient, err := getEtcdClient()
	require.NoError(t, err)
	uuidPrefix := uuid.NewWithoutDashes()

	jobInfos := NewCollection(etcdClient, uuidPrefix, []Index{pipelineIndex}, &pps.JobInfo{})

	j1 := &pps.JobInfo{
		Job:      &pps.Job{"j1"},
		Pipeline: &pps.Pipeline{"p1"},
	}
	_, err = NewSTM(context.Background(), etcdClient, func(stm STM) error {
		jobInfos := jobInfos.ReadWrite(stm)
		jobInfos.Put(j1.Job.ID, j1)
		return nil
	})
	require.NoError(t, err)

	jobInfosReadonly := jobInfos.ReadOnly(context.Background())

	watcher, err := jobInfosReadonly.WatchByIndex(pipelineIndex, j1.Pipeline.String())
	eventCh := watcher.Watch()
	require.NoError(t, err)
	var ID string
	job := new(pps.JobInfo)
	event := <-eventCh
	require.NoError(t, event.Err)
	require.Equal(t, event.Type, watch.EventPut)
	require.NoError(t, event.Unmarshal(&ID, job))
	require.Equal(t, j1.Job.ID, ID)
	require.Equal(t, j1, job)

	// Now we will put j1 again, unchanged.  We want to make sure
	// that we do not receive an event.
	_, err = NewSTM(context.Background(), etcdClient, func(stm STM) error {
		jobInfos := jobInfos.ReadWrite(stm)
		jobInfos.Put(j1.Job.ID, j1)
		return nil
	})

	select {
	case event := <-eventCh:
		t.Fatalf("should not have received an event %v", event)
	case <-time.After(2 * time.Second):
	}

	j2 := &pps.JobInfo{
		Job:      &pps.Job{"j2"},
		Pipeline: &pps.Pipeline{"p1"},
	}

	_, err = NewSTM(context.Background(), etcdClient, func(stm STM) error {
		jobInfos := jobInfos.ReadWrite(stm)
		jobInfos.Put(j2.Job.ID, j2)
		return nil
	})
	require.NoError(t, err)

	event = <-eventCh
	require.NoError(t, event.Err)
	require.Equal(t, event.Type, watch.EventPut)
	require.NoError(t, event.Unmarshal(&ID, job))
	require.Equal(t, j2.Job.ID, ID)
	require.Equal(t, j2, job)

	j1Prime := &pps.JobInfo{
		Job:      &pps.Job{"j1"},
		Pipeline: &pps.Pipeline{"p3"},
	}
	_, err = NewSTM(context.Background(), etcdClient, func(stm STM) error {
		jobInfos := jobInfos.ReadWrite(stm)
		jobInfos.Put(j1.Job.ID, j1Prime)
		return nil
	})
	require.NoError(t, err)

	event = <-eventCh
	require.NoError(t, event.Err)
	require.Equal(t, event.Type, watch.EventDelete)
	require.NoError(t, event.Unmarshal(&ID, job))
	require.Equal(t, j1.Job.ID, ID)

	_, err = NewSTM(context.Background(), etcdClient, func(stm STM) error {
		jobInfos := jobInfos.ReadWrite(stm)
		jobInfos.Delete(j2.Job.ID)
		return nil
	})
	require.NoError(t, err)

	event = <-eventCh
	require.NoError(t, event.Err)
	require.Equal(t, event.Type, watch.EventDelete)
	require.NoError(t, event.Unmarshal(&ID, job))
	require.Equal(t, j2.Job.ID, ID)
}

func TestMultiIndex(t *testing.T) {
	etcdClient, err := getEtcdClient()
	require.NoError(t, err)
	uuidPrefix := uuid.NewWithoutDashes()

	jobInfos := NewCollection(etcdClient, uuidPrefix, []Index{inputsMultiIndex}, &pps.JobInfo{})

	j1 := &pps.JobInfo{
		Job: &pps.Job{"j1"},
		Inputs: []*pps.JobInput{
			{Name: "input1"},
			{Name: "input2"},
			{Name: "input3"},
		},
	}
	j2 := &pps.JobInfo{
		Job: &pps.Job{"j2"},
		Inputs: []*pps.JobInput{
			{Name: "input1"},
			{Name: "input2"},
			{Name: "input3"},
		},
	}
	_, err = NewSTM(context.Background(), etcdClient, func(stm STM) error {
		jobInfos := jobInfos.ReadWrite(stm)
		jobInfos.Put(j1.Job.ID, j1)
		jobInfos.Put(j2.Job.ID, j2)
		return nil
	})
	require.NoError(t, err)

	jobInfosReadonly := jobInfos.ReadOnly(context.Background())

	iter, err := jobInfosReadonly.GetByIndex(inputsMultiIndex, &pps.JobInput{
		Name: "input1",
	})
	require.NoError(t, err)
	var ID string
	job := new(pps.JobInfo)
	ok, err := iter.Next(&ID, job)
	require.NoError(t, err)
	require.True(t, ok)
	require.Equal(t, j1.Job.ID, ID)
	require.Equal(t, j1, job)
	ok, err = iter.Next(&ID, job)
	require.NoError(t, err)
	require.True(t, ok)
	require.Equal(t, j2.Job.ID, ID)
	require.Equal(t, j2, job)

	iter, err = jobInfosReadonly.GetByIndex(inputsMultiIndex, &pps.JobInput{
		Name: "input2",
	})
	require.NoError(t, err)
	ok, err = iter.Next(&ID, job)
	require.NoError(t, err)
	require.True(t, ok)
	require.Equal(t, j1.Job.ID, ID)
	require.Equal(t, j1, job)
	ok, err = iter.Next(&ID, job)
	require.NoError(t, err)
	require.True(t, ok)
	require.Equal(t, j2.Job.ID, ID)
	require.Equal(t, j2, job)

	j1.Inputs[2] = &pps.JobInput{Name: "input4"}
	_, err = NewSTM(context.Background(), etcdClient, func(stm STM) error {
		jobInfos := jobInfos.ReadWrite(stm)
		jobInfos.Put(j1.Job.ID, j1)
		return nil
	})
	require.NoError(t, err)

	iter, err = jobInfosReadonly.GetByIndex(inputsMultiIndex, &pps.JobInput{
		Name: "input3",
	})
	require.NoError(t, err)
	ok, err = iter.Next(&ID, job)
	require.NoError(t, err)
	require.True(t, ok)
	require.Equal(t, j2.Job.ID, ID)
	require.Equal(t, j2, job)

	iter, err = jobInfosReadonly.GetByIndex(inputsMultiIndex, &pps.JobInput{
		Name: "input4",
	})
	require.NoError(t, err)
	ok, err = iter.Next(&ID, job)
	require.NoError(t, err)
	require.True(t, ok)
	require.Equal(t, j1.Job.ID, ID)
	require.Equal(t, j1, job)

	_, err = NewSTM(context.Background(), etcdClient, func(stm STM) error {
		jobInfos := jobInfos.ReadWrite(stm)
		jobInfos.Delete(j1.Job.ID)
		return nil
	})

	iter, err = jobInfosReadonly.GetByIndex(inputsMultiIndex, &pps.JobInput{
		Name: "input1",
	})
	require.NoError(t, err)
	ok, err = iter.Next(&ID, job)
	require.NoError(t, err)
	require.True(t, ok)
	require.Equal(t, j2.Job.ID, ID)
	require.Equal(t, j2, job)
}

func getEtcdClient() (*etcd.Client, error) {
	etcdClient, err := etcd.New(etcd.Config{
		Endpoints:   []string{"localhost:2379"},
		DialOptions: client.EtcdDialOptions(),
	})
	if err != nil {
		return nil, err
	}
	return etcdClient, nil
}
<commit_msg>Fix collection tests<commit_after>package collection

import (
	"context"
	"testing"
	"time"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pkg/require"
	"github.com/pachyderm/pachyderm/src/client/pkg/uuid"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/watch"

	etcd "github.com/coreos/etcd/clientv3"
)

var (
	pipelineIndex    Index = Index{"Pipeline", false}
	inputsMultiIndex Index = Index{"Inputs", true}
)

func TestIndex(t *testing.T) {
	etcdClient, err := getEtcdClient()
	require.NoError(t, err)
	uuidPrefix := uuid.NewWithoutDashes()

	jobInfos := NewCollection(etcdClient, uuidPrefix, []Index{pipelineIndex}, &pps.JobInfo{}, nil)

	j1 := &pps.JobInfo{
		Job:      &pps.Job{"j1"},
		Pipeline: &pps.Pipeline{"p1"},
	}
	j2 := &pps.JobInfo{
		Job:      &pps.Job{"j2"},
		Pipeline: &pps.Pipeline{"p1"},
	}
	j3 := &pps.JobInfo{
		Job:      &pps.Job{"j3"},
		Pipeline: &pps.Pipeline{"p2"},
	}
	_, err = NewSTM(context.Background(), etcdClient, func(stm STM) error {
		jobInfos := jobInfos.ReadWrite(stm)
		jobInfos.Put(j1.Job.ID, j1)
		jobInfos.Put(j2.Job.ID, j2)
		jobInfos.Put(j3.Job.ID, j3)
		return nil
	})
	require.NoError(t, err)

	jobInfosReadonly := jobInfos.ReadOnly(context.Background())

	iter, err := jobInfosReadonly.GetByIndex(pipelineIndex, j1.Pipeline)
	require.NoError(t, err)
	var ID string
	job := new(pps.JobInfo)
	ok, err := iter.Next(&ID, job)
	require.NoError(t, err)
	require.True(t, ok)
	require.Equal(t, j1.Job.ID, ID)
	require.Equal(t, j1, job)
	ok, err = iter.Next(&ID, job)
	require.NoError(t, err)
	require.True(t, ok)
	require.Equal(t, j2.Job.ID, ID)
	require.Equal(t, j2, job)
	ok, err = iter.Next(&ID, job)
	require.NoError(t, err)
	require.False(t, ok)

	iter, err = jobInfosReadonly.GetByIndex(pipelineIndex, j3.Pipeline)
	require.NoError(t, err)
	ok, err = iter.Next(&ID, job)
	require.NoError(t, err)
	require.True(t, ok)
	require.Equal(t, j3.Job.ID, ID)
	require.Equal(t, j3, job)
	ok, err = iter.Next(&ID, job)
	require.NoError(t, err)
	require.False(t, ok)
}

func TestIndexWatch(t *testing.T) {
	etcdClient, err := getEtcdClient()
	require.NoError(t, err)
	uuidPrefix := uuid.NewWithoutDashes()

	jobInfos := NewCollection(etcdClient, uuidPrefix, []Index{pipelineIndex}, &pps.JobInfo{}, nil)

	j1 := &pps.JobInfo{
		Job:      &pps.Job{"j1"},
		Pipeline: &pps.Pipeline{"p1"},
	}
	_, err = NewSTM(context.Background(), etcdClient, func(stm STM) error {
		jobInfos := jobInfos.ReadWrite(stm)
		jobInfos.Put(j1.Job.ID, j1)
		return nil
	})
	require.NoError(t, err)

	jobInfosReadonly := jobInfos.ReadOnly(context.Background())

	watcher, err := jobInfosReadonly.WatchByIndex(pipelineIndex, j1.Pipeline.String())
	eventCh := watcher.Watch()
	require.NoError(t, err)
	var ID string
	job := new(pps.JobInfo)
	event := <-eventCh
	require.NoError(t, event.Err)
	require.Equal(t, event.Type, watch.EventPut)
	require.NoError(t, event.Unmarshal(&ID, job))
	require.Equal(t, j1.Job.ID, ID)
	require.Equal(t, j1, job)

	// Now we will put j1 again, unchanged.  We want to make sure
	// that we do not receive an event.
	_, err = NewSTM(context.Background(), etcdClient, func(stm STM) error {
		jobInfos := jobInfos.ReadWrite(stm)
		jobInfos.Put(j1.Job.ID, j1)
		return nil
	})

	select {
	case event := <-eventCh:
		t.Fatalf("should not have received an event %v", event)
	case <-time.After(2 * time.Second):
	}

	j2 := &pps.JobInfo{
		Job:      &pps.Job{"j2"},
		Pipeline: &pps.Pipeline{"p1"},
	}

	_, err = NewSTM(context.Background(), etcdClient, func(stm STM) error {
		jobInfos := jobInfos.ReadWrite(stm)
		jobInfos.Put(j2.Job.ID, j2)
		return nil
	})
	require.NoError(t, err)

	event = <-eventCh
	require.NoError(t, event.Err)
	require.Equal(t, event.Type, watch.EventPut)
	require.NoError(t, event.Unmarshal(&ID, job))
	require.Equal(t, j2.Job.ID, ID)
	require.Equal(t, j2, job)

	j1Prime := &pps.JobInfo{
		Job:      &pps.Job{"j1"},
		Pipeline: &pps.Pipeline{"p3"},
	}
	_, err = NewSTM(context.Background(), etcdClient, func(stm STM) error {
		jobInfos := jobInfos.ReadWrite(stm)
		jobInfos.Put(j1.Job.ID, j1Prime)
		return nil
	})
	require.NoError(t, err)

	event = <-eventCh
	require.NoError(t, event.Err)
	require.Equal(t, event.Type, watch.EventDelete)
	require.NoError(t, event.Unmarshal(&ID, job))
	require.Equal(t, j1.Job.ID, ID)

	_, err = NewSTM(context.Background(), etcdClient, func(stm STM) error {
		jobInfos := jobInfos.ReadWrite(stm)
		jobInfos.Delete(j2.Job.ID)
		return nil
	})
	require.NoError(t, err)

	event = <-eventCh
	require.NoError(t, event.Err)
	require.Equal(t, event.Type, watch.EventDelete)
	require.NoError(t, event.Unmarshal(&ID, job))
	require.Equal(t, j2.Job.ID, ID)
}

func TestMultiIndex(t *testing.T) {
	etcdClient, err := getEtcdClient()
	require.NoError(t, err)
	uuidPrefix := uuid.NewWithoutDashes()

	jobInfos := NewCollection(etcdClient, uuidPrefix, []Index{inputsMultiIndex}, &pps.JobInfo{}, nil)

	j1 := &pps.JobInfo{
		Job: &pps.Job{"j1"},
		Inputs: []*pps.JobInput{
			{Name: "input1"},
			{Name: "input2"},
			{Name: "input3"},
		},
	}
	j2 := &pps.JobInfo{
		Job: &pps.Job{"j2"},
		Inputs: []*pps.JobInput{
			{Name: "input1"},
			{Name: "input2"},
			{Name: "input3"},
		},
	}
	_, err = NewSTM(context.Background(), etcdClient, func(stm STM) error {
		jobInfos := jobInfos.ReadWrite(stm)
		jobInfos.Put(j1.Job.ID, j1)
		jobInfos.Put(j2.Job.ID, j2)
		return nil
	})
	require.NoError(t, err)

	jobInfosReadonly := jobInfos.ReadOnly(context.Background())

	iter, err := jobInfosReadonly.GetByIndex(inputsMultiIndex, &pps.JobInput{
		Name: "input1",
	})
	require.NoError(t, err)
	var ID string
	job := new(pps.JobInfo)
	ok, err := iter.Next(&ID, job)
	require.NoError(t, err)
	require.True(t, ok)
	require.Equal(t, j1.Job.ID, ID)
	require.Equal(t, j1, job)
	job = new(pps.JobInfo)
	ok, err = iter.Next(&ID, job)
	require.NoError(t, err)
	require.True(t, ok)
	require.Equal(t, j2.Job.ID, ID)
	require.Equal(t, j2, job)

	iter, err = jobInfosReadonly.GetByIndex(inputsMultiIndex, &pps.JobInput{
		Name: "input2",
	})
	require.NoError(t, err)
	job = new(pps.JobInfo)
	ok, err = iter.Next(&ID, job)
	require.NoError(t, err)
	require.True(t, ok)
	require.Equal(t, j1.Job.ID, ID)
	require.Equal(t, j1, job)
	job = new(pps.JobInfo)
	ok, err = iter.Next(&ID, job)
	require.NoError(t, err)
	require.True(t, ok)
	require.Equal(t, j2.Job.ID, ID)
	require.Equal(t, j2, job)

	j1.Inputs[2] = &pps.JobInput{Name: "input4"}
	_, err = NewSTM(context.Background(), etcdClient, func(stm STM) error {
		jobInfos := jobInfos.ReadWrite(stm)
		jobInfos.Put(j1.Job.ID, j1)
		return nil
	})
	require.NoError(t, err)

	iter, err = jobInfosReadonly.GetByIndex(inputsMultiIndex, &pps.JobInput{
		Name: "input3",
	})
	require.NoError(t, err)
	job = new(pps.JobInfo)
	ok, err = iter.Next(&ID, job)
	require.NoError(t, err)
	require.True(t, ok)
	require.Equal(t, j2.Job.ID, ID)
	require.Equal(t, j2, job)

	iter, err = jobInfosReadonly.GetByIndex(inputsMultiIndex, &pps.JobInput{
		Name: "input4",
	})
	require.NoError(t, err)
	job = new(pps.JobInfo)
	ok, err = iter.Next(&ID, job)
	require.NoError(t, err)
	require.True(t, ok)
	require.Equal(t, j1.Job.ID, ID)
	require.Equal(t, j1, job)

	_, err = NewSTM(context.Background(), etcdClient, func(stm STM) error {
		jobInfos := jobInfos.ReadWrite(stm)
		jobInfos.Delete(j1.Job.ID)
		return nil
	})

	iter, err = jobInfosReadonly.GetByIndex(inputsMultiIndex, &pps.JobInput{
		Name: "input1",
	})
	require.NoError(t, err)
	job = new(pps.JobInfo)
	ok, err = iter.Next(&ID, job)
	require.NoError(t, err)
	require.True(t, ok)
	require.Equal(t, j2.Job.ID, ID)
	require.Equal(t, j2, job)
}

func getEtcdClient() (*etcd.Client, error) {
	etcdClient, err := etcd.New(etcd.Config{
		Endpoints:   []string{"localhost:32379"},
		DialOptions: client.EtcdDialOptions(),
	})
	if err != nil {
		return nil, err
	}
	return etcdClient, nil
}
<|endoftext|>


<commit_before>// Copyright 2013 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package web

import (
	"code.google.com/p/gorest"
	"flag"
	"github.com/prometheus/client_golang"
	"github.com/prometheus/prometheus/appstate"
	"github.com/prometheus/prometheus/web/api"
	"github.com/prometheus/prometheus/web/blob"
	"html/template"
	"log"
	"net/http"
	_ "net/http/pprof"
)

// Commandline flags.
var (
	listenAddress  = flag.String("listenAddress", ":9090", "Address to listen on for web interface.")
	useLocalAssets = flag.Bool("useLocalAssets", false, "Read assets/templates from file instead of binary.")
)

func StartServing(appState *appstate.ApplicationState) {
	gorest.RegisterService(api.NewMetricsService(appState))

	http.Handle("/", &StatusHandler{appState: appState})
	http.HandleFunc("/graph", graphHandler)
	http.HandleFunc("/console", consoleHandler)

	http.Handle("/api/", gorest.Handle())
	http.Handle("/metrics.json", registry.DefaultRegistry.Handler())
	if *useLocalAssets {
		http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("web/static"))))
	} else {
		http.Handle("/static/", http.StripPrefix("/static/", new(blob.Handler)))
	}

	go http.ListenAndServe(*listenAddress, nil)
}

func getTemplate(name string) (t *template.Template, err error) {
	if *useLocalAssets {
		return template.ParseFiles("web/templates/_base.html", "web/templates/"+name+".html")
	}

	t = template.New("_base")

	file, err := blob.GetFile(blob.TemplateFiles, "_base.html")
	if err != nil {
		log.Printf("Could not read base template: %s", err)
		return nil, err
	}
	t.Parse(string(file))

	file, err = blob.GetFile(blob.TemplateFiles, name+".html")
	if err != nil {
		log.Printf("Could not read %s template: %s", name, err)
		return nil, err
	}
	t.Parse(string(file))

	return
}

func executeTemplate(w http.ResponseWriter, name string, data interface{}) {
	tpl, err := getTemplate(name)
	if err != nil {
		log.Printf("Error preparing layout template: %s", err)
		return
	}
	tpl.Execute(w, data)
}
<commit_msg>Use client_golang.exp for automatic telemetry.<commit_after>// Copyright 2013 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package web

import (
	"code.google.com/p/gorest"
	"flag"
	"github.com/prometheus/client_golang"
	"github.com/prometheus/client_golang/exp"
	"github.com/prometheus/prometheus/appstate"
	"github.com/prometheus/prometheus/web/api"
	"github.com/prometheus/prometheus/web/blob"
	"html/template"
	"log"
	"net/http"
	_ "net/http/pprof"
)

// Commandline flags.
var (
	listenAddress  = flag.String("listenAddress", ":9090", "Address to listen on for web interface.")
	useLocalAssets = flag.Bool("useLocalAssets", false, "Read assets/templates from file instead of binary.")
)

func StartServing(appState *appstate.ApplicationState) {
	gorest.RegisterService(api.NewMetricsService(appState))

	exp.Handle("/", &StatusHandler{appState: appState})
	exp.HandleFunc("/graph", graphHandler)
	exp.HandleFunc("/console", consoleHandler)

	exp.Handle("/api/", gorest.Handle())
	exp.Handle("/metrics.json", registry.DefaultHandler)
	if *useLocalAssets {
		exp.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("web/static"))))
	} else {
		exp.Handle("/static/", http.StripPrefix("/static/", new(blob.Handler)))
	}

	go http.ListenAndServe(*listenAddress, exp.DefaultCoarseMux)
}

func getTemplate(name string) (t *template.Template, err error) {
	if *useLocalAssets {
		return template.ParseFiles("web/templates/_base.html", "web/templates/"+name+".html")
	}

	t = template.New("_base")

	file, err := blob.GetFile(blob.TemplateFiles, "_base.html")
	if err != nil {
		log.Printf("Could not read base template: %s", err)
		return nil, err
	}
	t.Parse(string(file))

	file, err = blob.GetFile(blob.TemplateFiles, name+".html")
	if err != nil {
		log.Printf("Could not read %s template: %s", name, err)
		return nil, err
	}
	t.Parse(string(file))

	return
}

func executeTemplate(w http.ResponseWriter, name string, data interface{}) {
	tpl, err := getTemplate(name)
	if err != nil {
		log.Printf("Error preparing layout template: %s", err)
		return
	}
	tpl.Execute(w, data)
}
<|endoftext|>


<commit_before>// +build linux

package sandbox

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"sync"

	nspkg "github.com/containernetworking/plugins/pkg/ns"
	"github.com/containers/storage/pkg/idtools"
	"github.com/cri-o/cri-o/pkg/config"
	"github.com/google/uuid"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"golang.org/x/sys/unix"
)

// Namespace handles data pertaining to a namespace
type Namespace struct {
	sync.Mutex
	ns          NS
	closed      bool
	initialized bool
	nsType      NSType
	nsPath      string
}

// NS is a wrapper for the containernetworking plugin's NetNS interface
// It exists because while NetNS is specifically called such, it is really a generic
// namespace, and can be used for other namespaces
type NS interface {
	nspkg.NetNS
}

// Get returns the Namespace for a given NsIface
func (n *Namespace) Get() *Namespace {
	return n
}

// Initialized returns true if the Namespace is already initialized
func (n *Namespace) Initialized() bool {
	return n.initialized
}

// Initialize does the necessary setup for a Namespace
// It does not do the bind mounting and nspinning
func (n *Namespace) Initialize() NamespaceIface {
	n.closed = false
	n.initialized = true
	return n
}

func getMappingsForPinns(mappings []idtools.IDMap) string {
	g := new(bytes.Buffer)
	for _, m := range mappings {
		fmt.Fprintf(g, "%d-%d-%d@", m.ContainerID, m.HostID, m.Size)
	}
	return g.String()
}

// Creates a new persistent namespace and returns an object
// representing that namespace, without switching to it
func pinNamespaces(nsTypes []NSType, cfg *config.Config, idMappings *idtools.IDMappings) ([]NamespaceIface, error) {
	typeToArg := map[NSType]string{
		IPCNS:  "-i",
		UTSNS:  "-u",
		USERNS: "-U",
		NETNS:  "-n",
	}

	pinnedNamespace := uuid.New().String()
	pinnsArgs := []string{
		"-d", cfg.NamespacesDir,
		"-f", pinnedNamespace,
	}
	type namespaceInfo struct {
		path   string
		nsType NSType
	}

	mountedNamespaces := make([]namespaceInfo, 0, len(nsTypes))

	var rootPair idtools.IDPair
	if idMappings != nil {
		rootPair = idMappings.RootPair()
	}

	for _, nsType := range nsTypes {
		arg, ok := typeToArg[nsType]
		if !ok {
			return nil, errors.Errorf("Invalid namespace type: %s", nsType)
		}
		pinnsArgs = append(pinnsArgs, arg)
		pinPath := filepath.Join(cfg.NamespacesDir, fmt.Sprintf("%sns", string(nsType)), pinnedNamespace)
		mountedNamespaces = append(mountedNamespaces, namespaceInfo{
			path:   pinPath,
			nsType: nsType,
		})
		if idMappings != nil {
			err := os.MkdirAll(filepath.Dir(pinPath), 0o755)
			if err != nil {
				return nil, err
			}
			f, err := os.Create(pinPath)
			if err != nil {
				return nil, err
			}
			f.Close()
			if err := os.Chown(pinPath, rootPair.UID, rootPair.GID); err != nil {
				return nil, err
			}
		}
	}

	if idMappings != nil {
		pinnsArgs = append(pinnsArgs,
			fmt.Sprintf("--uid-mapping=%s", getMappingsForPinns(idMappings.UIDs())),
			fmt.Sprintf("--gid-mapping=%s", getMappingsForPinns(idMappings.GIDs())))
	}

	pinns := cfg.PinnsPath

	logrus.Debugf("calling pinns with %v", pinnsArgs)
	output, err := exec.Command(pinns, pinnsArgs...).Output()
	if len(output) != 0 {
		logrus.Debugf("pinns output: %s", string(output))
	}
	if err != nil {
		// cleanup after ourselves
		failedUmounts := make([]string, 0)
		for _, info := range mountedNamespaces {
			if unmountErr := unix.Unmount(info.path, unix.MNT_DETACH); unmountErr != nil {
				failedUmounts = append(failedUmounts, info.path)
			}
		}
		if len(failedUmounts) != 0 {
			return nil, fmt.Errorf("failed to cleanup %v after pinns failure %s %v", failedUmounts, output, err)
		}
		return nil, fmt.Errorf("failed to pin namespaces %v: %s %v", nsTypes, output, err)
	}

	returnedNamespaces := make([]NamespaceIface, 0)
	for _, info := range mountedNamespaces {
		ret, err := nspkg.GetNS(info.path)
		if err != nil {
			return nil, err
		}

		returnedNamespaces = append(returnedNamespaces, &Namespace{
			ns:     ret.(NS),
			nsType: info.nsType,
			nsPath: info.path,
		})
	}
	return returnedNamespaces, nil
}

// getNamespace takes a path, checks if it is a namespace, and if so
// returns a Namespace
func getNamespace(nsPath string) (*Namespace, error) {
	if err := nspkg.IsNSorErr(nsPath); err != nil {
		return nil, err
	}

	ns, err := nspkg.GetNS(nsPath)
	if err != nil {
		return nil, err
	}

	return &Namespace{ns: ns, closed: false, nsPath: nsPath}, nil
}

// Path returns the path of the namespace handle
func (n *Namespace) Path() string {
	if n == nil || n.ns == nil {
		return ""
	}
	return n.nsPath
}

// Type returns which namespace this structure represents
func (n *Namespace) Type() NSType {
	return n.nsType
}

// Close closes this namespace
func (n *Namespace) Close() error {
	if n == nil || n.ns == nil {
		return nil
	}
	return n.ns.Close()
}

// Remove ensures this namespace handle is closed and removed
func (n *Namespace) Remove() error {
	n.Lock()
	defer n.Unlock()

	if n.closed {
		// nsRemove() can be called multiple
		// times without returning an error.
		return nil
	}

	if err := n.Close(); err != nil {
		return err
	}

	n.closed = true

	fp := n.Path()
	if fp == "" {
		return nil
	}

	// try to unmount, ignoring "not mounted" (EINVAL) error
	if err := unix.Unmount(fp, unix.MNT_DETACH); err != nil && err != unix.EINVAL {
		return errors.Wrapf(err, "unable to unmount %s", fp)
	}
	return os.RemoveAll(fp)
}
<commit_msg>pinNamespaces: use string concat instead of fmt.Sprintf<commit_after>// +build linux

package sandbox

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"sync"

	nspkg "github.com/containernetworking/plugins/pkg/ns"
	"github.com/containers/storage/pkg/idtools"
	"github.com/cri-o/cri-o/pkg/config"
	"github.com/google/uuid"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"golang.org/x/sys/unix"
)

// Namespace handles data pertaining to a namespace
type Namespace struct {
	sync.Mutex
	ns          NS
	closed      bool
	initialized bool
	nsType      NSType
	nsPath      string
}

// NS is a wrapper for the containernetworking plugin's NetNS interface
// It exists because while NetNS is specifically called such, it is really a generic
// namespace, and can be used for other namespaces
type NS interface {
	nspkg.NetNS
}

// Get returns the Namespace for a given NsIface
func (n *Namespace) Get() *Namespace {
	return n
}

// Initialized returns true if the Namespace is already initialized
func (n *Namespace) Initialized() bool {
	return n.initialized
}

// Initialize does the necessary setup for a Namespace
// It does not do the bind mounting and nspinning
func (n *Namespace) Initialize() NamespaceIface {
	n.closed = false
	n.initialized = true
	return n
}

func getMappingsForPinns(mappings []idtools.IDMap) string {
	g := new(bytes.Buffer)
	for _, m := range mappings {
		fmt.Fprintf(g, "%d-%d-%d@", m.ContainerID, m.HostID, m.Size)
	}
	return g.String()
}

// Creates a new persistent namespace and returns an object
// representing that namespace, without switching to it
func pinNamespaces(nsTypes []NSType, cfg *config.Config, idMappings *idtools.IDMappings) ([]NamespaceIface, error) {
	typeToArg := map[NSType]string{
		IPCNS:  "-i",
		UTSNS:  "-u",
		USERNS: "-U",
		NETNS:  "-n",
	}

	pinnedNamespace := uuid.New().String()
	pinnsArgs := []string{
		"-d", cfg.NamespacesDir,
		"-f", pinnedNamespace,
	}
	type namespaceInfo struct {
		path   string
		nsType NSType
	}

	mountedNamespaces := make([]namespaceInfo, 0, len(nsTypes))

	var rootPair idtools.IDPair
	if idMappings != nil {
		rootPair = idMappings.RootPair()
	}

	for _, nsType := range nsTypes {
		arg, ok := typeToArg[nsType]
		if !ok {
			return nil, errors.Errorf("Invalid namespace type: %s", nsType)
		}
		pinnsArgs = append(pinnsArgs, arg)
		pinPath := filepath.Join(cfg.NamespacesDir, string(nsType)+"ns", pinnedNamespace)
		mountedNamespaces = append(mountedNamespaces, namespaceInfo{
			path:   pinPath,
			nsType: nsType,
		})
		if idMappings != nil {
			err := os.MkdirAll(filepath.Dir(pinPath), 0o755)
			if err != nil {
				return nil, err
			}
			f, err := os.Create(pinPath)
			if err != nil {
				return nil, err
			}
			f.Close()
			if err := os.Chown(pinPath, rootPair.UID, rootPair.GID); err != nil {
				return nil, err
			}
		}
	}

	if idMappings != nil {
		pinnsArgs = append(pinnsArgs,
			fmt.Sprintf("--uid-mapping=%s", getMappingsForPinns(idMappings.UIDs())),
			fmt.Sprintf("--gid-mapping=%s", getMappingsForPinns(idMappings.GIDs())))
	}

	pinns := cfg.PinnsPath

	logrus.Debugf("calling pinns with %v", pinnsArgs)
	output, err := exec.Command(pinns, pinnsArgs...).Output()
	if len(output) != 0 {
		logrus.Debugf("pinns output: %s", string(output))
	}
	if err != nil {
		// cleanup after ourselves
		failedUmounts := make([]string, 0)
		for _, info := range mountedNamespaces {
			if unmountErr := unix.Unmount(info.path, unix.MNT_DETACH); unmountErr != nil {
				failedUmounts = append(failedUmounts, info.path)
			}
		}
		if len(failedUmounts) != 0 {
			return nil, fmt.Errorf("failed to cleanup %v after pinns failure %s %v", failedUmounts, output, err)
		}
		return nil, fmt.Errorf("failed to pin namespaces %v: %s %v", nsTypes, output, err)
	}

	returnedNamespaces := make([]NamespaceIface, 0)
	for _, info := range mountedNamespaces {
		ret, err := nspkg.GetNS(info.path)
		if err != nil {
			return nil, err
		}

		returnedNamespaces = append(returnedNamespaces, &Namespace{
			ns:     ret.(NS),
			nsType: info.nsType,
			nsPath: info.path,
		})
	}
	return returnedNamespaces, nil
}

// getNamespace takes a path, checks if it is a namespace, and if so
// returns a Namespace
func getNamespace(nsPath string) (*Namespace, error) {
	if err := nspkg.IsNSorErr(nsPath); err != nil {
		return nil, err
	}

	ns, err := nspkg.GetNS(nsPath)
	if err != nil {
		return nil, err
	}

	return &Namespace{ns: ns, closed: false, nsPath: nsPath}, nil
}

// Path returns the path of the namespace handle
func (n *Namespace) Path() string {
	if n == nil || n.ns == nil {
		return ""
	}
	return n.nsPath
}

// Type returns which namespace this structure represents
func (n *Namespace) Type() NSType {
	return n.nsType
}

// Close closes this namespace
func (n *Namespace) Close() error {
	if n == nil || n.ns == nil {
		return nil
	}
	return n.ns.Close()
}

// Remove ensures this namespace handle is closed and removed
func (n *Namespace) Remove() error {
	n.Lock()
	defer n.Unlock()

	if n.closed {
		// nsRemove() can be called multiple
		// times without returning an error.
		return nil
	}

	if err := n.Close(); err != nil {
		return err
	}

	n.closed = true

	fp := n.Path()
	if fp == "" {
		return nil
	}

	// try to unmount, ignoring "not mounted" (EINVAL) error
	if err := unix.Unmount(fp, unix.MNT_DETACH); err != nil && err != unix.EINVAL {
		return errors.Wrapf(err, "unable to unmount %s", fp)
	}
	return os.RemoveAll(fp)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os/exec"
	"strings"
	"time"

	"github.com/adnanh/webhook/helpers"
	"github.com/adnanh/webhook/hooks"

	"github.com/go-martini/martini"

	l4g "code.google.com/p/log4go"
)

const (
	version string = "1.0.3"
)

var (
	webhooks      *hooks.Hooks
	appStart      time.Time
	ip            = flag.String("ip", "", "ip the webhook server should listen on")
	port          = flag.Int("port", 9000, "port the webhook server should listen on")
	hooksFilename = flag.String("hooks", "hooks.json", "path to the json file containing defined hooks the webhook should serve")
	logFilename   = flag.String("log", "webhook.log", "path to the log file")
)

func init() {
	flag.Parse()

	fileLogWriter := l4g.NewFileLogWriter(*logFilename, false)
	fileLogWriter.SetRotateDaily(false)

	martini.Env = "production"

	l4g.AddFilter("file", l4g.FINE, fileLogWriter)
}

func main() {
	appStart = time.Now()
	var e error

	webhooks, e = hooks.New(*hooksFilename)

	if e != nil {
		l4g.Warn("Error occurred while loading hooks from %s: %s", *hooksFilename, e)
	}

	web := martini.Classic()

	web.Get("/", rootHandler)
	web.Get("/hook/:id", hookHandler)
	web.Post("/hook/:id", hookHandler)

	l4g.Info("Starting webhook %s with %d hook(s) on %s:%d", version, webhooks.Count(), *ip, *port)

	web.RunOnAddr(fmt.Sprintf("%s:%d", *ip, *port))
}

func rootHandler() string {
	return fmt.Sprintf("webhook %s running for %s serving %d hook(s)\n", version, time.Since(appStart).String(), webhooks.Count())
}

func jsonHandler(id string, body []byte, signature string, payload interface{}) {
	if hook := webhooks.Match(id, payload); hook != nil {
		if hook.Secret != "" {
			if signature == "" {
				l4g.Error("Hook %s got matched and contains the secret, but the request didn't contain any signature.", hook.ID)
				return
			}

			if expectedMAC, ok := helpers.CheckPayloadSignature(body, hook.Secret, signature); ok {
				l4g.Error("Hook %s got matched and contains the secret, but the request contained invalid signature. Expected %s, got %s.", hook.ID, expectedMAC, signature)
				return
			}
		}

		cmd := exec.Command(hook.Command)
		cmd.Args = hook.ParseJSONArgs(payload)
		cmd.Dir = hook.Cwd
		out, err := cmd.Output()
		l4g.Info("Hook %s triggered successfully! Command output:\n%s\n%+v", hook.ID, out, err)
	}
}

func formHandler(id string, formValues url.Values) {
	if hook := webhooks.Match(id, helpers.FormValuesToMap(formValues)); hook != nil {
		cmd := exec.Command(hook.Command)
		cmd.Args = hook.ParseFormArgs(formValues)
		cmd.Dir = hook.Cwd
		out, err := cmd.Output()
		l4g.Info("Hook %s triggered successfully! Command output:\n%s\n%+v", hook.ID, out, err)
	}
}

func hookHandler(req *http.Request, params martini.Params) string {
	if req.Header.Get("Content-Type") == "application/json" {
		defer req.Body.Close()

		body, err := ioutil.ReadAll(req.Body)
		if err != nil {
			l4g.Warn("Error occurred while trying to read the request body: %s", err)
		}

		payloadJSON := make(map[string]interface{})

		decoder := json.NewDecoder(strings.NewReader(string(body)))
		decoder.UseNumber()

		err = decoder.Decode(&payloadJSON)

		if err != nil {
			l4g.Warn("Error occurred while trying to parse the payload as JSON: %s", err)
		}

		payloadSignature := ""

		if strings.Contains(req.Header.Get("User-Agent"), "GitHub-Hookshot") {
			if len(req.Header.Get("X-Hub-Signature")) > 5 {
				payloadSignature = req.Header.Get("X-Hub-Signature")[5:]
			}

			go jsonHandler(params["id"], body, payloadSignature, payloadJSON)
		}
	} else {
		req.ParseForm()
		go formHandler(params["id"], req.Form)
	}

	return "Got it, thanks. :-)"
}
<commit_msg>logical error<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os/exec"
	"strings"
	"time"

	"github.com/adnanh/webhook/helpers"
	"github.com/adnanh/webhook/hooks"

	"github.com/go-martini/martini"

	l4g "code.google.com/p/log4go"
)

const (
	version string = "1.0.3"
)

var (
	webhooks      *hooks.Hooks
	appStart      time.Time
	ip            = flag.String("ip", "", "ip the webhook server should listen on")
	port          = flag.Int("port", 9000, "port the webhook server should listen on")
	hooksFilename = flag.String("hooks", "hooks.json", "path to the json file containing defined hooks the webhook should serve")
	logFilename   = flag.String("log", "webhook.log", "path to the log file")
)

func init() {
	flag.Parse()

	fileLogWriter := l4g.NewFileLogWriter(*logFilename, false)
	fileLogWriter.SetRotateDaily(false)

	martini.Env = "production"

	l4g.AddFilter("file", l4g.FINE, fileLogWriter)
}

func main() {
	appStart = time.Now()
	var e error

	webhooks, e = hooks.New(*hooksFilename)

	if e != nil {
		l4g.Warn("Error occurred while loading hooks from %s: %s", *hooksFilename, e)
	}

	web := martini.Classic()

	web.Get("/", rootHandler)
	web.Get("/hook/:id", hookHandler)
	web.Post("/hook/:id", hookHandler)

	l4g.Info("Starting webhook %s with %d hook(s) on %s:%d", version, webhooks.Count(), *ip, *port)

	web.RunOnAddr(fmt.Sprintf("%s:%d", *ip, *port))
}

func rootHandler() string {
	return fmt.Sprintf("webhook %s running for %s serving %d hook(s)\n", version, time.Since(appStart).String(), webhooks.Count())
}

func jsonHandler(id string, body []byte, signature string, payload interface{}) {
	if hook := webhooks.Match(id, payload); hook != nil {
		if hook.Secret != "" {
			if signature == "" {
				l4g.Error("Hook %s got matched and contains the secret, but the request didn't contain any signature.", hook.ID)
				return
			}

			if expectedMAC, ok := helpers.CheckPayloadSignature(body, hook.Secret, signature); !ok {
				l4g.Error("Hook %s got matched and contains the secret, but the request contained invalid signature. Expected %s, got %s.", hook.ID, expectedMAC, signature)
				return
			}
		}

		cmd := exec.Command(hook.Command)
		cmd.Args = hook.ParseJSONArgs(payload)
		cmd.Dir = hook.Cwd
		out, err := cmd.Output()
		l4g.Info("Hook %s triggered successfully! Command output:\n%s\n%+v", hook.ID, out, err)
	}
}

func formHandler(id string, formValues url.Values) {
	if hook := webhooks.Match(id, helpers.FormValuesToMap(formValues)); hook != nil {
		cmd := exec.Command(hook.Command)
		cmd.Args = hook.ParseFormArgs(formValues)
		cmd.Dir = hook.Cwd
		out, err := cmd.Output()
		l4g.Info("Hook %s triggered successfully! Command output:\n%s\n%+v", hook.ID, out, err)
	}
}

func hookHandler(req *http.Request, params martini.Params) string {
	if req.Header.Get("Content-Type") == "application/json" {
		defer req.Body.Close()

		body, err := ioutil.ReadAll(req.Body)
		if err != nil {
			l4g.Warn("Error occurred while trying to read the request body: %s", err)
		}

		payloadJSON := make(map[string]interface{})

		decoder := json.NewDecoder(strings.NewReader(string(body)))
		decoder.UseNumber()

		err = decoder.Decode(&payloadJSON)

		if err != nil {
			l4g.Warn("Error occurred while trying to parse the payload as JSON: %s", err)
		}

		payloadSignature := ""

		if strings.Contains(req.Header.Get("User-Agent"), "GitHub-Hookshot") {
			if len(req.Header.Get("X-Hub-Signature")) > 5 {
				payloadSignature = req.Header.Get("X-Hub-Signature")[5:]
			}

			go jsonHandler(params["id"], body, payloadSignature, payloadJSON)
		}
	} else {
		req.ParseForm()
		go formHandler(params["id"], req.Form)
	}

	return "Got it, thanks. :-)"
}
<|endoftext|>


<commit_before>// Convert Wikipedia XML dump to JSON or extract categories
// Example inputs:
// wikidata: http://dumps.wikimedia.org/wikidatawiki/20140612/wikidatawiki-20140612-pages-articles.xml.bz2
// wikipedia:  http://dumps.wikimedia.org/huwiki/latest/huwiki-latest-pages-articles.xml.bz2
package main

import (
	"encoding/json"
	"encoding/xml"
	"flag"
	"fmt"
	"io"
	"net/url"
	"os"
	"regexp"
	"strings"
)

const AppVersion = "1.0.7"

// Here is an example article from the Wikipedia XML dump
//
// <page>
// 	<title>Apollo 11</title>
//      <redirect title="Foo bar" />
// 	...
// 	<revision>
// 	...
// 	  <text xml:space="preserve">
// 	  {{Infobox Space mission
// 	  |mission_name=<!--See above-->
// 	  |insignia=Apollo_11_insignia.png
// 	...
// 	  </text>
// 	</revision>
// </page>
//
// Note how the tags on the fields of Page and Redirect below
// describe the XML schema structure.

type Redirect struct {
	Title string `xml:"title,attr" json:"title"`
}

// A page as it occurs on Wikipedia
type Page struct {
	Title          string   `xml:"title" json:"title"`
	CanonicalTitle string   `xml:"ctitle" json:"ctitle"`
	Redir          Redirect `xml:"redirect" json:"redirect"`
	Text           string   `xml:"revision>text" json:"text"`
}

// A page as it occurs on Wikidata, content will be turned from a string
// into a substructure with -d switch
type WikidataPage struct {
	Title          string      `xml:"title" json:"title"`
	CanonicalTitle string      `xml:"ctitle" json:"ctitle"`
	Redir          Redirect    `xml:"redirect" json:"redirect"`
	Content        interface{} `json:"content"`
}

func CanonicalizeTitle(title string) string {
	can := strings.ToLower(title)
	can = strings.Replace(can, " ", "_", -1)
	can = url.QueryEscape(can)
	return can
}

func main() {

	version := flag.Bool("v", false, "prints current version and exits")
	extractCategories := flag.String("c", "", "only extract categories TSV(page, category)")
	extractAuthorityData := flag.Bool("a", false, "only extract authority data (Normdaten)")
	decodeWikiData := flag.Bool("d", false, "decode the text key value")
	filter, _ := regexp.Compile("^file:.*|^talk:.*|^special:.*|^wikipedia:.*|^wiktionary:.*|^user:.*|^user_talk:.*")

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "\nExtract and convert things from wikipedia/wikidata XML dumps.\n")
		fmt.Fprintf(os.Stderr, "\nVersion: %s\n\n", AppVersion)
		flag.PrintDefaults()
	}

	flag.Parse()

	if *extractCategories != "" && *extractAuthorityData {
		fmt.Println("It's either -a or -c")
		os.Exit(1)
	}

	if *version {
		fmt.Println(AppVersion)
		os.Exit(0)
	}

	if flag.NArg() < 1 {
		flag.Usage()
		os.Exit(1)
	}
	inputFile := flag.Args()[0]

	xmlFile, err := os.Open(inputFile)
	if err != nil {
		fmt.Println("Error opening file:", err)
		return
	}
	defer xmlFile.Close()

	// xml decoder
	decoder := xml.NewDecoder(xmlFile)
	var inElement string
	// category pattern depends on the language, e.g. Kategorie or Category, ...
	categoryPattern := regexp.MustCompile(`\[\[` + *extractCategories + `:([^\[]+)\]\]`)
	authorityDataPattern := regexp.MustCompile(`(?mi){{Normdaten[^}]*}}`)

	// for wikidata
	var container interface{}

	for {
		// Read tokens from the XML document in a stream.
		t, _ := decoder.Token()
		if t == nil {
			break
		}
		// Inspect the type of the token just read.
		switch se := t.(type) {
		case xml.StartElement:
			// If we just read a StartElement token
			inElement = se.Name.Local
			// ...and its name is "page"
			if inElement == "page" {
				var p Page
				// decode a whole chunk of following XML into the
				// variable p which is a Page (se above)
				decoder.DecodeElement(&p, &se)

				// Do some stuff with the page.
				p.CanonicalTitle = CanonicalizeTitle(p.Title)
				m := filter.MatchString(p.CanonicalTitle)
				if !m && p.Redir.Title == "" {
					if *extractCategories != "" {
						result := categoryPattern.FindAllStringSubmatch(p.Text, -1)
						for _, value := range result {
							// replace anything after a |
							category := strings.TrimSpace(value[1])
							firstIndex := strings.Index(category, "|")
							if firstIndex != -1 {
								category = category[0:firstIndex]
							}
							fmt.Printf("%s\t%s\n", p.Title, category)
						}
					} else if *extractAuthorityData {
						result := authorityDataPattern.FindString(p.Text)
						if result != "" {
							// https://cdn.mediacru.sh/JsdjtGoLZBcR.png
							result = strings.Replace(result, "\t", "", -1)
							fmt.Printf("%s\t%s\n", p.Title, result)
						}
					} else if *decodeWikiData {

						dec := json.NewDecoder(strings.NewReader(p.Text))
						dec.UseNumber()

						if err := dec.Decode(&container); err == io.EOF {
							break
						} else if err != nil {
							fmt.Fprintf(os.Stderr, "%s\n", err)
							continue
						}

						parsed := WikidataPage{Title: p.Title,
							CanonicalTitle: p.CanonicalTitle,
							Content:        container,
							Redir:          p.Redir}

						b, err := json.Marshal(parsed)
						if err != nil {
							os.Exit(2)
						}
						fmt.Println(string(b))
					} else {
						b, err := json.Marshal(p)
						if err != nil {
							os.Exit(2)
						}
						fmt.Println(string(b))
					}
				}
			}
		default:
		}
	}
}
<commit_msg>added support for i18n authority data<commit_after>// Convert Wikipedia XML dump to JSON or extract categories
// Example inputs:
// wikidata: http://dumps.wikimedia.org/wikidatawiki/20140612/wikidatawiki-20140612-pages-articles.xml.bz2
// wikipedia:  http://dumps.wikimedia.org/huwiki/latest/huwiki-latest-pages-articles.xml.bz2
package main

import (
	"encoding/json"
	"encoding/xml"
	"flag"
	"fmt"
	"io"
	"net/url"
	"os"
	"regexp"
	"strings"
)

const AppVersion = "1.0.7"

// Here is an example article from the Wikipedia XML dump
//
// <page>
// 	<title>Apollo 11</title>
//      <redirect title="Foo bar" />
// 	...
// 	<revision>
// 	...
// 	  <text xml:space="preserve">
// 	  {{Infobox Space mission
// 	  |mission_name=<!--See above-->
// 	  |insignia=Apollo_11_insignia.png
// 	...
// 	  </text>
// 	</revision>
// </page>
//
// Note how the tags on the fields of Page and Redirect below
// describe the XML schema structure.

type Redirect struct {
	Title string `xml:"title,attr" json:"title"`
}

// A page as it occurs on Wikipedia
type Page struct {
	Title          string   `xml:"title" json:"title"`
	CanonicalTitle string   `xml:"ctitle" json:"ctitle"`
	Redir          Redirect `xml:"redirect" json:"redirect"`
	Text           string   `xml:"revision>text" json:"text"`
}

// A page as it occurs on Wikidata, content will be turned from a string
// into a substructure with -d switch
type WikidataPage struct {
	Title          string      `xml:"title" json:"title"`
	CanonicalTitle string      `xml:"ctitle" json:"ctitle"`
	Redir          Redirect    `xml:"redirect" json:"redirect"`
	Content        interface{} `json:"content"`
}

func CanonicalizeTitle(title string) string {
	can := strings.ToLower(title)
	can = strings.Replace(can, " ", "_", -1)
	can = url.QueryEscape(can)
	return can
}

func main() {

	version := flag.Bool("v", false, "prints current version and exits")
	extractCategories := flag.String("c", "", "only extract categories TSV(page, category), argument is the prefix, e.g. Kategorie or Category, ... ")
	extractAuthorityData := flag.String("a", "", "only extract authority data (Normdaten, Authority control, ...)")
	decodeWikiData := flag.Bool("d", false, "decode the text key value")
	filter, _ := regexp.Compile("^file:.*|^talk:.*|^special:.*|^wikipedia:.*|^wiktionary:.*|^user:.*|^user_talk:.*")

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "\nExtract and convert things from wikipedia/wikidata XML dumps.\n")
		fmt.Fprintf(os.Stderr, "\nVersion: %s\n\n", AppVersion)
		flag.PrintDefaults()
	}

	flag.Parse()

	if *extractCategories != "" && *extractAuthorityData != "" {
		fmt.Fprintln(os.Stderr, "it's either -a or -c")
		os.Exit(1)
	}

	if *version {
		fmt.Println(AppVersion)
		os.Exit(0)
	}

	if flag.NArg() < 1 {
		flag.Usage()
		os.Exit(1)
	}

	inputFile := flag.Args()[0]

	xmlFile, err := os.Open(inputFile)
	if err != nil {
		fmt.Println("Error opening file:", err)
		return
	}
	defer xmlFile.Close()

	// xml decoder
	decoder := xml.NewDecoder(xmlFile)
	var inElement string
	// category pattern depends on the language, e.g. Kategorie or Category, ...
	categoryPattern := regexp.MustCompile(`\[\[` + *extractCategories + `:([^\[]+)\]\]`)
	// Authority data (German only for now)
	authorityDataPattern := regexp.MustCompile(`(?mi){{` + *extractAuthorityData + `[^}]*}}`)

	// for wikidata
	var container interface{}

	for {
		// Read tokens from the XML document in a stream.
		t, _ := decoder.Token()
		if t == nil {
			break
		}
		// Inspect the type of the token just read.
		switch se := t.(type) {
		case xml.StartElement:
			// If we just read a StartElement token
			inElement = se.Name.Local
			// ...and its name is "page"
			if inElement == "page" {
				var p Page
				// decode a whole chunk of following XML into the
				// variable p which is a Page (se above)
				decoder.DecodeElement(&p, &se)

				// Do some stuff with the page.
				p.CanonicalTitle = CanonicalizeTitle(p.Title)
				m := filter.MatchString(p.CanonicalTitle)
				if !m && p.Redir.Title == "" {
					if *extractCategories != "" {
						result := categoryPattern.FindAllStringSubmatch(p.Text, -1)
						for _, value := range result {
							// replace anything after a |
							category := strings.TrimSpace(value[1])
							firstIndex := strings.Index(category, "|")
							if firstIndex != -1 {
								category = category[0:firstIndex]
							}
							fmt.Printf("%s\t%s\n", p.Title, category)
						}
					} else if *extractAuthorityData != "" {
						result := authorityDataPattern.FindString(p.Text)
						if result != "" {
							// https://cdn.mediacru.sh/JsdjtGoLZBcR.png
							result = strings.Replace(result, "\t", "", -1)
							fmt.Printf("%s\t%s\n", p.Title, result)
						}
					} else if *decodeWikiData {

						dec := json.NewDecoder(strings.NewReader(p.Text))
						dec.UseNumber()

						if err := dec.Decode(&container); err == io.EOF {
							break
						} else if err != nil {
							fmt.Fprintf(os.Stderr, "%s\n", err)
							continue
						}

						parsed := WikidataPage{Title: p.Title,
							CanonicalTitle: p.CanonicalTitle,
							Content:        container,
							Redir:          p.Redir}

						b, err := json.Marshal(parsed)
						if err != nil {
							os.Exit(2)
						}
						fmt.Println(string(b))
					} else {
						b, err := json.Marshal(p)
						if err != nil {
							os.Exit(2)
						}
						fmt.Println(string(b))
					}
				}
			}
		default:
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Tim O'Brien. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build windows

package javabind

import "syscall"

func GetThreadId() int {
//	return 1
	return int(syscall.GetCurrentThreadId())
}
<commit_msg>Add windows get thread id.<commit_after>// Copyright 2016 Tim O'Brien. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build windows

package javabind

import "golang.org/x/sys/windows"

func GetThreadId() int {
//	return 1
	return int(windows.GetCurrentThreadId())
}
<|endoftext|>


<commit_before>// Main package wraps sprite_sass tool for use with the command line
// See -h for list of available options
package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"runtime/pprof"
	"strings"
	"sync"
	"time"

	"github.com/fatih/color"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
	libsass "github.com/wellington/go-libsass"
	"github.com/wellington/wellington/version"

	wt "github.com/wellington/wellington"
	_ "github.com/wellington/wellington/handlers"
)

var (
	font, dir, gen, includes      string
	mainFile, style               string
	comments, watch               bool
	cpuprofile, buildDir          string
	jsDir                         string
	ishttp, showHelp, showVersion bool
	httpPath                      string
	timeB                         bool
	config                        string
	debug                         bool

	// unused
	noLineComments bool
	relativeAssets bool
	cssDir         string
)

/*
   --app APP                    Tell compass what kind of application it is integrating with. E.g. rails
   --fonts-dir FONTS_DIR        The directory where you keep your fonts.
*/
func init() {

	// Interoperability args
}

func flags(set *pflag.FlagSet) {
	// Unused cli args
	set.StringVarP(&buildDir, "build", "b", "",
		"Path to target directory to place generated CSS, relative paths inside project directory are preserved")
	set.BoolVarP(&comments, "comment", "", true, "Turn on source comments")
	set.BoolVar(&debug, "debug", false, "Show detailed debug information")

	set.StringVarP(&dir, "dir", "d", "",
		"Path to locate images for spriting and image functions")
	set.StringVar(&dir, "images-dir", "", "Compass backwards compat, use -d instead")

	set.StringVar(&font, "font", ".",
		"Path to directory containing fonts")
	set.StringVar(&gen, "gen", ".",
		"Path to place generated images")

	set.StringVarP(&includes, "proj", "p", "",
		"Path to directory containing Sass stylesheets")
	set.BoolVar(&noLineComments, "no-line-comments", false, "UNSUPPORTED: Disable line comments")
	set.BoolVar(&relativeAssets, "relative-assets", false, "UNSUPPORTED: Make compass asset helpers generate relative urls to assets.")

	set.BoolVarP(&showVersion, "version", "v", false, "Show the app version")
	set.StringVarP(&style, "style", "s", "nested",
		`nested style of output CSS
                        available options: nested, expanded, compact, compressed`)
	set.BoolVar(&timeB, "time", false, "Retrieve timing information")

	var nothing string
	set.StringVar(&nothing, "css-dir", "",
		"Compass backwards compat, does nothing. Reference locations relative to Sass project directory")
	set.StringVar(&jsDir, "javascripts-dir", "",
		"Compass backwards compat, ignored")
	set.StringVar(&includes, "sass-dir", "",
		"Compass backwards compat, use -p instead")
	set.StringVarP(&config, "config", "c", "",
		"Temporarily disabled: Location of the config file")

	set.StringVar(&cpuprofile, "cpuprofile", "", "Go runtime cpu profilling for debugging")
}

var compileCmd = &cobra.Command{
	Use:   "compile",
	Short: "Compile Sass stylesheets to CSS",
	Long: `Fast compilation of Sass stylesheets to CSS. For usage consult
the documentation at https://github.com/wellington/wellington#wellington`,
	Run: Compile,
}

var watchCmd = &cobra.Command{
	Use:   "watch",
	Short: "Watch Sass files for changes and rebuild CSS",
	Long:  ``,
	Run:   Watch,
}

var httpCmd = &cobra.Command{
	Use:   "serve",
	Short: "Starts a http server that will convert Sass to CSS",
	Long:  ``,
	Run:   Serve,
}

func init() {
	hostname := os.Getenv("HOSTNAME")
	if len(hostname) > 0 {
		if !strings.HasPrefix(hostname, "http") {
			hostname = "http://" + hostname
		}
	} else if host, err := os.Hostname(); err == nil {
		hostname = "http://" + host
	}
	httpCmd.Flags().StringVar(&httpPath, "httppath", hostname,
		"Only for HTTP, overrides generated sprite paths to support http")

}

func root() {
	flags(wtCmd.PersistentFlags())
}

// AddCommands attaches the cli subcommands ie. http, compile to the
// main cli entrypoint.
func AddCommands() {
	wtCmd.AddCommand(httpCmd)
	wtCmd.AddCommand(compileCmd)
	wtCmd.AddCommand(watchCmd)
}

var wtCmd = &cobra.Command{
	Use:   "wt",
	Short: "wt is a Sass project tool made to handle large projects. It uses the libSass compiler for efficiency and speed.",
	Run:   Compile,
}

func main() {
	AddCommands()
	root()
	wtCmd.Execute()
}

func argExit() bool {

	if showVersion {
		fmt.Printf("   libsass: %s\n", libsass.Version())
		fmt.Printf("Wellington: %s\n", version.Version)
		return true
	}

	if showHelp {
		fmt.Println("Please specify input filepath.")
		fmt.Println("\nAvailable options:")
		//flag.PrintDefaults()
		return true
	}
	return false

}

func parseBuildArgs(paths []string) *wt.BuildArgs {
	style, ok := libsass.Style[style]

	if !ok {
		style = libsass.NESTED_STYLE
	}
	incs := strings.Split(includes, ",")
	incs = append(incs, paths...)
	gba := &wt.BuildArgs{
		ImageDir: dir,
		BuildDir: buildDir,
		Includes: incs,
		Font:     font,
		Style:    style,
		Gen:      gen,
		Comments: comments,
	}
	gba.WithPaths(paths)

	return gba
}

func globalRun(paths []string) (*wt.SafePartialMap, *wt.BuildArgs) {

	if argExit() {
		return nil, nil
	}

	// Profiling code
	if cpuprofile != "" {
		f, err := os.Create(cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		log.Println("Starting profiler")
		pprof.StartCPUProfile(f)
		defer func() {
			pprof.StopCPUProfile()
			err := f.Close()
			if err != nil {
				log.Fatal(err)
			}
			log.Println("Stopping Profiller")
		}()
	}

	for _, v := range paths {
		if strings.HasPrefix(v, "-") {
			log.Fatalf("Please specify flags before other arguments: %s", v)
		}
	}

	if gen != "" {
		err := os.MkdirAll(gen, 0755)
		if err != nil {
			log.Fatal(err)
		}
	}

	pMap := wt.NewPartialMap()
	gba := parseBuildArgs(paths)
	if debug {
		fmt.Printf("      Font  Dir: %s\n", gba.Font)
		fmt.Printf("      Image Dir: %s\n", gba.ImageDir)
		fmt.Printf("      Build Dir: %s\n", gba.BuildDir)
		fmt.Printf("Build Image Dir: %s\n", gba.Gen)
		fmt.Printf(" Include Dir(s): %s\n", gba.Includes)
		fmt.Println("===================================")
	}
	return pMap, gba

}

// Watch accepts a set of paths starting a recursive file watcher
func Watch(cmd *cobra.Command, paths []string) {
	pMap, gba := globalRun(paths)
	var err error
	bOpts := wt.NewBuild(paths, gba, pMap)
	err = bOpts.Run()
	if err != nil {
		log.Fatal(err)
	}
	w, err := wt.NewWatcher(&wt.WatchOptions{
		Paths:      paths,
		BArgs:      gba,
		PartialMap: pMap,
	})
	if err != nil {
		log.Fatal("failed to start watcher: ", err)
	}
	err = w.Watch()
	if err != nil {
		log.Fatal("filewatcher error: ", err)
	}

	fmt.Println("File watcher started use `ctrl+d` to exit")
	in := bufio.NewReader(os.Stdin)
	for {
		_, err := in.ReadString(' ')
		if err != nil {
			if err == io.EOF {
				os.Exit(0)
			}
			fmt.Println("error", err)
		}
	}
}

// Serve starts a web server accepting POST calls and return CSS
func Serve(cmd *cobra.Command, paths []string) {

	_, gba := globalRun(paths)
	if len(gba.Gen) == 0 {
		log.Fatal("Must pass an image build directory to use HTTP")
	}

	http.Handle("/build/", wt.FileHandler(gba.Gen))
	log.Println("Web server started on :12345")

	http.HandleFunc("/", wt.HTTPHandler(gba))
	err := http.ListenAndServe(":12345", nil)
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}

}

// Compile handles compile files and stdin operations.
func Compile(cmd *cobra.Command, paths []string) {
	start := time.Now()
	pMap, gba := globalRun(paths)
	if gba == nil {
		return
	}

	defer func() {
		log.Printf("Compilation took: %s\n", time.Since(start))
	}()

	run(paths, pMap, gba)
}

// Run is the main entrypoint for the cli.
func run(paths []string, pMap *wt.SafePartialMap, gba *wt.BuildArgs) {

	// No paths given, read from stdin and wait
	if len(paths) == 0 {

		fmt.Println("Reading from stdin, -h for help")
		out := os.Stdout
		in := os.Stdin
		comp, err := wt.FromBuildArgs(out, in, gba)
		if err != nil {
			log.Fatal(err)
		}
		err = comp.Run()
		if err != nil {
			color.Red(err.Error())
		}
		return
	}

	bOpts := wt.NewBuild(paths, gba, pMap)

	err := bOpts.Run()
	if err != nil {
		log.Fatal(err)
	}

	// FIXME: move this to a Payload.Close() method

	// Before shutting down, check that every sprite has been
	// flushed to disk.
	img := sync.WaitGroup{}
	pMap.RLock()
	// It's not currently possible to wait on Image. This is often
	// to inline images, so it shouldn't be a factor...
	// for _, s := range gba.Payload.Image().M {
	// 	img.Add(1)
	// 	err := s.Wait()
	// 	img.Done()
	// 	if err != nil {
	// 		log.Printf("error writing image: %s\n", err)
	// 	}
	// }
	for _, s := range gba.Payload.Sprite().M {
		img.Add(1)
		err := s.Wait()
		img.Done()
		if err != nil {
			log.Printf("error writing sprite: %s\n", err)
		}
	}
	img.Wait()
	pMap.RUnlock()

}
<commit_msg>add httpPath<commit_after>// Main package wraps sprite_sass tool for use with the command line
// See -h for list of available options
package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"runtime/pprof"
	"strings"
	"sync"
	"time"

	"github.com/fatih/color"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
	libsass "github.com/wellington/go-libsass"
	"github.com/wellington/wellington/version"

	wt "github.com/wellington/wellington"
	_ "github.com/wellington/wellington/handlers"
)

var (
	font, dir, gen, includes      string
	mainFile, style               string
	comments, watch               bool
	cpuprofile, buildDir          string
	jsDir                         string
	ishttp, showHelp, showVersion bool
	httpPath                      string
	timeB                         bool
	config                        string
	debug                         bool

	// unused
	noLineComments bool
	relativeAssets bool
	cssDir         string
)

/*
   --app APP                    Tell compass what kind of application it is integrating with. E.g. rails
   --fonts-dir FONTS_DIR        The directory where you keep your fonts.
*/
func init() {

	// Interoperability args
}

func flags(set *pflag.FlagSet) {
	// Unused cli args
	set.StringVarP(&buildDir, "build", "b", "",
		"Path to target directory to place generated CSS, relative paths inside project directory are preserved")
	set.BoolVarP(&comments, "comment", "", true, "Turn on source comments")
	set.BoolVar(&debug, "debug", false, "Show detailed debug information")

	set.StringVarP(&dir, "dir", "d", "",
		"Path to locate images for spriting and image functions")
	set.StringVar(&dir, "images-dir", "", "Compass backwards compat, use -d instead")

	set.StringVar(&font, "font", ".",
		"Path to directory containing fonts")
	set.StringVar(&gen, "gen", ".",
		"Path to place generated images")

	set.StringVarP(&includes, "proj", "p", "",
		"Path to directory containing Sass stylesheets")
	set.BoolVar(&noLineComments, "no-line-comments", false, "UNSUPPORTED: Disable line comments")
	set.BoolVar(&relativeAssets, "relative-assets", false, "UNSUPPORTED: Make compass asset helpers generate relative urls to assets.")

	set.BoolVarP(&showVersion, "version", "v", false, "Show the app version")
	set.StringVarP(&style, "style", "s", "nested",
		`nested style of output CSS
                        available options: nested, expanded, compact, compressed`)
	set.BoolVar(&timeB, "time", false, "Retrieve timing information")

	var nothing string
	set.StringVar(&nothing, "css-dir", "",
		"Compass backwards compat, does nothing. Reference locations relative to Sass project directory")
	set.StringVar(&jsDir, "javascripts-dir", "",
		"Compass backwards compat, ignored")
	set.StringVar(&includes, "sass-dir", "",
		"Compass backwards compat, use -p instead")
	set.StringVarP(&config, "config", "c", "",
		"Temporarily disabled: Location of the config file")

	set.StringVar(&cpuprofile, "cpuprofile", "", "Go runtime cpu profilling for debugging")
}

var compileCmd = &cobra.Command{
	Use:   "compile",
	Short: "Compile Sass stylesheets to CSS",
	Long: `Fast compilation of Sass stylesheets to CSS. For usage consult
the documentation at https://github.com/wellington/wellington#wellington`,
	Run: Compile,
}

var watchCmd = &cobra.Command{
	Use:   "watch",
	Short: "Watch Sass files for changes and rebuild CSS",
	Long:  ``,
	Run:   Watch,
}

var httpCmd = &cobra.Command{
	Use:   "serve",
	Short: "Starts a http server that will convert Sass to CSS",
	Long:  ``,
	Run:   Serve,
}

func init() {
	hostname := os.Getenv("HOSTNAME")
	if len(hostname) > 0 {
		if !strings.HasPrefix(hostname, "http") {
			hostname = "http://" + hostname
		}
	} else if host, err := os.Hostname(); err == nil {
		hostname = "http://" + host
	}
	httpCmd.Flags().StringVar(&httpPath, "httppath", hostname,
		"Only for HTTP, overrides generated sprite paths to support http")

}

func root() {
	flags(wtCmd.PersistentFlags())
}

// AddCommands attaches the cli subcommands ie. http, compile to the
// main cli entrypoint.
func AddCommands() {
	wtCmd.AddCommand(httpCmd)
	wtCmd.AddCommand(compileCmd)
	wtCmd.AddCommand(watchCmd)
}

var wtCmd = &cobra.Command{
	Use:   "wt",
	Short: "wt is a Sass project tool made to handle large projects. It uses the libSass compiler for efficiency and speed.",
	Run:   Compile,
}

func main() {
	AddCommands()
	root()
	wtCmd.Execute()
}

func argExit() bool {

	if showVersion {
		fmt.Printf("   libsass: %s\n", libsass.Version())
		fmt.Printf("Wellington: %s\n", version.Version)
		return true
	}

	if showHelp {
		fmt.Println("Please specify input filepath.")
		fmt.Println("\nAvailable options:")
		//flag.PrintDefaults()
		return true
	}
	return false

}

func parseBuildArgs(paths []string) *wt.BuildArgs {
	style, ok := libsass.Style[style]

	if !ok {
		style = libsass.NESTED_STYLE
	}
	incs := strings.Split(includes, ",")
	incs = append(incs, paths...)
	gba := &wt.BuildArgs{
		ImageDir: dir,
		BuildDir: buildDir,
		Includes: incs,
		Font:     font,
		Style:    style,
		Gen:      gen,
		Comments: comments,
	}
	gba.WithPaths(paths)

	return gba
}

func globalRun(paths []string) (*wt.SafePartialMap, *wt.BuildArgs) {

	if argExit() {
		return nil, nil
	}

	// Profiling code
	if cpuprofile != "" {
		f, err := os.Create(cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		log.Println("Starting profiler")
		pprof.StartCPUProfile(f)
		defer func() {
			pprof.StopCPUProfile()
			err := f.Close()
			if err != nil {
				log.Fatal(err)
			}
			log.Println("Stopping Profiller")
		}()
	}

	for _, v := range paths {
		if strings.HasPrefix(v, "-") {
			log.Fatalf("Please specify flags before other arguments: %s", v)
		}
	}

	if gen != "" {
		err := os.MkdirAll(gen, 0755)
		if err != nil {
			log.Fatal(err)
		}
	}

	pMap := wt.NewPartialMap()
	gba := parseBuildArgs(paths)
	if debug {
		fmt.Printf("      Font  Dir: %s\n", gba.Font)
		fmt.Printf("      Image Dir: %s\n", gba.ImageDir)
		fmt.Printf("      Build Dir: %s\n", gba.BuildDir)
		fmt.Printf("Build Image Dir: %s\n", gba.Gen)
		fmt.Printf(" Include Dir(s): %s\n", gba.Includes)
		fmt.Println("===================================")
	}
	return pMap, gba

}

// Watch accepts a set of paths starting a recursive file watcher
func Watch(cmd *cobra.Command, paths []string) {
	pMap, gba := globalRun(paths)
	var err error
	bOpts := wt.NewBuild(paths, gba, pMap)
	err = bOpts.Run()
	if err != nil {
		log.Fatal(err)
	}
	w, err := wt.NewWatcher(&wt.WatchOptions{
		Paths:      paths,
		BArgs:      gba,
		PartialMap: pMap,
	})
	if err != nil {
		log.Fatal("failed to start watcher: ", err)
	}
	err = w.Watch()
	if err != nil {
		log.Fatal("filewatcher error: ", err)
	}

	fmt.Println("File watcher started use `ctrl+d` to exit")
	in := bufio.NewReader(os.Stdin)
	for {
		_, err := in.ReadString(' ')
		if err != nil {
			if err == io.EOF {
				os.Exit(0)
			}
			fmt.Println("error", err)
		}
	}
}

// Serve starts a web server accepting POST calls and return CSS
func Serve(cmd *cobra.Command, paths []string) {

	_, gba := globalRun(paths)
	if len(gba.Gen) == 0 {
		log.Fatal("Must pass an image build directory to use HTTP")
	}

	http.Handle("/build/", wt.FileHandler(gba.Gen))
	log.Println("Web server started on :12345")

	http.HandleFunc("/", wt.HTTPHandler(gba, httpPath))
	err := http.ListenAndServe(":12345", nil)
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}

}

// Compile handles compile files and stdin operations.
func Compile(cmd *cobra.Command, paths []string) {
	start := time.Now()
	pMap, gba := globalRun(paths)
	if gba == nil {
		return
	}

	defer func() {
		log.Printf("Compilation took: %s\n", time.Since(start))
	}()

	run(paths, pMap, gba)
}

// Run is the main entrypoint for the cli.
func run(paths []string, pMap *wt.SafePartialMap, gba *wt.BuildArgs) {

	// No paths given, read from stdin and wait
	if len(paths) == 0 {

		fmt.Println("Reading from stdin, -h for help")
		out := os.Stdout
		in := os.Stdin
		comp, err := wt.FromBuildArgs(out, in, gba)
		if err != nil {
			log.Fatal(err)
		}
		err = comp.Run()
		if err != nil {
			color.Red(err.Error())
		}
		return
	}

	bOpts := wt.NewBuild(paths, gba, pMap)

	err := bOpts.Run()
	if err != nil {
		log.Fatal(err)
	}

	// FIXME: move this to a Payload.Close() method

	// Before shutting down, check that every sprite has been
	// flushed to disk.
	img := sync.WaitGroup{}
	pMap.RLock()
	// It's not currently possible to wait on Image. This is often
	// to inline images, so it shouldn't be a factor...
	// for _, s := range gba.Payload.Image().M {
	// 	img.Add(1)
	// 	err := s.Wait()
	// 	img.Done()
	// 	if err != nil {
	// 		log.Printf("error writing image: %s\n", err)
	// 	}
	// }
	for _, s := range gba.Payload.Sprite().M {
		img.Add(1)
		err := s.Wait()
		img.Done()
		if err != nil {
			log.Printf("error writing sprite: %s\n", err)
		}
	}
	img.Wait()
	pMap.RUnlock()

}
<|endoftext|>


<commit_before>// Package xjs provides some simple, but often needed shortcut funcs for gopherJS
package xjs

import (
	"fmt"
	"strings"

	"github.com/gopherjs/gopherjs/js"
	"honnef.co/go/js/dom"
)

// DocumentFragment returns a new DocumentFragment as a dom.Node
func DocumentFragment() dom.Node {
	return dom.WrapNode(js.Global.Get("document").Call("createDocumentFragment"))
}

// RemoveChildren removes all of the child nodes of the node given
func RemoveChildren(node dom.Node) dom.Node {
	for node.HasChildNodes() {
		node.RemoveChild(node.LastChild())
	}
	return node
}

// SetInnerText removes all child nodes from the given node and sets a single
// Text Node with the given string
func SetInnerText(node dom.Node, text string) dom.Node {
	RemoveChildren(node)
	node.AppendChild(dom.GetWindow().Document().CreateTextNode(text))
	return node
}

// SetPreText does similar to SetInnerText, but linebreaks are converted to
s
func SetPreText(node dom.Node, text string) dom.Node {
	RemoveChildren(node)
	for n, part := range strings.Split(text, "\n") {
		if n > 0 {
			node.AppendChild(CreateElement("br"))
		}
		node.AppendChild(dom.GetWindow().Document().CreateTextNode(part))
	}
	return node
}

// CreateElement is a shortcut to create an element with the given name
func CreateElement(name string) dom.Element {
	return dom.GetWindow().Document().CreateElement(name)
}

// Alert provides for formated alert boxes
func Alert(format string, params ...interface{}) {
	dom.GetWindow().Alert(fmt.Sprintf(format, params...))
}
<commit_msg>Pulled out the document into a variable<commit_after>// Package xjs provides some simple, but often needed shortcut funcs for gopherJS
package xjs

import (
	"fmt"
	"strings"

	"honnef.co/go/js/dom"
)

var docNode = dom.GetWindow().Document()

// DocumentFragment returns a new DocumentFragment as a dom.Node
func DocumentFragment() dom.Node {
	return docNode.Underlying().Call("createDocumentFragment")
}

// RemoveChildren removes all of the child nodes of the node given
func RemoveChildren(node dom.Node) dom.Node {
	for node.HasChildNodes() {
		node.RemoveChild(node.LastChild())
	}
	return node
}

// SetInnerText removes all child nodes from the given node and sets a single
// Text Node with the given string
func SetInnerText(node dom.Node, text string) dom.Node {
	RemoveChildren(node)
	node.AppendChild(docNode.CreateTextNode(text))
	return node
}

// SetPreText does similar to SetInnerText, but linebreaks are converted to
s
func SetPreText(node dom.Node, text string) dom.Node {
	RemoveChildren(node)
	for n, part := range strings.Split(text, "\n") {
		if n > 0 {
			node.AppendChild(CreateElement("br"))
		}
		node.AppendChild(TextNode(part))
	}
	return node
}

// CreateElement is a shortcut to create an element with the given name
func CreateElement(name string) dom.Element {
	return dom.GetWindow().Document().CreateElement(name)
}

// Alert provides for formated alert boxes
func Alert(format string, params ...interface{}) {
	dom.GetWindow().Alert(fmt.Sprintf(format, params...))
}
<|endoftext|>


<commit_before>package cpu

import "testing"

func TestDecode(t *testing.T) {
	expected := [...]uint16{
		0xfc00, 0x1c00, 1024, // Adc
		0xfc00, 0x0c00, 1024, // Add
		0xff00, 0x9600, 256, // Adiw
		0xfc00, 0x2000, 1024, // And
		0xf000, 0x7000, 4096, // Andi
		0xfe0f, 0x9405, 32, // Asr
		0xff8f, 0x9488, 8, // Bclr
		0xfe08, 0xf800, 256, // Bld
		0xfc00, 0xf400, 1024, // Brbc
		0xfc00, 0xf000, 1024, // Brbs
		0xffff, 0x9598, 1, // Break
		0xff8f, 0x9408, 8, // Bset
		0xfe08, 0xfa00, 256, // Bst
		0xfe0e, 0x940e, 64, // Call
		0xff00, 0x9800, 256, // Cbi
		0xfe0f, 0x9400, 32, // Com
		0xfc00, 0x1400, 1024, // Cp
		0xfc00, 0x0400, 1024, // Cpc
		0xf000, 0x3000, 4096, // Cpi
		0xfc00, 0x1000, 1024, // Cpse
		0xfe0f, 0x940a, 32, // Dec
		0xff0f, 0x940b, 16, // Des
		0xffff, 0x9519, 1, // Eicall
		0xffff, 0x9419, 1, // Eijmp
		0x0000, 0x0000, 65, // Elpm
		0xfc00, 0x2400, 1024, // Eor
		0xff88, 0x0308, 64, // Fmul
		0xff88, 0x0380, 64, // Fmuls
		0xff88, 0x0388, 64, // Fmulsu
		0xffff, 0x9509, 1, // Icall
		0xffff, 0x9409, 1, // Ijmp
		0xf800, 0xb000, 2048, // In
		0xfe0f, 0x9403, 32, // Inc
		0xfe0e, 0x940c, 64, // Jmp
		0xfe0f, 0x9206, 32, // Lac
		0xfe0f, 0x9205, 32, // Las
		0xfe0f, 0x9207, 32, // Lat
		0x0000, 0x0000, 288, // Ld
		0x0000, 0x0000, 4032, // Ldd
		0xf000, 0xe000, 4096, // Ldi
		0xfe0f, 0x9000, 32, // Lds
		0x0000, 0x0000, 65, // Lpm
		0xfe0f, 0x9406, 32, // Lsr
		0xfc00, 0x2c00, 1024, // Mov
		0xff00, 0x0100, 256, // Movw
		0xfc00, 0x9c00, 1024, // Mul
		0xff00, 0x0200, 256, // Muls
		0xff88, 0x0300, 64, // Mulsu
		0xfe0f, 0x9401, 32, // Neg
		0xffff, 0x0000, 1, // Nop
		0xfc00, 0x2800, 1024, // Or
		0xf000, 0x6000, 4096, // Ori
		0xf800, 0xb800, 2048, // Out
		0xfe0f, 0x900f, 32, // Pop
		0xfe0f, 0x920f, 32, // Push
		0xf000, 0xd000, 4096, // Rcall
		0xffff, 0x9508, 1, // Ret
		0xffff, 0x9518, 1, // Reti
		0xf000, 0xc000, 4096, // Rjmp
		0xfe0f, 0x9407, 32, // Ror
		0xfc00, 0x0800, 1024, // Sbc
		0xf000, 0x4000, 4096, // Sbci
		0xff00, 0x9a00, 256, // Sbi
		0xff00, 0x9900, 256, // Sbic
		0xff00, 0x9b00, 256, // Sbis
		0xff00, 0x9700, 256, // Sbiw
		0xfe08, 0xfc00, 256, // Sbrc
		0xfe08, 0xfe00, 256, // Sbrs
		0xffff, 0x9588, 1, // Sleep
		0xffff, 0x95e8, 1, // Spm
		0xffff, 0x95f8, 1, // Spm2
		0x0000, 0x0000, 288, // St
		0x0000, 0x0000, 4032, // Std
		0xfe0f, 0x9200, 32, // Sts
		0xfc00, 0x1800, 1024, // Sub
		0xf000, 0x5000, 4096, // Subi
		0xfe0f, 0x9402, 32, // Swap
		0xffff, 0x95a8, 1, // Wdr
		0xfe0f, 0x9204, 32, // Xch
		0x0000, 0x0000, 1554, // Reserved
	}

	counts := make([]uint16, Illegal)

	for o := 0; o < 0x10000; o++ {
		ok := false
		inst := Decode(Opcode(o))
		counts[inst] += 1
		switch inst {
		case Elpm:
			masked := uint16(o) & 0xfe0f
			ok = o == 0x95d8 || masked == 0x9006 || masked == 0x9007
		case Ld:
			switch uint16(o) & 0xfe0f {
			case 0x8000, 0x8008:
				fallthrough
			case 0x9001, 0x9002, 0x9009, 0x900a, 0x900c, 0x900d, 0x900e:
				ok = true
			}
		case Ldd:
			ok = (uint16(o)&0xd208) == 0x8000 ||
				(uint16(o)&0xd208) == 0x8008 ||
				(uint16(o)&0xf800) == 0xa000
		case Lpm:
			masked := uint16(o) & 0xfe0f
			ok = o == 0x95c8 || masked == 0x9004 || masked == 0x9005
		case St:
			switch uint16(o) & 0xfe0f {
			case 0x8200, 0x8208:
				fallthrough
			case 0x9201, 0x9202, 0x9209, 0x920c, 0x920a, 0x920d, 0x920e:
				ok = true
			}
		case Std:
			ok = (uint16(o)&0xd208) == 0x8200 ||
				(uint16(o)&0xd208) == 0x8208
		default:
			if inst < Illegal {
				mask := expected[inst*3]
				fixed := expected[inst*3+1]
				ok = (uint16(o) & mask) == fixed
			}
		}
		if !ok {
			t.Errorf("%s(%x)", inst, o)
		}
	}
	for i := Adc; i < Illegal; i++ {
		wanted := expected[i*3+2]
		if wanted != 0 && counts[i] != wanted {
			t.Errorf("%s count was %d, not %d", i, counts[i], wanted)
		}
	}
}
<commit_msg>Better Go style in decode_test<commit_after>package cpu

import "testing"

func TestDecode(t *testing.T) {
	insts := []struct {
		mask, fixed, wanted uint16
	}{
		{0xfc00, 0x1c00, 1024}, // Adc
		{0xfc00, 0x0c00, 1024}, // Add
		{0xff00, 0x9600, 256},  // Adiw
		{0xfc00, 0x2000, 1024}, // And
		{0xf000, 0x7000, 4096}, // Andi
		{0xfe0f, 0x9405, 32},   // Asr
		{0xff8f, 0x9488, 8},    // Bclr
		{0xfe08, 0xf800, 256},  // Bld
		{0xfc00, 0xf400, 1024}, // Brbc
		{0xfc00, 0xf000, 1024}, // Brbs
		{0xffff, 0x9598, 1},    // Break
		{0xff8f, 0x9408, 8},    // Bset
		{0xfe08, 0xfa00, 256},  // Bst
		{0xfe0e, 0x940e, 64},   // Call
		{0xff00, 0x9800, 256},  // Cbi
		{0xfe0f, 0x9400, 32},   // Com
		{0xfc00, 0x1400, 1024}, // Cp
		{0xfc00, 0x0400, 1024}, // Cpc
		{0xf000, 0x3000, 4096}, // Cpi
		{0xfc00, 0x1000, 1024}, // Cpse
		{0xfe0f, 0x940a, 32},   // Dec
		{0xff0f, 0x940b, 16},   // Des
		{0xffff, 0x9519, 1},    // Eicall
		{0xffff, 0x9419, 1},    // Eijmp
		{0x0000, 0x0000, 65},   // Elpm
		{0xfc00, 0x2400, 1024}, // Eor
		{0xff88, 0x0308, 64},   // Fmul
		{0xff88, 0x0380, 64},   // Fmuls
		{0xff88, 0x0388, 64},   // Fmulsu
		{0xffff, 0x9509, 1},    // Icall
		{0xffff, 0x9409, 1},    // Ijmp
		{0xf800, 0xb000, 2048}, // In
		{0xfe0f, 0x9403, 32},   // Inc
		{0xfe0e, 0x940c, 64},   // Jmp
		{0xfe0f, 0x9206, 32},   // Lac
		{0xfe0f, 0x9205, 32},   // Las
		{0xfe0f, 0x9207, 32},   // Lat
		{0x0000, 0x0000, 288},  // Ld
		{0x0000, 0x0000, 4032}, // Ldd
		{0xf000, 0xe000, 4096}, // Ldi
		{0xfe0f, 0x9000, 32},   // Lds
		{0x0000, 0x0000, 65},   // Lpm
		{0xfe0f, 0x9406, 32},   // Lsr
		{0xfc00, 0x2c00, 1024}, // Mov
		{0xff00, 0x0100, 256},  // Movw
		{0xfc00, 0x9c00, 1024}, // Mul
		{0xff00, 0x0200, 256},  // Muls
		{0xff88, 0x0300, 64},   // Mulsu
		{0xfe0f, 0x9401, 32},   // Neg
		{0xffff, 0x0000, 1},    // Nop
		{0xfc00, 0x2800, 1024}, // Or
		{0xf000, 0x6000, 4096}, // Ori
		{0xf800, 0xb800, 2048}, // Out
		{0xfe0f, 0x900f, 32},   // Pop
		{0xfe0f, 0x920f, 32},   // Push
		{0xf000, 0xd000, 4096}, // Rcall
		{0xffff, 0x9508, 1},    // Ret
		{0xffff, 0x9518, 1},    // Reti
		{0xf000, 0xc000, 4096}, // Rjmp
		{0xfe0f, 0x9407, 32},   // Ror
		{0xfc00, 0x0800, 1024}, // Sbc
		{0xf000, 0x4000, 4096}, // Sbci
		{0xff00, 0x9a00, 256},  // Sbi
		{0xff00, 0x9900, 256},  // Sbic
		{0xff00, 0x9b00, 256},  // Sbis
		{0xff00, 0x9700, 256},  // Sbiw
		{0xfe08, 0xfc00, 256},  // Sbrc
		{0xfe08, 0xfe00, 256},  // Sbrs
		{0xffff, 0x9588, 1},    // Sleep
		{0xffff, 0x95e8, 1},    // Spm
		{0xffff, 0x95f8, 1},    // Spm2
		{0x0000, 0x0000, 288},  // St
		{0x0000, 0x0000, 4032}, // Std
		{0xfe0f, 0x9200, 32},   // Sts
		{0xfc00, 0x1800, 1024}, // Sub
		{0xf000, 0x5000, 4096}, // Subi
		{0xfe0f, 0x9402, 32},   // Swap
		{0xffff, 0x95a8, 1},    // Wdr
		{0xfe0f, 0x9204, 32},   // Xch
		{0x0000, 0x0000, 1554}, // Reserved
	}

	counts := make([]uint16, Illegal)

	for o := 0; o < 0x10000; o++ {
		ok := false
		inst := Decode(Opcode(o))
		counts[inst] += 1
		switch inst {
		case Elpm:
			masked := uint16(o) & 0xfe0f
			ok = o == 0x95d8 || masked == 0x9006 || masked == 0x9007
		case Ld:
			switch uint16(o) & 0xfe0f {
			case 0x8000, 0x8008:
				fallthrough
			case 0x9001, 0x9002, 0x9009, 0x900a, 0x900c, 0x900d, 0x900e:
				ok = true
			}
		case Ldd:
			ok = (uint16(o)&0xd208) == 0x8000 ||
				(uint16(o)&0xd208) == 0x8008 ||
				(uint16(o)&0xf800) == 0xa000
		case Lpm:
			masked := uint16(o) & 0xfe0f
			ok = o == 0x95c8 || masked == 0x9004 || masked == 0x9005
		case St:
			switch uint16(o) & 0xfe0f {
			case 0x8200, 0x8208:
				fallthrough
			case 0x9201, 0x9202, 0x9209, 0x920c, 0x920a, 0x920d, 0x920e:
				ok = true
			}
		case Std:
			ok = (uint16(o)&0xd208) == 0x8200 ||
				(uint16(o)&0xd208) == 0x8208
		default:
			if inst < Illegal {
				mask := insts[inst].mask
				fixed := insts[inst].fixed
				ok = (uint16(o) & mask) == fixed
			}
		}
		if !ok {
			t.Errorf("%s(%x)", inst, o)
		}
	}
	for i := Adc; i < Illegal; i++ {
		wanted := insts[i].wanted
		if wanted != 0 && counts[i] != wanted {
			t.Errorf("%s count was %d, not %d", i, counts[i], wanted)
		}
	}
}
<|endoftext|>


<commit_before>/*

bolt provides a bolt-backed database that implements both
boardgame.StorageManager and boardgame/server.StorageManager.

*/
package bolt

import (
	"encoding/json"
	"errors"
	"github.com/boltdb/bolt"
	"github.com/jkomoros/boardgame"
	"github.com/jkomoros/boardgame/server/api/users"
	"log"
	"os"
	"strconv"
	"strings"
)

//TODO: test this package

type StorageManager struct {
	db       *bolt.DB
	filename string
}

var (
	statesBucket    = []byte("States")
	gamesBucket     = []byte("Games")
	usersBucket     = []byte("Users")
	cookiesBucket   = []byte("Cookies")
	gameUsersBucket = []byte("GameUsers")
)

func NewStorageManager(fileName string) *StorageManager {
	db, err := bolt.Open(fileName, 0600, nil)

	if err != nil {
		panic("Couldn't open db")
	}

	err = db.Update(func(tx *bolt.Tx) error {
		if _, err := tx.CreateBucketIfNotExists(gamesBucket); err != nil {
			return errors.New("Cannot create games bucket" + err.Error())
		}
		if _, err := tx.CreateBucketIfNotExists(statesBucket); err != nil {
			return errors.New("Cannot create states bucket" + err.Error())
		}
		if _, err := tx.CreateBucketIfNotExists(usersBucket); err != nil {
			return errors.New("Cannot create users bucket" + err.Error())
		}
		if _, err := tx.CreateBucketIfNotExists(cookiesBucket); err != nil {
			return errors.New("Cannot create cookies bucket" + err.Error())
		}
		if _, err := tx.CreateBucketIfNotExists(gameUsersBucket); err != nil {
			return errors.New("Cannot create game users bucket" + err.Error())
		}
		return nil
	})

	if err != nil {
		//Not able to initalize DB
		return nil
	}
	//We don't defer DB close; our users need to.
	return &StorageManager{
		db:       db,
		filename: fileName,
	}

}

func keyForState(gameId string, version int) []byte {
	return []byte(gameId + "_" + strconv.Itoa(version))
}

func keyForGame(id string) []byte {
	return []byte(strings.ToUpper(id))
}

func keyForUser(uid string) []byte {
	return []byte(uid)
}

func keyForCookie(cookie string) []byte {
	return []byte(cookie)
}

func (s *StorageManager) State(gameId string, version int) (boardgame.StateStorageRecord, error) {
	if gameId == "" {
		return nil, errors.New("No game provided")
	}

	if version < 0 {
		return nil, errors.New("Invalid version")
	}

	var record []byte

	err := s.db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(statesBucket)

		if b == nil {
			return errors.New("Couldn't get bucket")
		}

		record = b.Get(keyForState(gameId, version))
		return nil
	})

	if err != nil {
		return nil, err
	}

	if record == nil {
		return nil, errors.New("No such version for game")
	}

	return record, nil
}

func (s *StorageManager) Game(id string) (*boardgame.GameStorageRecord, error) {

	var rawRecord []byte

	err := s.db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(gamesBucket)
		if b == nil {
			return errors.New("Couldn't open bucket")
		}
		rawRecord = b.Get(keyForGame(id))
		return nil
	})

	if err != nil {
		return nil, errors.New("Transacation error " + err.Error())
	}

	if rawRecord == nil {
		return nil, errors.New("No such game found")
	}

	var record boardgame.GameStorageRecord

	if err := json.Unmarshal(rawRecord, &record); err != nil {
		return nil, errors.New("Unmarshal error " + err.Error())
	}

	return &record, nil

}

func (s *StorageManager) SaveGameAndCurrentState(game *boardgame.GameStorageRecord, state boardgame.StateStorageRecord) error {

	version := game.Version

	serializedGameRecord, err := json.Marshal(game)

	if err != nil {
		return errors.New("Couldn't serialize the internal game record: " + err.Error())
	}

	return s.db.Update(func(tx *bolt.Tx) error {
		gBucket := tx.Bucket(gamesBucket)

		if gBucket == nil {
			return errors.New("Couldn't open games bucket")
		}

		sBucket := tx.Bucket(statesBucket)

		if sBucket == nil {
			return errors.New("Could open states bucket")
		}

		if err := gBucket.Put(keyForGame(game.Id), serializedGameRecord); err != nil {
			return err
		}

		if err := sBucket.Put(keyForState(game.Id, version), state); err != nil {
			return err
		}

		return nil

	})

}

func (s *StorageManager) ListGames(max int) []*boardgame.GameStorageRecord {

	var result []*boardgame.GameStorageRecord

	err := s.db.View(func(tx *bolt.Tx) error {

		gBucket := tx.Bucket(gamesBucket)

		if gBucket == nil {
			return errors.New("couldn't open games bucket")
		}

		c := gBucket.Cursor()
		for k, v := c.First(); k != nil; k, v = c.Next() {
			if len(result) >= max {
				break
			}

			var record boardgame.GameStorageRecord

			if err := json.Unmarshal(v, &record); err != nil {
				return errors.New("Couldn't deserialize a game: " + err.Error())
			}

			result = append(result, &record)
		}

		return nil

	})

	if err != nil {
		return nil
	}

	return result

}

func (s *StorageManager) SetPlayerForGame(gameId string, playerIndex int, userId string) error {

	ids := s.UserIdsForGame(gameId)

	if ids == nil {
		return errors.New("Couldn't fetch original player indexes for that game")
	}

	if playerIndex < 0 || playerIndex >= len(ids) {
		return errors.New("PlayerIndex " + strconv.Itoa(playerIndex) + " is not valid for this game")
	}

	if ids[playerIndex] != "" {
		return errors.New("PlayerIndex " + strconv.Itoa(playerIndex) + " is already taken")
	}

	user := s.GetUserById(userId)

	if user == nil {
		return errors.New("That userId does not describe an existing user")
	}

	ids[playerIndex] = userId

	err := s.db.Update(func(tx *bolt.Tx) error {
		gUBucket := tx.Bucket(gameUsersBucket)

		if gUBucket == nil {
			return errors.New("Couldn't open game useres bucket")
		}

		blob, err := json.Marshal(ids)

		if err != nil {
			return errors.New("Unable to marshal ids blob: " + err.Error())
		}

		return gUBucket.Put(keyForGame(gameId), blob)
	})

	if err != nil {
		return errors.New("Unable to form association: " + err.Error())
	}

	return nil

}

func (s *StorageManager) UserIdsForGame(gameId string) []string {

	noRecordErr := errors.New("No such record")

	var result []string

	err := s.db.View(func(tx *bolt.Tx) error {
		gUBucket := tx.Bucket(gameUsersBucket)

		if gUBucket == nil {
			return errors.New("Couldn't open game users bucket")
		}

		blob := gUBucket.Get(keyForGame(gameId))

		if blob == nil {
			//NO such game info.
			return noRecordErr
		}

		return json.Unmarshal(blob, &result)
	})

	if err == noRecordErr {
		//It's possible that we just haven't stored anything for this user before.

		gameRecord, err := s.Game(gameId)

		if err != nil {
			log.Println("Couldn fetch game: " + err.Error())
			return nil
		}

		if gameRecord == nil {
			log.Println("No such game")
			return nil
		}

		return make([]string, gameRecord.NumPlayers)
	}

	if err != nil {
		log.Println("Error in UserIdsForGame: ", err)
		return nil
	}

	return result
}

func (s *StorageManager) UpdateUser(user *users.StorageRecord) error {
	err := s.db.Update(func(tx *bolt.Tx) error {

		uBucket := tx.Bucket(usersBucket)

		if uBucket == nil {
			return errors.New("couldn't open users bucket")
		}

		blob, err := json.Marshal(user)

		if err != nil {
			return errors.New("Couldn't marshal user: " + err.Error())
		}

		return uBucket.Put(keyForUser(user.Id), blob)

	})

	return err
}

func (s *StorageManager) GetUserById(uid string) *users.StorageRecord {
	var result users.StorageRecord

	err := s.db.View(func(tx *bolt.Tx) error {
		uBucket := tx.Bucket(usersBucket)

		if uBucket == nil {
			return errors.New("Couldn't open users bucket")
		}

		uBlob := uBucket.Get(keyForUser(uid))

		if uBlob == nil {
			return errors.New("No such user")
		}

		return json.Unmarshal(uBlob, &result)
	})

	if err != nil {
		log.Println("Failure in GetUserById: ", err)
		return nil
	}

	return &result
}

func (s *StorageManager) GetUserByCookie(cookie string) *users.StorageRecord {

	var result users.StorageRecord

	err := s.db.View(func(tx *bolt.Tx) error {

		cBucket := tx.Bucket(cookiesBucket)

		if cBucket == nil {
			return errors.New("Couldn't open cookies bucket")
		}

		c := cBucket.Get(keyForCookie(cookie))

		if c == nil {
			return errors.New("No such cookie")
		}

		uBucket := tx.Bucket(usersBucket)

		if uBucket == nil {
			return errors.New("couldn't open users bucket")
		}

		uBlob := uBucket.Get(keyForUser(string(c)))

		if uBlob == nil {
			return errors.New("The user specified by cookie was not found")
		}

		if err := json.Unmarshal(uBlob, &result); err != nil {
			return errors.New("Unable to unmarshal user objet: " + err.Error())
		}

		return nil

	})

	if err != nil {
		log.Println("Failure in GetUserByCookie", err)
		return nil
	}

	return &result

}

func (s *StorageManager) ConnectCookieToUser(cookie string, user *users.StorageRecord) error {
	err := s.db.Update(func(tx *bolt.Tx) error {

		cBucket := tx.Bucket(cookiesBucket)

		if cBucket == nil {
			return errors.New("couldn't open cookies bucket")
		}

		if user == nil {
			//Delete the cookie.
			return cBucket.Delete(keyForCookie(cookie))
		}

		return cBucket.Put(keyForCookie(cookie), keyForUser(user.Id))

	})

	return err
}

func (s *StorageManager) Close() {
	s.db.Close()
}

func (s *StorageManager) CleanUp() {
	os.Remove(s.filename)
}
<commit_msg>Make one error message more descriptive. Part of #71.<commit_after>/*

bolt provides a bolt-backed database that implements both
boardgame.StorageManager and boardgame/server.StorageManager.

*/
package bolt

import (
	"encoding/json"
	"errors"
	"github.com/boltdb/bolt"
	"github.com/jkomoros/boardgame"
	"github.com/jkomoros/boardgame/server/api/users"
	"log"
	"os"
	"strconv"
	"strings"
)

//TODO: test this package

type StorageManager struct {
	db       *bolt.DB
	filename string
}

var (
	statesBucket    = []byte("States")
	gamesBucket     = []byte("Games")
	usersBucket     = []byte("Users")
	cookiesBucket   = []byte("Cookies")
	gameUsersBucket = []byte("GameUsers")
)

func NewStorageManager(fileName string) *StorageManager {
	db, err := bolt.Open(fileName, 0600, nil)

	if err != nil {
		panic("Couldn't open db")
	}

	err = db.Update(func(tx *bolt.Tx) error {
		if _, err := tx.CreateBucketIfNotExists(gamesBucket); err != nil {
			return errors.New("Cannot create games bucket" + err.Error())
		}
		if _, err := tx.CreateBucketIfNotExists(statesBucket); err != nil {
			return errors.New("Cannot create states bucket" + err.Error())
		}
		if _, err := tx.CreateBucketIfNotExists(usersBucket); err != nil {
			return errors.New("Cannot create users bucket" + err.Error())
		}
		if _, err := tx.CreateBucketIfNotExists(cookiesBucket); err != nil {
			return errors.New("Cannot create cookies bucket" + err.Error())
		}
		if _, err := tx.CreateBucketIfNotExists(gameUsersBucket); err != nil {
			return errors.New("Cannot create game users bucket" + err.Error())
		}
		return nil
	})

	if err != nil {
		//Not able to initalize DB
		return nil
	}
	//We don't defer DB close; our users need to.
	return &StorageManager{
		db:       db,
		filename: fileName,
	}

}

func keyForState(gameId string, version int) []byte {
	return []byte(gameId + "_" + strconv.Itoa(version))
}

func keyForGame(id string) []byte {
	return []byte(strings.ToUpper(id))
}

func keyForUser(uid string) []byte {
	return []byte(uid)
}

func keyForCookie(cookie string) []byte {
	return []byte(cookie)
}

func (s *StorageManager) State(gameId string, version int) (boardgame.StateStorageRecord, error) {
	if gameId == "" {
		return nil, errors.New("No game provided")
	}

	if version < 0 {
		return nil, errors.New("Invalid version")
	}

	var record []byte

	err := s.db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(statesBucket)

		if b == nil {
			return errors.New("Couldn't get bucket")
		}

		record = b.Get(keyForState(gameId, version))
		return nil
	})

	if err != nil {
		return nil, err
	}

	if record == nil {
		return nil, errors.New("No such version for game")
	}

	return record, nil
}

func (s *StorageManager) Game(id string) (*boardgame.GameStorageRecord, error) {

	var rawRecord []byte

	err := s.db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(gamesBucket)
		if b == nil {
			return errors.New("Couldn't open bucket")
		}
		rawRecord = b.Get(keyForGame(id))
		return nil
	})

	if err != nil {
		return nil, errors.New("Transacation error " + err.Error())
	}

	if rawRecord == nil {
		return nil, errors.New("No such game found")
	}

	var record boardgame.GameStorageRecord

	if err := json.Unmarshal(rawRecord, &record); err != nil {
		return nil, errors.New("Unmarshal error " + err.Error())
	}

	return &record, nil

}

func (s *StorageManager) SaveGameAndCurrentState(game *boardgame.GameStorageRecord, state boardgame.StateStorageRecord) error {

	version := game.Version

	serializedGameRecord, err := json.Marshal(game)

	if err != nil {
		return errors.New("Couldn't serialize the internal game record: " + err.Error())
	}

	return s.db.Update(func(tx *bolt.Tx) error {
		gBucket := tx.Bucket(gamesBucket)

		if gBucket == nil {
			return errors.New("Couldn't open games bucket")
		}

		sBucket := tx.Bucket(statesBucket)

		if sBucket == nil {
			return errors.New("Could open states bucket")
		}

		if err := gBucket.Put(keyForGame(game.Id), serializedGameRecord); err != nil {
			return err
		}

		if err := sBucket.Put(keyForState(game.Id, version), state); err != nil {
			return err
		}

		return nil

	})

}

func (s *StorageManager) ListGames(max int) []*boardgame.GameStorageRecord {

	var result []*boardgame.GameStorageRecord

	err := s.db.View(func(tx *bolt.Tx) error {

		gBucket := tx.Bucket(gamesBucket)

		if gBucket == nil {
			return errors.New("couldn't open games bucket")
		}

		c := gBucket.Cursor()
		for k, v := c.First(); k != nil; k, v = c.Next() {
			if len(result) >= max {
				break
			}

			var record boardgame.GameStorageRecord

			if err := json.Unmarshal(v, &record); err != nil {
				return errors.New("Couldn't deserialize a game: " + err.Error())
			}

			result = append(result, &record)
		}

		return nil

	})

	if err != nil {
		return nil
	}

	return result

}

func (s *StorageManager) SetPlayerForGame(gameId string, playerIndex int, userId string) error {

	ids := s.UserIdsForGame(gameId)

	if ids == nil {
		return errors.New("Couldn't fetch original player indexes for that game")
	}

	if playerIndex < 0 || playerIndex >= len(ids) {
		return errors.New("PlayerIndex " + strconv.Itoa(playerIndex) + " is not valid for this game")
	}

	if ids[playerIndex] != "" {
		return errors.New("PlayerIndex " + strconv.Itoa(playerIndex) + " is already taken")
	}

	user := s.GetUserById(userId)

	if user == nil {
		return errors.New("That userId does not describe an existing user")
	}

	ids[playerIndex] = userId

	err := s.db.Update(func(tx *bolt.Tx) error {
		gUBucket := tx.Bucket(gameUsersBucket)

		if gUBucket == nil {
			return errors.New("Couldn't open game useres bucket")
		}

		blob, err := json.Marshal(ids)

		if err != nil {
			return errors.New("Unable to marshal ids blob: " + err.Error())
		}

		return gUBucket.Put(keyForGame(gameId), blob)
	})

	if err != nil {
		return errors.New("Unable to form association: " + err.Error())
	}

	return nil

}

func (s *StorageManager) UserIdsForGame(gameId string) []string {

	noRecordErr := errors.New("No such record")

	var result []string

	err := s.db.View(func(tx *bolt.Tx) error {
		gUBucket := tx.Bucket(gameUsersBucket)

		if gUBucket == nil {
			return errors.New("Couldn't open game users bucket")
		}

		blob := gUBucket.Get(keyForGame(gameId))

		if blob == nil {
			//NO such game info.
			return noRecordErr
		}

		return json.Unmarshal(blob, &result)
	})

	if err == noRecordErr {
		//It's possible that we just haven't stored anything for this user before.

		gameRecord, err := s.Game(gameId)

		if err != nil {
			log.Println("Couldn fetch game: " + err.Error())
			return nil
		}

		if gameRecord == nil {
			log.Println("No such game")
			return nil
		}

		return make([]string, gameRecord.NumPlayers)
	}

	if err != nil {
		log.Println("Error in UserIdsForGame: ", err)
		return nil
	}

	return result
}

func (s *StorageManager) UpdateUser(user *users.StorageRecord) error {
	err := s.db.Update(func(tx *bolt.Tx) error {

		uBucket := tx.Bucket(usersBucket)

		if uBucket == nil {
			return errors.New("couldn't open users bucket")
		}

		blob, err := json.Marshal(user)

		if err != nil {
			return errors.New("Couldn't marshal user: " + err.Error())
		}

		return uBucket.Put(keyForUser(user.Id), blob)

	})

	return err
}

func (s *StorageManager) GetUserById(uid string) *users.StorageRecord {
	var result users.StorageRecord

	err := s.db.View(func(tx *bolt.Tx) error {
		uBucket := tx.Bucket(usersBucket)

		if uBucket == nil {
			return errors.New("Couldn't open users bucket")
		}

		uBlob := uBucket.Get(keyForUser(uid))

		if uBlob == nil {
			return errors.New("No such user")
		}

		return json.Unmarshal(uBlob, &result)
	})

	if err != nil {
		log.Println("Failure in GetUserById: ", err)
		return nil
	}

	return &result
}

func (s *StorageManager) GetUserByCookie(cookie string) *users.StorageRecord {

	var result users.StorageRecord

	err := s.db.View(func(tx *bolt.Tx) error {

		cBucket := tx.Bucket(cookiesBucket)

		if cBucket == nil {
			return errors.New("Couldn't open cookies bucket")
		}

		c := cBucket.Get(keyForCookie(cookie))

		if c == nil {
			return errors.New("No such cookie: " + cookie)
		}

		uBucket := tx.Bucket(usersBucket)

		if uBucket == nil {
			return errors.New("couldn't open users bucket")
		}

		uBlob := uBucket.Get(keyForUser(string(c)))

		if uBlob == nil {
			return errors.New("The user specified by cookie was not found")
		}

		if err := json.Unmarshal(uBlob, &result); err != nil {
			return errors.New("Unable to unmarshal user objet: " + err.Error())
		}

		return nil

	})

	if err != nil {
		log.Println("Failure in GetUserByCookie", err)
		return nil
	}

	return &result

}

func (s *StorageManager) ConnectCookieToUser(cookie string, user *users.StorageRecord) error {
	err := s.db.Update(func(tx *bolt.Tx) error {

		cBucket := tx.Bucket(cookiesBucket)

		if cBucket == nil {
			return errors.New("couldn't open cookies bucket")
		}

		if user == nil {
			//Delete the cookie.
			return cBucket.Delete(keyForCookie(cookie))
		}

		return cBucket.Put(keyForCookie(cookie), keyForUser(user.Id))

	})

	return err
}

func (s *StorageManager) Close() {
	s.db.Close()
}

func (s *StorageManager) CleanUp() {
	os.Remove(s.filename)
}
<|endoftext|>


<commit_before>package storage

import (
	"io"

	"github.com/anacrolix/torrent/metainfo"
)

// Represents data storage for an unspecified torrent.
type ClientImpl interface {
	OpenTorrent(info *metainfo.Info, infoHash metainfo.Hash) (TorrentImpl, error)
}

// Data storage bound to a torrent.
type TorrentImpl interface {
	Piece(metainfo.Piece) PieceImpl
	Close() error
}

// Interacts with torrent piece data.
type PieceImpl interface {
	// These interfaces are not as strict as normally required. They can
	// assume that the parameters are appropriate for the dimensions of the
	// piece.
	io.ReaderAt
	io.WriterAt
	// Called when the client believes the piece data will pass a hash check.
	// The storage can move or mark the piece data as read-only as it sees
	// fit.
	MarkComplete() error
	MarkNotComplete() error
	// Returns true if the piece is complete.
	GetIsComplete() bool
}
<commit_msg>storage: ClientImpl interface now includes Close<commit_after>package storage

import (
	"io"

	"github.com/anacrolix/torrent/metainfo"
)

// Represents data storage for an unspecified torrent.
type ClientImpl interface {
	OpenTorrent(info *metainfo.Info, infoHash metainfo.Hash) (TorrentImpl, error)
	Close() error
}

// Data storage bound to a torrent.
type TorrentImpl interface {
	Piece(metainfo.Piece) PieceImpl
	Close() error
}

// Interacts with torrent piece data.
type PieceImpl interface {
	// These interfaces are not as strict as normally required. They can
	// assume that the parameters are appropriate for the dimensions of the
	// piece.
	io.ReaderAt
	io.WriterAt
	// Called when the client believes the piece data will pass a hash check.
	// The storage can move or mark the piece data as read-only as it sees
	// fit.
	MarkComplete() error
	MarkNotComplete() error
	// Returns true if the piece is complete.
	GetIsComplete() bool
}
<|endoftext|>