<commit_before>package main

import (
	"fmt"
	"github.com/MG-RAST/Shock/shock-server/auth"
	"github.com/MG-RAST/Shock/shock-server/conf"
	"github.com/MG-RAST/Shock/shock-server/controller"
	"github.com/MG-RAST/Shock/shock-server/db"
	"github.com/MG-RAST/Shock/shock-server/logger"
	"github.com/MG-RAST/Shock/shock-server/node"
	"github.com/MG-RAST/Shock/shock-server/preauth"
	"github.com/MG-RAST/Shock/shock-server/user"
	"github.com/MG-RAST/golib/goweb"
	"os"
	"runtime"
)

func launchSite(control chan int) {
	goweb.ConfigureDefaultFormatters()
	r := &goweb.RouteManager{}
	r.MapFunc("/raw", RawDir)
	r.MapFunc("/assets", AssetsDir)
	r.MapFunc("*", Site)
	if conf.Bool(conf.Conf["ssl"]) {
		err := goweb.ListenAndServeRoutesTLS(":"+conf.Conf["site-port"], conf.Conf["ssl-cert"], conf.Conf["ssl-key"], r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: site: %v\n", err)
			logger.Error("ERROR: site: " + err.Error())
		}
	} else {
		err := goweb.ListenAndServeRoutes(":"+conf.Conf["site-port"], r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: site: %v\n", err)
			logger.Error("ERROR: site: " + err.Error())
		}
	}
	control <- 1 //we are ending
}

func launchAPI(control chan int) {
	c := controller.New()
	goweb.ConfigureDefaultFormatters()
	r := &goweb.RouteManager{}
	r.MapFunc("/preauth/{id}", c.Preauth, goweb.GetMethod)
	r.Map("/node/{nid}/acl/{type}", c.Acl["typed"])
	r.Map("/node/{nid}/acl", c.Acl["base"])
	r.Map("/node/{nid}/index/{type}", c.Index)
	r.Map("/node/{nid}/index", c.Index)
	r.MapRest("/node", c.Node)
	r.MapFunc("*", ResourceDescription, goweb.GetMethod)
	r.MapFunc("*", RespondOk, goweb.OptionsMethod)
	if conf.Bool(conf.Conf["ssl"]) {
		err := goweb.ListenAndServeRoutesTLS(":"+conf.Conf["api-port"], conf.Conf["ssl-cert"], conf.Conf["ssl-key"], r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: api: %v\n", err)
			logger.Error("ERROR: api: " + err.Error())
		}
	} else {
		err := goweb.ListenAndServeRoutes(":"+conf.Conf["api-port"], r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: api: %v\n", err)
			logger.Error("ERROR: api: " + err.Error())
		}
	}
	control <- 1 //we are ending
}

func main() {
	// init(s)
	conf.Initialize()
	logger.Initialize()
	if err := db.Initialize(); err != nil {
		logger.Error(err.Error())
	}
	user.Initialize()
	node.Initialize()
	preauth.Initialize()
	auth.Initialize()

	// print conf
	printLogo()
	conf.Print()

	if _, err := os.Stat(conf.Conf["data-path"] + "/temp"); err != nil && os.IsNotExist(err) {
		if err := os.Mkdir(conf.Conf["data-path"]+"/temp", 0777); err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
			logger.Error("ERROR: " + err.Error())
		}
	}

	// reload
	if conf.RELOAD != "" {
		fmt.Println("####### Reloading #######")
		err := reload(conf.RELOAD)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
			logger.Error("ERROR: " + err.Error())
		}
		fmt.Println("Done")
	}

	// increased cpu usage
	var max int
	avail := runtime.NumCPU()
	if avail <= 2 {
		max = 1
	} else if avail == 3 {
		max = 2
	} else {
		max = avail - 2
	}
	runtime.GOMAXPROCS(max)
	fmt.Fprintf(os.Stderr, "Number of available CPUs = %d\n", avail)
	fmt.Fprintf(os.Stderr, "Running Shock server using %d CPUs\n", max)

	//launch server
	control := make(chan int)
	go launchSite(control)
	go launchAPI(control)
	<-control //block till something dies
}
<commit_msg>Formatting fixes<commit_after>package main

import (
	"fmt"
	"github.com/MG-RAST/Shock/shock-server/auth"
	"github.com/MG-RAST/Shock/shock-server/conf"
	"github.com/MG-RAST/Shock/shock-server/controller"
	"github.com/MG-RAST/Shock/shock-server/db"
	"github.com/MG-RAST/Shock/shock-server/logger"
	"github.com/MG-RAST/Shock/shock-server/node"
	"github.com/MG-RAST/Shock/shock-server/preauth"
	"github.com/MG-RAST/Shock/shock-server/user"
	"github.com/MG-RAST/golib/goweb"
	"os"
	"runtime"
)

func launchSite(control chan int) {
	goweb.ConfigureDefaultFormatters()
	r := &goweb.RouteManager{}
	r.MapFunc("/raw", RawDir)
	r.MapFunc("/assets", AssetsDir)
	r.MapFunc("*", Site)
	if conf.Bool(conf.Conf["ssl"]) {
		err := goweb.ListenAndServeRoutesTLS(":"+conf.Conf["site-port"], conf.Conf["ssl-cert"], conf.Conf["ssl-key"], r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: site: %v\n", err)
			logger.Error("ERROR: site: " + err.Error())
		}
	} else {
		err := goweb.ListenAndServeRoutes(":"+conf.Conf["site-port"], r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: site: %v\n", err)
			logger.Error("ERROR: site: " + err.Error())
		}
	}
	control <- 1 //we are ending
}

func launchAPI(control chan int) {
	c := controller.New()
	goweb.ConfigureDefaultFormatters()
	r := &goweb.RouteManager{}
	r.MapFunc("/preauth/{id}", c.Preauth, goweb.GetMethod)
	r.Map("/node/{nid}/acl/{type}", c.Acl["typed"])
	r.Map("/node/{nid}/acl", c.Acl["base"])
	r.Map("/node/{nid}/index/{type}", c.Index)
	r.Map("/node/{nid}/index", c.Index)
	r.MapRest("/node", c.Node)
	r.MapFunc("*", ResourceDescription, goweb.GetMethod)
	r.MapFunc("*", RespondOk, goweb.OptionsMethod)
	if conf.Bool(conf.Conf["ssl"]) {
		err := goweb.ListenAndServeRoutesTLS(":"+conf.Conf["api-port"], conf.Conf["ssl-cert"], conf.Conf["ssl-key"], r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: api: %v\n", err)
			logger.Error("ERROR: api: " + err.Error())
		}
	} else {
		err := goweb.ListenAndServeRoutes(":"+conf.Conf["api-port"], r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: api: %v\n", err)
			logger.Error("ERROR: api: " + err.Error())
		}
	}
	control <- 1 //we are ending
}

func main() {
	// init(s)
	conf.Initialize()
	logger.Initialize()
	if err := db.Initialize(); err != nil {
		logger.Error(err.Error())
	}
	user.Initialize()
	node.Initialize()
	preauth.Initialize()
	auth.Initialize()

	// print conf
	printLogo()
	conf.Print()

	if _, err := os.Stat(conf.Conf["data-path"] + "/temp"); err != nil && os.IsNotExist(err) {
		if err := os.Mkdir(conf.Conf["data-path"]+"/temp", 0777); err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
			logger.Error("ERROR: " + err.Error())
		}
	}

	// reload
	if conf.RELOAD != "" {
		fmt.Println("####### Reloading #######")
		err := reload(conf.RELOAD)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
			logger.Error("ERROR: " + err.Error())
		}
		fmt.Println("Done")
	}

	// increased cpu usage
	var max int
	avail := runtime.NumCPU()
	if avail <= 2 {
		max = 1
	} else if avail == 3 {
		max = 2
	} else {
		max = avail - 2
	}
	runtime.GOMAXPROCS(max)
	fmt.Fprintf(os.Stderr, "Number of available CPUs = %d\n", avail)
	fmt.Fprintf(os.Stderr, "Running Shock server using %d CPUs\n", max)

	//launch server
	control := make(chan int)
	go launchSite(control)
	go launchAPI(control)
	<-control //block till something dies
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fs

import (
	"fmt"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcloud/syncutil"
	"github.com/jacobsa/gcsfuse/fs/inode"
	"golang.org/x/net/context"
)

// State required for reading from directories.
type dirHandle struct {
	/////////////////////////
	// Constant data
	/////////////////////////

	in *inode.DirInode

	/////////////////////////
	// Mutable state
	/////////////////////////

	Mu syncutil.InvariantMutex

	// Entries that we have buffered from a previous call to in.ReadEntries.
	//
	// INVARIANT: For each i in range, entries[i+1].Offset == entries[i].Offset + 1
	//
	// GUARDED_BY(Mu)
	entries []fuseutil.Dirent

	// The logical offset at which entries[0] lies.
	//
	// INVARIANT: If len(entries) > 0, entriesOffset + 1 == entries[0].Offset
	//
	// GUARDED_BY(Mu)
	entriesOffset fuse.DirOffset

	// The continuation token to supply in the next call to in.ReadEntries. If
	// empty, we have finished reading the directory and must be rewound to
	// obtain more entries.
	//
	// GUARDED_BY(Mu)
	tok string
}

// Create a directory handle that obtains listings from the supplied inode.
func newDirHandle(in *inode.DirInode) (dh *dirHandle) {
	// Set up the basic struct.
	dh = &dirHandle{
		in: in,
	}

	// Set up invariant checking.
	dh.Mu = syncutil.NewInvariantMutex(dh.checkInvariants)

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func (dh *dirHandle) checkInvariants() {
	// Check that the offsets increment by one each time.
	for i := 0; i < len(dh.entries)-1; i++ {
		if dh.entries[i].Offset+1 != dh.entries[i+1].Offset {
			panic(
				fmt.Sprintf(
					"Unexpected offset sequence: %v, %v",
					dh.entries[i].Offset,
					dh.entries[i+1].Offset))
		}
	}

	// Check the first offset.
	if len(dh.entries) > 0 && dh.entries[0].Offset != dh.entriesOffset+1 {
		panic(fmt.Sprintf("Unexpected entriesOffset value."))
	}
}

// Read some entries from the directory inode. Return newTok == "" (possibly
// with a non-empty list of entries) when the end of the directory has been
// hit.
//
// SHARED_LOCKS_REQUIRED(in.Mu)
func readEntries(
	ctx context.Context,
	in *inode.DirInode,
	tok string,
	firstEntryOffset fuse.DirOffset) (
	entries []fuseutil.Dirent, newTok string, err error) {
	// Fix up the offset of any entries returned.
	defer func() {
		for i := 0; i < len(entries); i++ {
			entries[i].Offset = firstEntryOffset + 1 + fuse.DirOffset(i)
		}
	}()

	// Loop until we get a non-empty result, we finish, or an error occurs.
	for {
		entries, newTok, err = in.ReadEntries(ctx, tok)

		// Propagate errors.
		if err != nil {
			return
		}

		// If some entries were returned, we are done.
		if len(entries) > 0 {
			return
		}

		// If we're at the end of the directory, we're done.
		if newTok == "" {
			return
		}

		// Otherwise, go around and ask for more entries.
		tok = newTok
	}
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

// Handle a request to read from the directory.
//
// Because the GCS API for listing objects has no notion of a stable offset
// like the posix telldir/seekdir API does, there is no way for us to
// efficiently support seeking backwards. Therefore we return EINVAL when an
// offset for an entry that we no longer have buffered is encountered.
//
// Special case: we assume that a zero offset indicates that rewinddir has been
// called (since fuse gives us no way to intercept and know for sure), and
// start the listing process over again.
//
// EXCLUSIVE_LOCKS_REQUIRED(dh.Mu)
// LOCKS_EXCLUDED(dh.in.Mu)
func (dh *dirHandle) ReadDir(
	ctx context.Context,
	req *fuse.ReadDirRequest) (resp *fuse.ReadDirResponse, err error) {
	resp = &fuse.ReadDirResponse{}

	dh.in.Mu.RLock()
	defer dh.in.Mu.RUnlock()

	// If the request is for offset zero, we assume that either this is the first
	// call or rewinddir has been called. Reset state.
	if req.Offset == 0 {
		dh.entries = nil
		dh.entriesOffset = 0
		dh.tok = ""
	}

	// Is the offset from before what we have buffered? If not, this represents a
	// seekdir we cannot support, as discussed in the method comments above.
	if req.Offset < dh.entriesOffset {
		err = fuse.EINVAL
		return
	}

	// Is the offset past the end of what we have buffered? If so, this must be
	// an invalid seekdir according to posix.
	index := int(req.Offset - dh.entriesOffset)
	if index > len(dh.entries) {
		err = fuse.EINVAL
		return
	}

	// Do we need to read more entries, and can we?
	if index == len(dh.entries) && dh.tok != "" {
		var newEntries []fuseutil.Dirent
		var newTok string

		// Read some entries.
		newEntries, newTok, err = readEntries(
			ctx,
			dh.in,
			dh.tok,
			dh.entriesOffset+fuse.DirOffset(len(dh.entries)))

		if err != nil {
			err = fmt.Errorf("readEntries: %v", err)
			return
		}

		// Update state.
		dh.entriesOffset += fuse.DirOffset(len(dh.entries))
		dh.entries = newEntries
		dh.tok = newTok

		// Now we want to read from the front of dh.entries.
		index = 0
	}

	// Now we copy out entries until we run out of entries or space.
	for i := index; i < len(dh.entries); i++ {
		resp.Data = fuseutil.AppendDirent(resp.Data, dh.entries[i])
		if len(resp.Data) > req.Size {
			resp.Data = resp.Data[:req.Size]
			break
		}
	}

	return
}
<commit_msg>Fixed a confusion between start and end of ReadDir.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fs

import (
	"fmt"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcloud/syncutil"
	"github.com/jacobsa/gcsfuse/fs/inode"
	"golang.org/x/net/context"
)

// State required for reading from directories.
type dirHandle struct {
	/////////////////////////
	// Constant data
	/////////////////////////

	in *inode.DirInode

	/////////////////////////
	// Mutable state
	/////////////////////////

	Mu syncutil.InvariantMutex

	// Entries that we have buffered from a previous call to in.ReadEntries.
	//
	// INVARIANT: For each i in range, entries[i+1].Offset == entries[i].Offset + 1
	//
	// GUARDED_BY(Mu)
	entries []fuseutil.Dirent

	// The logical offset at which entries[0] lies.
	//
	// INVARIANT: If len(entries) > 0, entriesOffset + 1 == entries[0].Offset
	//
	// GUARDED_BY(Mu)
	entriesOffset fuse.DirOffset

	// The continuation token to supply in the next call to in.ReadEntries. At
	// the start of the directory, this is the empty string. When we have hit the
	// end of the directory, this is nil.
	//
	// GUARDED_BY(Mu)
	tok *string
}

// Create a directory handle that obtains listings from the supplied inode.
func newDirHandle(in *inode.DirInode) (dh *dirHandle) {
	// Set up the basic struct.
	dh = &dirHandle{
		in:  in,
		tok: new(string),
	}

	// Set up invariant checking.
	dh.Mu = syncutil.NewInvariantMutex(dh.checkInvariants)

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func (dh *dirHandle) checkInvariants() {
	// Check that the offsets increment by one each time.
	for i := 0; i < len(dh.entries)-1; i++ {
		if dh.entries[i].Offset+1 != dh.entries[i+1].Offset {
			panic(
				fmt.Sprintf(
					"Unexpected offset sequence: %v, %v",
					dh.entries[i].Offset,
					dh.entries[i+1].Offset))
		}
	}

	// Check the first offset.
	if len(dh.entries) > 0 && dh.entries[0].Offset != dh.entriesOffset+1 {
		panic(fmt.Sprintf("Unexpected entriesOffset value."))
	}
}

// Read some entries from the directory inode. Return newTok == nil (possibly
// with a non-empty list of entries) when the end of the directory has been
// hit.
//
// SHARED_LOCKS_REQUIRED(in.Mu)
func readEntries(
	ctx context.Context,
	in *inode.DirInode,
	tok string,
	firstEntryOffset fuse.DirOffset) (
	entries []fuseutil.Dirent, newTok *string, err error) {
	// Fix up the offset of any entries returned.
	defer func() {
		for i := 0; i < len(entries); i++ {
			entries[i].Offset = firstEntryOffset + 1 + fuse.DirOffset(i)
		}
	}()

	// Return newTok != nil iff there is more to read. Note that we use tok in
	// the loop below.
	defer func() {
		if tok != "" {
			newTok = &tok
		}
	}()

	// Loop until we get a non-empty result, we finish, or an error occurs.
	for {
		entries, tok, err = in.ReadEntries(ctx, tok)

		// Propagate errors.
		if err != nil {
			return
		}

		// If some entries were returned, we are done.
		if len(entries) > 0 {
			return
		}

		// If we're at the end of the directory, we're done.
		if tok == "" {
			return
		}

		// Otherwise, go around and ask for more entries.
	}
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

// Handle a request to read from the directory.
//
// Because the GCS API for listing objects has no notion of a stable offset
// like the posix telldir/seekdir API does, there is no way for us to
// efficiently support seeking backwards. Therefore we return EINVAL when an
// offset for an entry that we no longer have buffered is encountered.
//
// Special case: we assume that a zero offset indicates that rewinddir has been
// called (since fuse gives us no way to intercept and know for sure), and
// start the listing process over again.
//
// EXCLUSIVE_LOCKS_REQUIRED(dh.Mu)
// LOCKS_EXCLUDED(dh.in.Mu)
func (dh *dirHandle) ReadDir(
	ctx context.Context,
	req *fuse.ReadDirRequest) (resp *fuse.ReadDirResponse, err error) {
	resp = &fuse.ReadDirResponse{}

	dh.in.Mu.RLock()
	defer dh.in.Mu.RUnlock()

	// If the request is for offset zero, we assume that either this is the first
	// call or rewinddir has been called. Reset state.
	if req.Offset == 0 {
		dh.entries = nil
		dh.entriesOffset = 0
		dh.tok = new(string)
	}

	// Is the offset from before what we have buffered? If not, this represents a
	// seekdir we cannot support, as discussed in the method comments above.
	if req.Offset < dh.entriesOffset {
		err = fuse.EINVAL
		return
	}

	// Is the offset past the end of what we have buffered? If so, this must be
	// an invalid seekdir according to posix.
	index := int(req.Offset - dh.entriesOffset)
	if index > len(dh.entries) {
		err = fuse.EINVAL
		return
	}

	// Do we need to read more entries, and can we?
	if index == len(dh.entries) && dh.tok != nil {
		var newEntries []fuseutil.Dirent
		var newTok *string

		// Read some entries.
		newEntries, newTok, err = readEntries(
			ctx,
			dh.in,
			*dh.tok,
			dh.entriesOffset+fuse.DirOffset(len(dh.entries)))

		if err != nil {
			err = fmt.Errorf("readEntries: %v", err)
			return
		}

		// Update state.
		dh.entriesOffset += fuse.DirOffset(len(dh.entries))
		dh.entries = newEntries
		dh.tok = newTok

		// Now we want to read from the front of dh.entries.
		index = 0
	}

	// Now we copy out entries until we run out of entries or space.
	for i := index; i < len(dh.entries); i++ {
		resp.Data = fuseutil.AppendDirent(resp.Data, dh.entries[i])
		if len(resp.Data) > req.Size {
			resp.Data = resp.Data[:req.Size]
			break
		}
	}

	return
}
<|endoftext|>


<commit_before>package cachemanager
import (
	"testing"
	"time"
	"github.com/stretchr/testify/assert"
)

func TestHandlerMemory(t *testing.T) {
	// Initialize
	k := "test"
	v := map[string]string{
		"key1": "value1",
		"key2": "value2",
		"key3": "value3",
	}
	c := ConfigurationMemory{
		CleanupInterval: 500,
		Configuration: Configuration{
			Prefix: "test_",
		},
	}
	m := NewHandlerMemoryFromConfiguration(c)

	// Set
	m.Set(k, v, time.Duration(5) * time.Microsecond)
	vc, e := m.Get(k)
	assert.NoError(t, e)
	assert.Equal(t, v, vc)

	// Wait for expiration
	time.Sleep(time.Duration(5) * time.Microsecond)
	vc, e = m.Get(k)
	assert.EqualError(t, e, ErrCacheMiss.Error())

	// Del
	m.Set(k, v, time.Duration(5) * time.Microsecond)
	m.Del(k)
	vc, e = m.Get(k)
	assert.EqualError(t, e, ErrCacheMiss.Error())

	// Increment
	m.Set(k, uint64(5), time.Duration(20) * time.Microsecond)
	_, e = m.Increment(k, 1)
	assert.NoError(t, e)
	vc, e = m.Get(k)
	assert.NoError(t, e)
	assert.Equal(t, uint64(6), vc)

	// Decrement
	m.Decrement(k, 2)
	vc, e = m.Get(k)
	assert.NoError(t, e)
	assert.Equal(t, uint64(4), vc)
}
<commit_msg>goimports<commit_after>package cachemanager

import (
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

func TestHandlerMemory(t *testing.T) {
	// Initialize
	k := "test"
	v := map[string]string{
		"key1": "value1",
		"key2": "value2",
		"key3": "value3",
	}
	c := ConfigurationMemory{
		CleanupInterval: 500,
		Configuration: Configuration{
			Prefix: "test_",
		},
	}
	m := NewHandlerMemoryFromConfiguration(c)

	// Set
	m.Set(k, v, time.Duration(5)*time.Microsecond)
	vc, e := m.Get(k)
	assert.NoError(t, e)
	assert.Equal(t, v, vc)

	// Wait for expiration
	time.Sleep(time.Duration(5) * time.Microsecond)
	vc, e = m.Get(k)
	assert.EqualError(t, e, ErrCacheMiss.Error())

	// Del
	m.Set(k, v, time.Duration(5)*time.Microsecond)
	m.Del(k)
	vc, e = m.Get(k)
	assert.EqualError(t, e, ErrCacheMiss.Error())

	// Increment
	m.Set(k, uint64(5), time.Duration(20)*time.Microsecond)
	_, e = m.Increment(k, 1)
	assert.NoError(t, e)
	vc, e = m.Get(k)
	assert.NoError(t, e)
	assert.Equal(t, uint64(6), vc)

	// Decrement
	m.Decrement(k, 2)
	vc, e = m.Get(k)
	assert.NoError(t, e)
	assert.Equal(t, uint64(4), vc)
}
<|endoftext|>


<commit_before>package benchmark

import (
	"testing"
)

type GetImpl interface {
	Get(document interface{}, pointer string) (interface{}, error)
}

type SetImpl interface {
	GetImpl
	Set(document *interface{}, pointer string, value interface{}) error
}

var implementations = map[string]GetImpl{
	"dolmen-go/jsonptr":     DolmenGoJsonPtr{},
	"xeipuuv/gojsonpointer": XeipuuvGoJsonPointer{},
	"mickep76/jsonptr":      Mickep76JsonPtr{},
	"lestrrat/go-jspointer": LestrratGoJsPointer{},
	"dustin/go-jsonpointer": DustinGoJsonPointer{},
	"rnd42/go-jsonpointer":  Rnd42JSONPointer{},
	"twindagger/jsonptr": TwindaggerJSONPtr{},
}

type PointerParser interface {
	Parse(pointer string) (Stringer, error)
}

var parsers = make(map[string]PointerParser)

func init() {
	for name, impl := range implementations {
		if impl, ok := impl.(PointerParser); ok {
			parsers[name] = impl
		}
	}
}

func BenchmarkGet(b *testing.B) {
	doc := map[string]interface{}{
		"foo": map[string]interface{}{
			"bar": []interface{}{
				true,
				nil,
				false,
			},
		},
	}
	for name, impl := range implementations {
		b.Run("["+name+"]", func(b *testing.B) {
			for i := 0; i < b.N; i++ {
				res, err := impl.Get(doc, "/foo/bar/2")
				if err != nil {
					b.FailNow()
				}
				if res != false {
					b.FailNow()
				}
			}
		})
	}
}

var pointers = []string{
	"/foo/bar",
	"/foo/bar/baz/~0~1",
}

func benchmarkParsers(b *testing.B, f func(func(string) (Stringer, error), string) func(*testing.B)) {
	for _, pointer := range pointers {
		b.Run(`"`+pointer+`"`, func(b *testing.B) {
			for name, p := range parsers {
				f2 := f(p.Parse, pointer)
				if f2 == nil {
					continue
				}
				b.Run("["+name+"]", func(b *testing.B) {
					for i := 0; i < b.N; i++ {
						f2(b)
					}
				})
			}
		})
	}
}

func BenchmarkParse(b *testing.B) {
	benchmarkParsers(b, func(Parse func(string) (Stringer, error), pointer string) func(*testing.B) {
		// Returns the body of the benchmark: parsing the given pointer
		return func(*testing.B) {
			_, _ = Parse(pointer)
		}
	})
}

func BenchmarkBackToString(b *testing.B) {
	benchmarkParsers(b, func(Parse func(string) (Stringer, error), pointer string) func(*testing.B) {
		// Parse the pointer, but before starting the benchmark
		ptr, err := Parse(pointer)
		if err != nil {
			b.FailNow()
		}
		// Returns the body of the benchmark: formatting the given pointer
		return func(b *testing.B) {
			out := ptr.String()
			if out != pointer {
				b.FailNow()
			}
		}
	})
}

func BenchmarkParseAndBackToString(b *testing.B) {
	benchmarkParsers(b, func(Parse func(string) (Stringer, error), pointer string) func(*testing.B) {
		// Parse the pointer, but before starting the benchmark
		// Returns the body of the benchmark: formatting the given pointer
		return func(b *testing.B) {
			ptr, err := Parse(pointer)
			if err != nil {
				b.FailNow()
			}
			out := ptr.String()
			if out != pointer {
				b.FailNow()
			}
		}
	})
}
<commit_msg>BenchmarkParse: use more realistic test cases<commit_after>package benchmark

import (
	"testing"
)

type GetImpl interface {
	Get(document interface{}, pointer string) (interface{}, error)
}

type SetImpl interface {
	GetImpl
	Set(document *interface{}, pointer string, value interface{}) error
}

var implementations = map[string]GetImpl{
	"dolmen-go/jsonptr":     DolmenGoJsonPtr{},
	"xeipuuv/gojsonpointer": XeipuuvGoJsonPointer{},
	"mickep76/jsonptr":      Mickep76JsonPtr{},
	"lestrrat/go-jspointer": LestrratGoJsPointer{},
	"dustin/go-jsonpointer": DustinGoJsonPointer{},
	"rnd42/go-jsonpointer":  Rnd42JSONPointer{},
	"twindagger/jsonptr": TwindaggerJSONPtr{},
}

type PointerParser interface {
	Parse(pointer string) (Stringer, error)
}

var parsers = make(map[string]PointerParser)

func init() {
	for name, impl := range implementations {
		if impl, ok := impl.(PointerParser); ok {
			parsers[name] = impl
		}
	}
}

func BenchmarkGet(b *testing.B) {
	doc := map[string]interface{}{
		"foo": map[string]interface{}{
			"bar": []interface{}{
				true,
				nil,
				false,
			},
		},
	}
	for name, impl := range implementations {
		b.Run("["+name+"]", func(b *testing.B) {
			for i := 0; i < b.N; i++ {
				res, err := impl.Get(doc, "/foo/bar/2")
				if err != nil {
					b.FailNow()
				}
				if res != false {
					b.FailNow()
				}
			}
		})
	}
}

var pointers = []string{
	// "/foo/bar",
	// "/foo/bar/baz/~0~1",
	"/definitions/Location",
	"/path/~1home~1dolmen",
	"/path/~0~1dolmen",
}

func benchmarkParsers(b *testing.B, f func(func(string) (Stringer, error), string) func(*testing.B)) {
	for _, pointer := range pointers {
		b.Run(`"`+pointer+`"`, func(b *testing.B) {
			for name, p := range parsers {
				f2 := f(p.Parse, pointer)
				if f2 == nil {
					continue
				}
				b.Run("["+name+"]", func(b *testing.B) {
					for i := 0; i < b.N; i++ {
						f2(b)
					}
				})
			}
		})
	}
}

func BenchmarkParse(b *testing.B) {
	benchmarkParsers(b, func(Parse func(string) (Stringer, error), pointer string) func(*testing.B) {
		// Returns the body of the benchmark: parsing the given pointer
		return func(*testing.B) {
			_, _ = Parse(pointer)
		}
	})
}

func BenchmarkBackToString(b *testing.B) {
	benchmarkParsers(b, func(Parse func(string) (Stringer, error), pointer string) func(*testing.B) {
		// Parse the pointer, but before starting the benchmark
		ptr, err := Parse(pointer)
		if err != nil {
			b.FailNow()
		}
		// Returns the body of the benchmark: formatting the given pointer
		return func(b *testing.B) {
			out := ptr.String()
			if out != pointer {
				b.FailNow()
			}
		}
	})
}

func BenchmarkParseAndBackToString(b *testing.B) {
	benchmarkParsers(b, func(Parse func(string) (Stringer, error), pointer string) func(*testing.B) {
		// Parse the pointer, but before starting the benchmark
		// Returns the body of the benchmark: formatting the given pointer
		return func(b *testing.B) {
			ptr, err := Parse(pointer)
			if err != nil {
				b.FailNow()
			}
			out := ptr.String()
			if out != pointer {
				b.FailNow()
			}
		}
	})
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"go/token"

	"github.com/davyxu/gosproto/meta"
)

const goCodeTemplate = `// Generated by github.com/davyxu/gosproto/sprotogen
// DO NOT EDIT!

package {{.PackageName}}

import (
	"reflect"
	"github.com/davyxu/gosproto"
	"github.com/davyxu/goobjfmt"
	{{if .CellnetReg}}"github.com/davyxu/cellnet/codec/sproto"{{end}}
	{{if .EnumValueGroup}}"fmt"{{end}}
)

{{range $a, $enumobj := .Enums}}
type {{.Name}} int32
const (	{{range .StFields}}
	{{$enumobj.Name}}_{{.Name}} {{.GoEnumTypeName}} = {{.TagNumber}} {{end}}
)

var (
{{$enumobj.Name}}MapperValueByName = map[string]int32{ {{range .StFields}}
	"{{.Name}}": {{.TagNumber}}, {{end}}
}

{{$enumobj.Name}}MapperNameByValue = map[int32]string{ {{range .StFields}}
	{{.TagNumber}}: "{{.Name}}" , {{end}}
}
)

func (self {{$enumobj.Name}}) String() string {
	return sproto.EnumName({{$enumobj.Name}}MapperNameByValue, int32(self))
}
{{end}}

{{range $a, $stobj := .Structs}}
type {{.Name}} struct{
	{{range .StFields}}
		{{.GoFieldName}} {{.GoTypeName}} {{.GoTags}}
	{{end}}
}

func (self *{{.Name}}) String() string { return goobjfmt.CompactTextString(self) }

{{range .StFields}}{{if .IsExtendType}}
func (self *{{$stobj.Name}}) {{.GoExtendFieldGetterName}}() {{.GoExtendFieldGetterType}} {
	{{.GoExtendFieldGetter}}
}
func (self *{{$stobj.Name}}) Set{{.GoExtendFieldGetterName}}(v {{.GoExtendFieldGetterType}})  {
	{{.GoExtendFieldSetter}}
}
{{end}} {{end}}
{{end}}

var SProtoStructs = []reflect.Type{
{{range .Structs}}
	reflect.TypeOf((*{{.Name}})(nil)).Elem(), // {{.MsgID}} {{end}}
}

var SProtoEnumValue = map[string]map[int32]string{ {{range .Enums}}
	"{{.Name}}": {{.Name}}MapperNameByValue,{{end}}
}

{{if .EnumValueGroup}}
func ResultToString(result int) string {
	switch( result ) {
		case 0: return "OK";
	{{range $a, $enumObj := .Enums}} {{if .IsResultEnum}} {{range .Fields}} {{if ne .TagNumber 0}}
		case {{.TagNumber}}: return "{{$enumObj.Name}}.{{.Name}}"; {{end}} {{end}} {{end}} {{end}}
	}

	return fmt.Sprintf("result: %d", result)
}
{{end}}
{{if .CellnetReg}}
func init() {
	sprotocodec.AutoRegisterMessageMeta(SProtoStructs)
}
{{end}}

`

func (self *fieldModel) GoEnumTypeName() string {

	if self.st.EnumValueIgnoreType {
		return ""
	}

	return self.st.Name
}

func (self *fieldModel) GoExtendFieldGetterName() string {
	pname := publicFieldName(self.Name)

	if token.Lookup(pname).IsKeyword() {
		return pname + "_"
	}

	return pname
}

func (self *fieldModel) GoExtendFieldGetter() string {

	switch self.Type {
	case meta.FieldType_Float32,
		meta.FieldType_Float64:
		return fmt.Sprintf("return %s(self.%s) * %f", self.GoExtendFieldGetterType(), self.GoFieldName(), 1.0/float32(self.ExtendTypePrecision()))
	}

	return "unknown extend type:" + self.Type.String()
}

func (self *fieldModel) GoExtendFieldSetter() string {

	switch self.Type {
	case meta.FieldType_Float32:
		return fmt.Sprintf("self.%s = int32(v* %d)", self.GoFieldName(), self.ExtendTypePrecision())
	case meta.FieldType_Float64:
		return fmt.Sprintf("self.%s = int64(v* %d)", self.GoFieldName(), self.ExtendTypePrecision())
	}

	return "unknown extend type:" + self.Type.String()
}

func (self *fieldModel) GoExtendFieldGetterType() string {
	var b bytes.Buffer
	if self.Repeatd {
		b.WriteString("[]")
	}

	// 字段类型映射go的类型
	switch self.Type {
	case meta.FieldType_Float32:
		b.WriteString("float32")
	case meta.FieldType_Float64:
		b.WriteString("float64")
	default:
		b.WriteString("unknown extend type:" + self.Type.String())
	}

	return b.String()
}

func (self *fieldModel) GoFieldName() string {

	var pname string

	// 扩展类型不能直接访问
	if self.IsExtendType() {
		pname = "Extend_" + self.Name
	} else {
		pname = publicFieldName(self.Name)
	}

	// 碰到关键字在尾部加_
	if token.Lookup(pname).IsKeyword() {
		return pname + "_"
	}

	return pname
}

func (self *fieldModel) GoTypeName() string {

	var b bytes.Buffer
	if self.Repeatd {
		b.WriteString("[]")
	}

	if self.Type == meta.FieldType_Struct {
		b.WriteString("*")
	}

	// 字段类型映射go的类型
	switch self.Type {
	case meta.FieldType_Integer:
		b.WriteString("int")
	case meta.FieldType_Bool:
		b.WriteString("bool")
	case meta.FieldType_Struct,
		meta.FieldType_Enum:
		b.WriteString(self.Complex.Name)
	case meta.FieldType_Float32:
		b.WriteString("int32")
	case meta.FieldType_Float64:
		b.WriteString("int64")
	case meta.FieldType_Bytes:
		b.WriteString("[]byte")
	default:
		b.WriteString(self.Type.String())
	}

	return b.String()
}

func (self *fieldModel) GoTags() string {

	var b bytes.Buffer

	b.WriteString("`sproto:\"")

	// 整形类型对解码层都视为整形
	switch self.Type {
	case meta.FieldType_Int32,
		meta.FieldType_Int64,
		meta.FieldType_UInt32,
		meta.FieldType_UInt64,
		meta.FieldType_Float32,
		meta.FieldType_Float64,
		meta.FieldType_Enum:
		b.WriteString("integer")
	case meta.FieldType_Bytes:
		b.WriteString("string")
	default:
		b.WriteString(self.Kind())
	}

	b.WriteString(",")

	b.WriteString(fmt.Sprintf("%d", self.TagNumber()))
	b.WriteString(",")

	if self.Repeatd {
		b.WriteString("array,")
	}

	b.WriteString(fmt.Sprintf("name=%s", self.GoFieldName()))

	b.WriteString("\"`")

	return b.String()
}

func gen_go(fm *fileModel, filename string) {

	addData(fm, "go")

	generateCode("sp->go", goCodeTemplate, filename, fm, &generateOption{
		formatGoCode: true,
	})

}
<commit_msg>修正: 调整ResultToString参数类型<commit_after>package main

import (
	"bytes"
	"fmt"
	"go/token"

	"github.com/davyxu/gosproto/meta"
)

const goCodeTemplate = `// Generated by github.com/davyxu/gosproto/sprotogen
// DO NOT EDIT!

package {{.PackageName}}

import (
	"reflect"
	"github.com/davyxu/gosproto"
	"github.com/davyxu/goobjfmt"
	{{if .CellnetReg}}"github.com/davyxu/cellnet/codec/sproto"{{end}}
	{{if .EnumValueGroup}}"fmt"{{end}}
)

{{range $a, $enumobj := .Enums}}
type {{.Name}} int32
const (	{{range .StFields}}
	{{$enumobj.Name}}_{{.Name}} {{.GoEnumTypeName}} = {{.TagNumber}} {{end}}
)

var (
{{$enumobj.Name}}MapperValueByName = map[string]int32{ {{range .StFields}}
	"{{.Name}}": {{.TagNumber}}, {{end}}
}

{{$enumobj.Name}}MapperNameByValue = map[int32]string{ {{range .StFields}}
	{{.TagNumber}}: "{{.Name}}" , {{end}}
}
)

func (self {{$enumobj.Name}}) String() string {
	return sproto.EnumName({{$enumobj.Name}}MapperNameByValue, int32(self))
}
{{end}}

{{range $a, $stobj := .Structs}}
type {{.Name}} struct{
	{{range .StFields}}
		{{.GoFieldName}} {{.GoTypeName}} {{.GoTags}}
	{{end}}
}

func (self *{{.Name}}) String() string { return goobjfmt.CompactTextString(self) }

{{range .StFields}}{{if .IsExtendType}}
func (self *{{$stobj.Name}}) {{.GoExtendFieldGetterName}}() {{.GoExtendFieldGetterType}} {
	{{.GoExtendFieldGetter}}
}
func (self *{{$stobj.Name}}) Set{{.GoExtendFieldGetterName}}(v {{.GoExtendFieldGetterType}})  {
	{{.GoExtendFieldSetter}}
}
{{end}} {{end}}
{{end}}

var SProtoStructs = []reflect.Type{
{{range .Structs}}
	reflect.TypeOf((*{{.Name}})(nil)).Elem(), // {{.MsgID}} {{end}}
}

var SProtoEnumValue = map[string]map[int32]string{ {{range .Enums}}
	"{{.Name}}": {{.Name}}MapperNameByValue,{{end}}
}

{{if .EnumValueGroup}}
func ResultToString(result int32) string {
	switch( result ) {
		case 0: return "OK";
	{{range $a, $enumObj := .Enums}} {{if .IsResultEnum}} {{range .Fields}} {{if ne .TagNumber 0}}
		case {{.TagNumber}}: return "{{$enumObj.Name}}.{{.Name}}"; {{end}} {{end}} {{end}} {{end}}
	}

	return fmt.Sprintf("result: %d", result)
}
{{end}}
{{if .CellnetReg}}
func init() {
	sprotocodec.AutoRegisterMessageMeta(SProtoStructs)
}
{{end}}

`

func (self *fieldModel) GoEnumTypeName() string {

	if self.st.EnumValueIgnoreType {
		return ""
	}

	return self.st.Name
}

func (self *fieldModel) GoExtendFieldGetterName() string {
	pname := publicFieldName(self.Name)

	if token.Lookup(pname).IsKeyword() {
		return pname + "_"
	}

	return pname
}

func (self *fieldModel) GoExtendFieldGetter() string {

	switch self.Type {
	case meta.FieldType_Float32,
		meta.FieldType_Float64:
		return fmt.Sprintf("return %s(self.%s) * %f", self.GoExtendFieldGetterType(), self.GoFieldName(), 1.0/float32(self.ExtendTypePrecision()))
	}

	return "unknown extend type:" + self.Type.String()
}

func (self *fieldModel) GoExtendFieldSetter() string {

	switch self.Type {
	case meta.FieldType_Float32:
		return fmt.Sprintf("self.%s = int32(v* %d)", self.GoFieldName(), self.ExtendTypePrecision())
	case meta.FieldType_Float64:
		return fmt.Sprintf("self.%s = int64(v* %d)", self.GoFieldName(), self.ExtendTypePrecision())
	}

	return "unknown extend type:" + self.Type.String()
}

func (self *fieldModel) GoExtendFieldGetterType() string {
	var b bytes.Buffer
	if self.Repeatd {
		b.WriteString("[]")
	}

	// 字段类型映射go的类型
	switch self.Type {
	case meta.FieldType_Float32:
		b.WriteString("float32")
	case meta.FieldType_Float64:
		b.WriteString("float64")
	default:
		b.WriteString("unknown extend type:" + self.Type.String())
	}

	return b.String()
}

func (self *fieldModel) GoFieldName() string {

	var pname string

	// 扩展类型不能直接访问
	if self.IsExtendType() {
		pname = "Extend_" + self.Name
	} else {
		pname = publicFieldName(self.Name)
	}

	// 碰到关键字在尾部加_
	if token.Lookup(pname).IsKeyword() {
		return pname + "_"
	}

	return pname
}

func (self *fieldModel) GoTypeName() string {

	var b bytes.Buffer
	if self.Repeatd {
		b.WriteString("[]")
	}

	if self.Type == meta.FieldType_Struct {
		b.WriteString("*")
	}

	// 字段类型映射go的类型
	switch self.Type {
	case meta.FieldType_Integer:
		b.WriteString("int")
	case meta.FieldType_Bool:
		b.WriteString("bool")
	case meta.FieldType_Struct,
		meta.FieldType_Enum:
		b.WriteString(self.Complex.Name)
	case meta.FieldType_Float32:
		b.WriteString("int32")
	case meta.FieldType_Float64:
		b.WriteString("int64")
	case meta.FieldType_Bytes:
		b.WriteString("[]byte")
	default:
		b.WriteString(self.Type.String())
	}

	return b.String()
}

func (self *fieldModel) GoTags() string {

	var b bytes.Buffer

	b.WriteString("`sproto:\"")

	// 整形类型对解码层都视为整形
	switch self.Type {
	case meta.FieldType_Int32,
		meta.FieldType_Int64,
		meta.FieldType_UInt32,
		meta.FieldType_UInt64,
		meta.FieldType_Float32,
		meta.FieldType_Float64,
		meta.FieldType_Enum:
		b.WriteString("integer")
	case meta.FieldType_Bytes:
		b.WriteString("string")
	default:
		b.WriteString(self.Kind())
	}

	b.WriteString(",")

	b.WriteString(fmt.Sprintf("%d", self.TagNumber()))
	b.WriteString(",")

	if self.Repeatd {
		b.WriteString("array,")
	}

	b.WriteString(fmt.Sprintf("name=%s", self.GoFieldName()))

	b.WriteString("\"`")

	return b.String()
}

func gen_go(fm *fileModel, filename string) {

	addData(fm, "go")

	generateCode("sp->go", goCodeTemplate, filename, fm, &generateOption{
		formatGoCode: true,
	})

}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"path"
	"regexp"
	"strconv"
	"time"

	"github.com/gorilla/feeds"
	xmlpath "gopkg.in/xmlpath.v2"
)

const (
	baseURL = "http://tv.so-net.ne.jp"
	tvURL   = baseURL + "/chart/23.action"
	bsURL   = baseURL + "/chart/bs1.action"
)

var (
	// in chart page
	schedulePath = xmlpath.MustCompile(`//a[@class="schedule-link"]`)
	linkPath     = xmlpath.MustCompile(`./@href`)

	// in schedule page
	infoPath = xmlpath.MustCompile(`//dl[@class="basicTxt"]/dd`)
	textPath = xmlpath.MustCompile(`./text()`)

	// schedule handling
	dateLayout      = "200601021504"
	durationPattern = regexp.MustCompile(`\d+`)
	interval        = 3 * time.Second
	maxOffsetIndex  = 33
	idSet           = make(map[string]bool)
)

// Program stores information of one tv program extracted from tv prgram detail page.
// eg. http://tv.so-net.ne.jp/schedule/101048201511291800.action
type Program struct {
	ChannelID int           `json:"channel_id"`
	Title     string        `json:"title"`
	StartTime time.Time     `json:"start_time"`
	EndTime   time.Time     `json:"end_time"`
	Duration  time.Duration `json:"duration"`
	Link      string        `json:"link"`
}

// NewProgram is a constructor of Program. uri should be URL of the tv program page.
func NewProgram(uri string) (*Program, error) {
	resp, err := http.Get(uri)
	if err != nil {
		return nil, err
	}

	// parse HTML data.
	root, err := xmlpath.ParseHTML(resp.Body)
	if err != nil {
		return nil, err
	}
	infos := infoPath.Iter(root)
	infos.Next()
	titleNode := textPath.Iter(infos.Node())
	titleNode.Next()
	title := titleNode.Node().String()
	infos.Next()
	scheduleNode := textPath.Iter(infos.Node())
	scheduleNode.Next()
	scheduleDesc := durationPattern.FindAllStringSubmatch(scheduleNode.Node().String(), -1)
	durn := scheduleDesc[len(scheduleDesc)-1][0]
	dur, err := strconv.Atoi(durn)
	if err != nil {
		return nil, err
	}

	// parse info in URL
	urlid := path.Base(uri)[0:18]
	cidd := urlid[2:6]
	startd := urlid[6:]

	cid, err := strconv.Atoi(cidd)
	if err != nil {
		return nil, err
	}

	date, err := time.Parse(dateLayout, startd)
	if err != nil {
		return nil, err
	}
	return &Program{
		ChannelID: cid,
		Title:     title,
		StartTime: date,
		EndTime:   date.Add(time.Duration(dur) * time.Minute),
		Duration:  time.Duration(dur) * time.Minute,
		Link:      uri,
	}, nil
}

func offsetParam(hours int) string {
	now := time.Now()
	offset := now.Add(time.Duration(hours) * time.Hour)
	return "head=" + offset.Format(dateLayout)
}

func main() {
	chartURLs := make(chan string)
	go func() {
		for i := 0; i < maxOffsetIndex; i++ {
			param := offsetParam(i * 5)
			chartURLs <- tvURL + "?" + param
			chartURLs <- bsURL + "?" + param
		}
	}()
	programURLs := RenderChart(chartURLs)
	programs := ProgramInfo(programURLs)
	feed := generateFeed(programs)
	file, err := os.Create("feed.atom")
	if err != nil {
		log.Fatal(err)
	}
	err = feed.WriteAtom(file)
	if err != nil {
		log.Fatal(err)
	}
}

func RenderChart(links <-chan string) chan string {
	programURLs := make(chan string, 100)
	go func() {
		for l := range links {
			time.Sleep(interval)
			resp, err := http.Get(l)
			if err != nil {
				log.Fatal(err)
			}
			defer resp.Body.Close()
			root, err := xmlpath.ParseHTML(resp.Body)
			if err != nil {
				log.Fatal(err)
			}
			schedules := schedulePath.Iter(root)
			for schedules.Next() {
				n := schedules.Node()
				link := linkPath.Iter(n)
				link.Next()
				programURLs <- link.Node().String()
			}
		}
		close(programURLs)
	}()
	return programURLs
}

func ProgramInfo(links <-chan string) chan *Program {
	programs := make(chan *Program, 100)
	go func() {
		for l := range links {
			uri := baseURL + l
			id := path.Base(uri)
			if idSet[id] {
				continue
			} else {
				idSet[id] = true
			}
			time.Sleep(interval)
			p, err := NewProgram(uri)
			if err != nil {
				log.Println(err)
				continue
			}
			programs <- p
			time.Sleep(2 * time.Second)
		}
		close(programs)
	}()
	return programs
}

func generateFeed(programs <-chan *Program) *feeds.Feed {
	feed := &feeds.Feed{
		Title:       "Gガイド",
		Link:        &feeds.Link{Href: "https://tv.so-net.ne.jp/"},
		Description: "GガイドのRSS",
		Author:      &feeds.Author{"Anonymous", "john.doe@example.com"},
	}
	i := 0
	items := make([]*feeds.Item, 10000)
	for p := range programs {
		items[i] = &feeds.Item{
			Title:       p.Title,
			Link:        &feeds.Link{Href: p.Link},
			Description: fmt.Sprintf("%v - %v (%v)", p.StartTime, p.EndTime, p.Duration),
			Author:      &feeds.Author{strconv.Itoa(p.ChannelID), strconv.Itoa(p.ChannelID) + "@example.com"},
			Created:     p.StartTime,
		}
		i++
		if i > 10000 {
			break
		}
	}
	return feed
}
<commit_msg>decreased max feed num<commit_after>package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"path"
	"regexp"
	"strconv"
	"time"

	"github.com/gorilla/feeds"
	xmlpath "gopkg.in/xmlpath.v2"
)

const (
	baseURL = "http://tv.so-net.ne.jp"
	tvURL   = baseURL + "/chart/23.action"
	bsURL   = baseURL + "/chart/bs1.action"
)

var (
	// in chart page
	schedulePath = xmlpath.MustCompile(`//a[@class="schedule-link"]`)
	linkPath     = xmlpath.MustCompile(`./@href`)

	// in schedule page
	infoPath = xmlpath.MustCompile(`//dl[@class="basicTxt"]/dd`)
	textPath = xmlpath.MustCompile(`./text()`)

	// schedule handling
	dateLayout      = "200601021504"
	durationPattern = regexp.MustCompile(`\d+`)
	interval        = 3500 * time.Millisecond
	maxOffsetIndex  = 33
	maxFeedItems    = 200
	idSet           = make(map[string]bool)
)

// Program stores information of one tv program extracted from tv prgram detail page.
// eg. http://tv.so-net.ne.jp/schedule/101048201511291800.action
type Program struct {
	ChannelID int           `json:"channel_id"`
	Title     string        `json:"title"`
	StartTime time.Time     `json:"start_time"`
	EndTime   time.Time     `json:"end_time"`
	Duration  time.Duration `json:"duration"`
	Link      string        `json:"link"`
}

// NewProgram is a constructor of Program. uri should be URL of the tv program page.
func NewProgram(uri string) (*Program, error) {
	log.Println("[NewProgram]", uri)
	resp, err := http.Get(uri)
	if err != nil {
		return nil, err
	}

	// parse HTML data.
	root, err := xmlpath.ParseHTML(resp.Body)
	if err != nil {
		return nil, err
	}
	infos := infoPath.Iter(root)
	infos.Next()
	titleNode := textPath.Iter(infos.Node())
	titleNode.Next()
	title := titleNode.Node().String()
	infos.Next()
	scheduleNode := textPath.Iter(infos.Node())
	scheduleNode.Next()
	scheduleDesc := durationPattern.FindAllStringSubmatch(scheduleNode.Node().String(), -1)
	durn := scheduleDesc[len(scheduleDesc)-1][0]
	dur, err := strconv.Atoi(durn)
	if err != nil {
		return nil, err
	}

	// parse info in URL
	urlid := path.Base(uri)[0:18]
	cidd := urlid[2:6]
	startd := urlid[6:]

	cid, err := strconv.Atoi(cidd)
	if err != nil {
		return nil, err
	}

	date, err := time.Parse(dateLayout, startd)
	if err != nil {
		return nil, err
	}
	return &Program{
		ChannelID: cid,
		Title:     title,
		StartTime: date,
		EndTime:   date.Add(time.Duration(dur) * time.Minute),
		Duration:  time.Duration(dur) * time.Minute,
		Link:      uri,
	}, nil
}

func offsetParam(hours int) string {
	now := time.Now()
	offset := now.Add(time.Duration(hours) * time.Hour)
	return "head=" + offset.Format(dateLayout)
}

func main() {
	chartURLs := make(chan string)
	go func() {
		for i := 0; i < maxOffsetIndex; i++ {
			param := offsetParam(i * 5)
			chartURLs <- tvURL + "?" + param
			chartURLs <- bsURL + "?" + param
		}
	}()
	programURLs := RenderChart(chartURLs)
	programs := ProgramInfo(programURLs)
	feed := generateFeed(programs)
	file, err := os.Create("feed.atom")
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()
	err = feed.WriteAtom(file)
	if err != nil {
		log.Fatal(err)
	}
}

func RenderChart(links <-chan string) chan string {
	programURLs := make(chan string, 100)
	go func() {
		for l := range links {
			time.Sleep(interval)
			log.Println("[RenderChart]", l)
			resp, err := http.Get(l)
			if err != nil {
				log.Println("[RenderChart]", err)
			}
			defer resp.Body.Close()
			root, err := xmlpath.ParseHTML(resp.Body)
			if err != nil {
				log.Println("[RenderChart]", err)
			}
			schedules := schedulePath.Iter(root)
			for schedules.Next() {
				n := schedules.Node()
				link := linkPath.Iter(n)
				link.Next()
				programURLs <- link.Node().String()
			}
		}
		close(programURLs)
	}()
	return programURLs
}

func ProgramInfo(links chan string) chan *Program {
	programs := make(chan *Program, 100)
	go func() {
		for l := range links {
			uri := baseURL + l
			id := path.Base(uri)
			if idSet[id] {
				continue
			} else {
				idSet[id] = true
			}
			time.Sleep(interval)
			p, err := NewProgram(uri)
			if err != nil {
				log.Println("[ProgramInfo]", err)
				continue
			}
			programs <- p
		}
		close(programs)
	}()
	return programs
}

func generateFeed(programs <-chan *Program) *feeds.Feed {
	feed := &feeds.Feed{
		Title:       "Gガイド",
		Link:        &feeds.Link{Href: "https://tv.so-net.ne.jp/"},
		Description: "GガイドのRSS",
		Author:      &feeds.Author{"Anonymous", "john.doe@example.com"},
	}
	i := 0
	items := make([]*feeds.Item, maxFeedItems)
	for p := range programs {
		items[i] = &feeds.Item{
			Title:       p.Title,
			Link:        &feeds.Link{Href: p.Link},
			Description: fmt.Sprintf("%v - %v (%v)", p.StartTime, p.EndTime, p.Duration),
			Author:      &feeds.Author{strconv.Itoa(p.ChannelID), strconv.Itoa(p.ChannelID) + "@example.com"},
			Created:     p.StartTime,
		}
		i++
		if i >= maxFeedItems {
			break
		}
	}
	feed.Items = items
	return feed
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"

	"github.com/gorilla/mux"
	"github.com/pkg/errors"

	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/operations"
	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/lxd/response"
	"github.com/lxc/lxd/lxd/state"
	storagePools "github.com/lxc/lxd/lxd/storage"
	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/validate"
	"github.com/lxc/lxd/shared/version"
)

func instanceSnapshotsGet(d *Daemon, r *http.Request) response.Response {
	instanceType, err := urlInstanceTypeDetect(r)
	if err != nil {
		return response.SmartError(err)
	}

	projectName := projectParam(r)
	cname := mux.Vars(r)["name"]

	// Handle requests targeted to a container on a different node
	resp, err := forwardedResponseIfInstanceIsRemote(d, r, projectName, cname, instanceType)
	if err != nil {
		return response.SmartError(err)
	}
	if resp != nil {
		return resp
	}

	recursion := util.IsRecursionRequest(r)
	resultString := []string{}
	resultMap := []*api.InstanceSnapshot{}

	if !recursion {
		snaps, err := d.cluster.GetInstanceSnapshotsNames(projectName, cname)
		if err != nil {
			return response.SmartError(err)
		}

		for _, snap := range snaps {
			_, snapName, _ := shared.InstanceGetParentAndSnapshotName(snap)
			if projectName == project.Default {
				url := fmt.Sprintf("/%s/instances/%s/snapshots/%s", version.APIVersion, cname, snapName)
				resultString = append(resultString, url)
			} else {
				url := fmt.Sprintf("/%s/instances/%s/snapshots/%s?project=%s", version.APIVersion, cname, snapName, projectName)
				resultString = append(resultString, url)
			}
		}
	} else {
		c, err := instance.LoadByProjectAndName(d.State(), projectName, cname)
		if err != nil {
			return response.SmartError(err)
		}

		snaps, err := c.Snapshots()
		if err != nil {
			return response.SmartError(err)
		}

		for _, snap := range snaps {
			render, _, err := snap.Render(storagePools.RenderSnapshotUsage(d.State(), snap))
			if err != nil {
				continue
			}

			resultMap = append(resultMap, render.(*api.InstanceSnapshot))
		}
	}

	if !recursion {
		return response.SyncResponse(true, resultString)
	}

	return response.SyncResponse(true, resultMap)
}

func instanceSnapshotsPost(d *Daemon, r *http.Request) response.Response {
	instanceType, err := urlInstanceTypeDetect(r)
	if err != nil {
		return response.SmartError(err)
	}

	project := projectParam(r)
	name := mux.Vars(r)["name"]

	// Handle requests targeted to a container on a different node
	resp, err := forwardedResponseIfInstanceIsRemote(d, r, project, name, instanceType)
	if err != nil {
		return response.SmartError(err)
	}
	if resp != nil {
		return resp
	}

	/*
	 * snapshot is a three step operation:
	 * 1. choose a new name
	 * 2. copy the database info over
	 * 3. copy over the rootfs
	 */
	inst, err := instance.LoadByProjectAndName(d.State(), project, name)
	if err != nil {
		return response.SmartError(err)
	}

	req := api.InstanceSnapshotsPost{}
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		return response.BadRequest(err)
	}

	if req.Name == "" {
		req.Name, err = instanceDetermineNextSnapshotName(d, inst, "snap%d")
		if err != nil {
			return response.SmartError(err)
		}
	}

	// Validate the name
	err = validate.IsURLSegmentSafe(req.Name)
	if err != nil {
		return response.BadRequest(errors.Wrap(err, "Invalid snapshot name"))
	}

	var expiry time.Time
	if req.ExpiresAt != nil {
		expiry = *req.ExpiresAt
	} else {
		expiry, err = shared.GetSnapshotExpiry(time.Now(), inst.ExpandedConfig()["snapshots.expiry"])
		if err != nil {
			return response.BadRequest(err)
		}
	}

	snapshot := func(op *operations.Operation) error {
		inst.SetOperation(op)
		return inst.Snapshot(req.Name, expiry, req.Stateful)
	}

	resources := map[string][]string{}
	resources["instances"] = []string{name}
	resources["containers"] = resources["instances"]

	op, err := operations.OperationCreate(d.State(), project, operations.OperationClassTask, db.OperationSnapshotCreate, resources, nil, snapshot, nil, nil)
	if err != nil {
		return response.InternalError(err)
	}

	return operations.OperationResponse(op)
}

func instanceSnapshotHandler(d *Daemon, r *http.Request) response.Response {
	instanceType, err := urlInstanceTypeDetect(r)
	if err != nil {
		return response.SmartError(err)
	}

	project := projectParam(r)
	containerName := mux.Vars(r)["name"]
	snapshotName := mux.Vars(r)["snapshotName"]

	resp, err := forwardedResponseIfInstanceIsRemote(d, r, project, containerName, instanceType)
	if err != nil {
		return response.SmartError(err)
	}
	if resp != nil {
		return resp
	}

	snapshotName, err = url.QueryUnescape(snapshotName)
	if err != nil {
		return response.SmartError(err)
	}
	inst, err := instance.LoadByProjectAndName(
		d.State(),
		project, containerName+
			shared.SnapshotDelimiter+
			snapshotName)
	if err != nil {
		return response.SmartError(err)
	}

	switch r.Method {
	case "GET":
		return snapshotGet(d.State(), inst, snapshotName)
	case "POST":
		return snapshotPost(d, r, inst, containerName)
	case "DELETE":
		return snapshotDelete(d.State(), inst, snapshotName)
	case "PUT":
		return snapshotPut(d, r, inst, snapshotName)
	case "PATCH":
		return snapshotPatch(d, r, inst, snapshotName)
	default:
		return response.NotFound(fmt.Errorf("Method '%s' not found", r.Method))
	}
}

func snapshotPatch(d *Daemon, r *http.Request, sc instance.Instance, name string) response.Response {
	// Only expires_at is currently editable, so PATCH is equivalent to PUT.
	return snapshotPut(d, r, sc, name)
}

func snapshotPut(d *Daemon, r *http.Request, sc instance.Instance, name string) response.Response {
	// Validate the ETag
	etag := []interface{}{sc.ExpiryDate()}
	err := util.EtagCheck(r, etag)
	if err != nil {
		return response.PreconditionFailed(err)
	}

	rj := shared.Jmap{}

	err = json.NewDecoder(r.Body).Decode(&rj)
	if err != nil {
		return response.InternalError(err)
	}

	var do func(op *operations.Operation) error

	_, err = rj.GetString("expires_at")
	if err != nil {
		// Skip updating the snapshot since the requested key wasn't provided
		do = func(op *operations.Operation) error {
			return nil
		}
	} else {
		body, err := json.Marshal(rj)
		if err != nil {
			return response.InternalError(err)
		}

		configRaw := api.InstanceSnapshotPut{}

		err = json.Unmarshal(body, &configRaw)
		if err != nil {
			return response.BadRequest(err)
		}

		// Update container configuration
		do = func(op *operations.Operation) error {
			args := db.InstanceArgs{
				Architecture: sc.Architecture(),
				Config:       sc.LocalConfig(),
				Description:  sc.Description(),
				Devices:      sc.LocalDevices(),
				Ephemeral:    sc.IsEphemeral(),
				Profiles:     sc.Profiles(),
				Project:      sc.Project(),
				ExpiryDate:   configRaw.ExpiresAt,
				Type:         sc.Type(),
				Snapshot:     sc.IsSnapshot(),
			}

			err = sc.Update(args, false)
			if err != nil {
				return err
			}

			return nil
		}
	}

	opType := db.OperationSnapshotUpdate

	resources := map[string][]string{}
	resources["containers"] = []string{name}

	op, err := operations.OperationCreate(d.State(), sc.Project(), operations.OperationClassTask, opType, resources, nil,
		do, nil, nil)
	if err != nil {
		return response.InternalError(err)
	}

	return operations.OperationResponse(op)
}

func snapshotGet(s *state.State, snapInst instance.Instance, name string) response.Response {
	render, _, err := snapInst.Render(storagePools.RenderSnapshotUsage(s, snapInst))
	if err != nil {
		return response.SmartError(err)
	}

	return response.SyncResponseETag(true, render.(*api.InstanceSnapshot), render.(*api.InstanceSnapshot))
}

func snapshotPost(d *Daemon, r *http.Request, sc instance.Instance, containerName string) response.Response {
	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return response.InternalError(err)
	}

	rdr1 := ioutil.NopCloser(bytes.NewBuffer(body))

	raw := shared.Jmap{}
	if err := json.NewDecoder(rdr1).Decode(&raw); err != nil {
		return response.BadRequest(err)
	}

	migration, err := raw.GetBool("migration")
	if err == nil && migration {
		rdr2 := ioutil.NopCloser(bytes.NewBuffer(body))
		rdr3 := ioutil.NopCloser(bytes.NewBuffer(body))

		req := api.InstancePost{}
		err = json.NewDecoder(rdr2).Decode(&req)
		if err != nil {
			return response.BadRequest(err)
		}

		reqNew := api.InstanceSnapshotPost{}
		err = json.NewDecoder(rdr3).Decode(&reqNew)
		if err != nil {
			return response.BadRequest(err)
		}

		if reqNew.Name == "" {
			return response.BadRequest(fmt.Errorf(`A new name for the ` +
				`container must be provided`))
		}

		if reqNew.Live {
			sourceName, _, _ := shared.InstanceGetParentAndSnapshotName(containerName)
			if sourceName != reqNew.Name {
				return response.BadRequest(fmt.Errorf(`Copying `+
					`stateful containers requires that `+
					`source "%s" and `+`target "%s" name `+
					`be identical`, sourceName, reqNew.Name))
			}
		}

		ws, err := newMigrationSource(sc, reqNew.Live, true)
		if err != nil {
			return response.SmartError(err)
		}

		resources := map[string][]string{}
		resources["containers"] = []string{containerName}

		run := func(op *operations.Operation) error {
			return ws.Do(d.State(), op)
		}

		if req.Target != nil {
			// Push mode
			err := ws.ConnectContainerTarget(*req.Target)
			if err != nil {
				return response.InternalError(err)
			}

			op, err := operations.OperationCreate(d.State(), sc.Project(), operations.OperationClassTask, db.OperationSnapshotTransfer, resources, nil, run, nil, nil)
			if err != nil {
				return response.InternalError(err)
			}

			return operations.OperationResponse(op)
		}

		// Pull mode
		op, err := operations.OperationCreate(d.State(), sc.Project(), operations.OperationClassWebsocket, db.OperationSnapshotTransfer, resources, ws.Metadata(), run, nil, ws.Connect)
		if err != nil {
			return response.InternalError(err)
		}

		return operations.OperationResponse(op)
	}

	newName, err := raw.GetString("name")
	if err != nil {
		return response.BadRequest(err)
	}

	// Validate the name
	err = validate.IsURLSegmentSafe(newName)
	if err != nil {
		return response.BadRequest(errors.Wrap(err, "Invalid snapshot name"))
	}

	fullName := containerName + shared.SnapshotDelimiter + newName

	// Check that the name isn't already in use
	id, _ := d.cluster.GetInstanceSnapshotID(sc.Project(), containerName, newName)
	if id > 0 {
		return response.Conflict(fmt.Errorf("Name '%s' already in use", fullName))
	}

	rename := func(op *operations.Operation) error {
		return sc.Rename(fullName, false)
	}

	resources := map[string][]string{}
	resources["containers"] = []string{containerName}

	op, err := operations.OperationCreate(d.State(), sc.Project(), operations.OperationClassTask, db.OperationSnapshotRename, resources, nil, rename, nil, nil)
	if err != nil {
		return response.InternalError(err)
	}

	return operations.OperationResponse(op)
}

func snapshotDelete(s *state.State, sc instance.Instance, name string) response.Response {
	remove := func(op *operations.Operation) error {
		return sc.Delete(false)
	}

	resources := map[string][]string{}
	resources["containers"] = []string{sc.Name()}

	op, err := operations.OperationCreate(s, sc.Project(), operations.OperationClassTask, db.OperationSnapshotDelete, resources, nil, remove, nil, nil)
	if err != nil {
		return response.InternalError(err)
	}

	return operations.OperationResponse(op)
}
<commit_msg>lxd/instance/snapshot: Renames sc to snapInst and makes containers resources conditional on instance type<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"

	"github.com/gorilla/mux"
	"github.com/pkg/errors"

	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	"github.com/lxc/lxd/lxd/operations"
	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/lxd/response"
	"github.com/lxc/lxd/lxd/state"
	storagePools "github.com/lxc/lxd/lxd/storage"
	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/validate"
	"github.com/lxc/lxd/shared/version"
)

func instanceSnapshotsGet(d *Daemon, r *http.Request) response.Response {
	instanceType, err := urlInstanceTypeDetect(r)
	if err != nil {
		return response.SmartError(err)
	}

	projectName := projectParam(r)
	cname := mux.Vars(r)["name"]

	// Handle requests targeted to a container on a different node
	resp, err := forwardedResponseIfInstanceIsRemote(d, r, projectName, cname, instanceType)
	if err != nil {
		return response.SmartError(err)
	}
	if resp != nil {
		return resp
	}

	recursion := util.IsRecursionRequest(r)
	resultString := []string{}
	resultMap := []*api.InstanceSnapshot{}

	if !recursion {
		snaps, err := d.cluster.GetInstanceSnapshotsNames(projectName, cname)
		if err != nil {
			return response.SmartError(err)
		}

		for _, snap := range snaps {
			_, snapName, _ := shared.InstanceGetParentAndSnapshotName(snap)
			if projectName == project.Default {
				url := fmt.Sprintf("/%s/instances/%s/snapshots/%s", version.APIVersion, cname, snapName)
				resultString = append(resultString, url)
			} else {
				url := fmt.Sprintf("/%s/instances/%s/snapshots/%s?project=%s", version.APIVersion, cname, snapName, projectName)
				resultString = append(resultString, url)
			}
		}
	} else {
		c, err := instance.LoadByProjectAndName(d.State(), projectName, cname)
		if err != nil {
			return response.SmartError(err)
		}

		snaps, err := c.Snapshots()
		if err != nil {
			return response.SmartError(err)
		}

		for _, snap := range snaps {
			render, _, err := snap.Render(storagePools.RenderSnapshotUsage(d.State(), snap))
			if err != nil {
				continue
			}

			resultMap = append(resultMap, render.(*api.InstanceSnapshot))
		}
	}

	if !recursion {
		return response.SyncResponse(true, resultString)
	}

	return response.SyncResponse(true, resultMap)
}

func instanceSnapshotsPost(d *Daemon, r *http.Request) response.Response {
	instanceType, err := urlInstanceTypeDetect(r)
	if err != nil {
		return response.SmartError(err)
	}

	project := projectParam(r)
	name := mux.Vars(r)["name"]

	// Handle requests targeted to a container on a different node
	resp, err := forwardedResponseIfInstanceIsRemote(d, r, project, name, instanceType)
	if err != nil {
		return response.SmartError(err)
	}
	if resp != nil {
		return resp
	}

	/*
	 * snapshot is a three step operation:
	 * 1. choose a new name
	 * 2. copy the database info over
	 * 3. copy over the rootfs
	 */
	inst, err := instance.LoadByProjectAndName(d.State(), project, name)
	if err != nil {
		return response.SmartError(err)
	}

	req := api.InstanceSnapshotsPost{}
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		return response.BadRequest(err)
	}

	if req.Name == "" {
		req.Name, err = instanceDetermineNextSnapshotName(d, inst, "snap%d")
		if err != nil {
			return response.SmartError(err)
		}
	}

	// Validate the name
	err = validate.IsURLSegmentSafe(req.Name)
	if err != nil {
		return response.BadRequest(errors.Wrap(err, "Invalid snapshot name"))
	}

	var expiry time.Time
	if req.ExpiresAt != nil {
		expiry = *req.ExpiresAt
	} else {
		expiry, err = shared.GetSnapshotExpiry(time.Now(), inst.ExpandedConfig()["snapshots.expiry"])
		if err != nil {
			return response.BadRequest(err)
		}
	}

	snapshot := func(op *operations.Operation) error {
		inst.SetOperation(op)
		return inst.Snapshot(req.Name, expiry, req.Stateful)
	}

	resources := map[string][]string{}
	resources["instances"] = []string{name}

	if inst.Type() == instancetype.Container {
		resources["containers"] = resources["instances"]
	}

	op, err := operations.OperationCreate(d.State(), project, operations.OperationClassTask, db.OperationSnapshotCreate, resources, nil, snapshot, nil, nil)
	if err != nil {
		return response.InternalError(err)
	}

	return operations.OperationResponse(op)
}

func instanceSnapshotHandler(d *Daemon, r *http.Request) response.Response {
	instanceType, err := urlInstanceTypeDetect(r)
	if err != nil {
		return response.SmartError(err)
	}

	project := projectParam(r)
	containerName := mux.Vars(r)["name"]
	snapshotName := mux.Vars(r)["snapshotName"]

	resp, err := forwardedResponseIfInstanceIsRemote(d, r, project, containerName, instanceType)
	if err != nil {
		return response.SmartError(err)
	}
	if resp != nil {
		return resp
	}

	snapshotName, err = url.QueryUnescape(snapshotName)
	if err != nil {
		return response.SmartError(err)
	}
	inst, err := instance.LoadByProjectAndName(
		d.State(),
		project, containerName+
			shared.SnapshotDelimiter+
			snapshotName)
	if err != nil {
		return response.SmartError(err)
	}

	switch r.Method {
	case "GET":
		return snapshotGet(d.State(), inst, snapshotName)
	case "POST":
		return snapshotPost(d, r, inst, containerName)
	case "DELETE":
		return snapshotDelete(d.State(), inst, snapshotName)
	case "PUT":
		return snapshotPut(d, r, inst, snapshotName)
	case "PATCH":
		return snapshotPatch(d, r, inst, snapshotName)
	default:
		return response.NotFound(fmt.Errorf("Method '%s' not found", r.Method))
	}
}

func snapshotPatch(d *Daemon, r *http.Request, snapInst instance.Instance, name string) response.Response {
	// Only expires_at is currently editable, so PATCH is equivalent to PUT.
	return snapshotPut(d, r, snapInst, name)
}

func snapshotPut(d *Daemon, r *http.Request, snapInst instance.Instance, name string) response.Response {
	// Validate the ETag
	etag := []interface{}{snapInst.ExpiryDate()}
	err := util.EtagCheck(r, etag)
	if err != nil {
		return response.PreconditionFailed(err)
	}

	rj := shared.Jmap{}

	err = json.NewDecoder(r.Body).Decode(&rj)
	if err != nil {
		return response.InternalError(err)
	}

	var do func(op *operations.Operation) error

	_, err = rj.GetString("expires_at")
	if err != nil {
		// Skip updating the snapshot since the requested key wasn't provided
		do = func(op *operations.Operation) error {
			return nil
		}
	} else {
		body, err := json.Marshal(rj)
		if err != nil {
			return response.InternalError(err)
		}

		configRaw := api.InstanceSnapshotPut{}

		err = json.Unmarshal(body, &configRaw)
		if err != nil {
			return response.BadRequest(err)
		}

		// Update instance configuration
		do = func(op *operations.Operation) error {
			args := db.InstanceArgs{
				Architecture: snapInst.Architecture(),
				Config:       snapInst.LocalConfig(),
				Description:  snapInst.Description(),
				Devices:      snapInst.LocalDevices(),
				Ephemeral:    snapInst.IsEphemeral(),
				Profiles:     snapInst.Profiles(),
				Project:      snapInst.Project(),
				ExpiryDate:   configRaw.ExpiresAt,
				Type:         snapInst.Type(),
				Snapshot:     snapInst.IsSnapshot(),
			}

			err = snapInst.Update(args, false)
			if err != nil {
				return err
			}

			return nil
		}
	}

	opType := db.OperationSnapshotUpdate

	resources := map[string][]string{}
	resources["instances"] = []string{name}

	if snapInst.Type() == instancetype.Container {
		resources["containers"] = resources["instances"]
	}

	op, err := operations.OperationCreate(d.State(), snapInst.Project(), operations.OperationClassTask, opType, resources, nil,
		do, nil, nil)
	if err != nil {
		return response.InternalError(err)
	}

	return operations.OperationResponse(op)
}

func snapshotGet(s *state.State, snapInst instance.Instance, name string) response.Response {
	render, _, err := snapInst.Render(storagePools.RenderSnapshotUsage(s, snapInst))
	if err != nil {
		return response.SmartError(err)
	}

	return response.SyncResponseETag(true, render.(*api.InstanceSnapshot), render.(*api.InstanceSnapshot))
}

func snapshotPost(d *Daemon, r *http.Request, snapInst instance.Instance, containerName string) response.Response {
	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return response.InternalError(err)
	}

	rdr1 := ioutil.NopCloser(bytes.NewBuffer(body))

	raw := shared.Jmap{}
	if err := json.NewDecoder(rdr1).Decode(&raw); err != nil {
		return response.BadRequest(err)
	}

	migration, err := raw.GetBool("migration")
	if err == nil && migration {
		rdr2 := ioutil.NopCloser(bytes.NewBuffer(body))
		rdr3 := ioutil.NopCloser(bytes.NewBuffer(body))

		req := api.InstancePost{}
		err = json.NewDecoder(rdr2).Decode(&req)
		if err != nil {
			return response.BadRequest(err)
		}

		reqNew := api.InstanceSnapshotPost{}
		err = json.NewDecoder(rdr3).Decode(&reqNew)
		if err != nil {
			return response.BadRequest(err)
		}

		if reqNew.Name == "" {
			return response.BadRequest(fmt.Errorf(`A new name for the ` +
				`container must be provided`))
		}

		if reqNew.Live {
			sourceName, _, _ := shared.InstanceGetParentAndSnapshotName(containerName)
			if sourceName != reqNew.Name {
				return response.BadRequest(fmt.Errorf(`Copying `+
					`stateful containers requires that `+
					`source "%s" and `+`target "%s" name `+
					`be identical`, sourceName, reqNew.Name))
			}
		}

		ws, err := newMigrationSource(snapInst, reqNew.Live, true)
		if err != nil {
			return response.SmartError(err)
		}

		resources := map[string][]string{}
		resources["instances"] = []string{containerName}

		if snapInst.Type() == instancetype.Container {
			resources["containers"] = resources["instances"]
		}

		run := func(op *operations.Operation) error {
			return ws.Do(d.State(), op)
		}

		if req.Target != nil {
			// Push mode
			err := ws.ConnectContainerTarget(*req.Target)
			if err != nil {
				return response.InternalError(err)
			}

			op, err := operations.OperationCreate(d.State(), snapInst.Project(), operations.OperationClassTask, db.OperationSnapshotTransfer, resources, nil, run, nil, nil)
			if err != nil {
				return response.InternalError(err)
			}

			return operations.OperationResponse(op)
		}

		// Pull mode
		op, err := operations.OperationCreate(d.State(), snapInst.Project(), operations.OperationClassWebsocket, db.OperationSnapshotTransfer, resources, ws.Metadata(), run, nil, ws.Connect)
		if err != nil {
			return response.InternalError(err)
		}

		return operations.OperationResponse(op)
	}

	newName, err := raw.GetString("name")
	if err != nil {
		return response.BadRequest(err)
	}

	// Validate the name
	err = validate.IsURLSegmentSafe(newName)
	if err != nil {
		return response.BadRequest(errors.Wrap(err, "Invalid snapshot name"))
	}

	fullName := containerName + shared.SnapshotDelimiter + newName

	// Check that the name isn't already in use
	id, _ := d.cluster.GetInstanceSnapshotID(snapInst.Project(), containerName, newName)
	if id > 0 {
		return response.Conflict(fmt.Errorf("Name '%s' already in use", fullName))
	}

	rename := func(op *operations.Operation) error {
		return snapInst.Rename(fullName, false)
	}

	resources := map[string][]string{}
	resources["instances"] = []string{containerName}

	if snapInst.Type() == instancetype.Container {
		resources["containers"] = resources["instances"]
	}

	op, err := operations.OperationCreate(d.State(), snapInst.Project(), operations.OperationClassTask, db.OperationSnapshotRename, resources, nil, rename, nil, nil)
	if err != nil {
		return response.InternalError(err)
	}

	return operations.OperationResponse(op)
}

func snapshotDelete(s *state.State, snapInst instance.Instance, name string) response.Response {
	remove := func(op *operations.Operation) error {
		return snapInst.Delete(false)
	}

	resources := map[string][]string{}
	resources["instances"] = []string{snapInst.Name()}

	if snapInst.Type() == instancetype.Container {
		resources["containers"] = resources["instances"]
	}

	op, err := operations.OperationCreate(s, snapInst.Project(), operations.OperationClassTask, db.OperationSnapshotDelete, resources, nil, remove, nil, nil)
	if err != nil {
		return response.InternalError(err)
	}

	return operations.OperationResponse(op)
}
<|endoftext|>


<commit_before>package app

import (
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/moncho/dry/appui"
	"github.com/moncho/dry/ui"
	"github.com/nsf/termbox-go"
)

type focusTracker struct {
	mutex sync.Locker
	focus bool
}

func (f *focusTracker) set(b bool) {
	f.mutex.Lock()
	defer f.mutex.Unlock()
	f.focus = b
}

func (f *focusTracker) hasFocus() bool {
	f.mutex.Lock()
	defer f.mutex.Unlock()
	return f.focus
}

func (f *focusTracker) flip() {
	f.mutex.Lock()
	defer f.mutex.Unlock()
	f.focus = !f.focus
}

//RenderLoop renders dry until it quits
func RenderLoop(dry *Dry, screen *ui.Screen) {
	if ok, _ := dry.Ok(); !ok {
		return
	}

	termuiEvents, done := ui.EventChannel()
	keyboardQueue := make(chan termbox.Event)

	timer := time.NewTicker(1 * time.Second)

	viewClosed := make(chan struct{})
	//On receive dry is rendered
	renderChan := make(chan struct{}, 1)

	keyboardQueueForView := make(chan termbox.Event)
	dryOutputChan := dry.OuputChannel()
	statusBar := ui.NewStatusBar(0, screen.Width, appui.DryTheme)
	eventHandlerFactory := &eventHandlerFactory{
		dry:                  dry,
		screen:               screen,
		keyboardQueueForView: keyboardQueueForView,
		viewClosed:           viewClosed,
		renderChan:           renderChan}

	defer timer.Stop()
	defer close(done)
	defer close(keyboardQueue)
	defer close(keyboardQueueForView)
	defer close(viewClosed)
	defer close(renderChan)

	//tracks if the main loop has the focus (and responds to events),
	//or if events have to be delegated.
	//creation belongs outside the loop
	focus := &focusTracker{&sync.Mutex{}, true}

	//renders dry on message until renderChan is closed
	go func() {
		for {
			_, ok := <-renderChan
			if ok {
				screen.Clear()
				Render(dry, screen, statusBar)
			} else {
				return
			}
		}
	}()

	renderChan <- struct{}{}

	//timer and status bar are shown if the main loop has the focus
	go func(focus *focusTracker) {
		for {
			select {
			case <-timer.C:
				if focus.hasFocus() {
					timestamp := time.Now().Format(`15:04:05`)
					screen.RenderLine(0, 0, `<right><white>`+timestamp+`</></right>`)
					screen.Flush()
				}
			case dryMessage, ok := <-dryOutputChan:
				if ok {
					if focus.hasFocus() {
						statusBar.StatusMessage(dryMessage, 10*time.Second)
						if dry.Changed() {
							renderChan <- struct{}{}
						} else {
							statusBar.Render()
						}
						screen.Flush()
					} else {
						//stop the status bar until the focus is retrieved
						statusBar.Stop()
					}
				} else {
					return
				}
			}
		}
	}(focus)

	go func() {
		for range viewClosed {
			focus.flip()
			dry.ShowMainView()
			renderChan <- struct{}{}
		}
	}()

	go func() {
		for event := range keyboardQueue {
			if focus.hasFocus() {
				handler := eventHandlerFactory.handlerFor(dry.viewMode())
				if handler != nil {
					handler.handle(event)
					focus.set(handler.hasFocus())
				} else {
					log.Panic("There is no event handler")
				}
			} else {
				//Whoever has the focus, handles the event
				select {
				case keyboardQueueForView <- event:
				default:
				}
			}
		}
	}()

	//main loop that handles termui events
loop:
	for event := range termuiEvents {
		switch event.Type {
		case termbox.EventInterrupt:
			break loop
		case termbox.EventKey:
			//Ctrl+C breaks the loop (and exits dry) no matter what
			if event.Key == termbox.KeyCtrlC || (focus.hasFocus() && (event.Ch == 'q' || event.Ch == 'Q')) {
				break loop
			} else {
				select {
				case keyboardQueue <- event:
				default:
				}
			}
		case termbox.EventResize:
			screen.Resize()
		}
	}

	log.Debug("something broke the loop. Time to die")
}
<commit_msg>Remove Flush call when rendering status bar<commit_after>package app

import (
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/moncho/dry/appui"
	"github.com/moncho/dry/ui"
	"github.com/nsf/termbox-go"
)

type focusTracker struct {
	mutex sync.Locker
	focus bool
}

func (f *focusTracker) set(b bool) {
	f.mutex.Lock()
	defer f.mutex.Unlock()
	f.focus = b
}

func (f *focusTracker) hasFocus() bool {
	f.mutex.Lock()
	defer f.mutex.Unlock()
	return f.focus
}

func (f *focusTracker) flip() {
	f.mutex.Lock()
	defer f.mutex.Unlock()
	f.focus = !f.focus
}

//RenderLoop renders dry until it quits
func RenderLoop(dry *Dry, screen *ui.Screen) {
	if ok, _ := dry.Ok(); !ok {
		return
	}

	termuiEvents, done := ui.EventChannel()
	keyboardQueue := make(chan termbox.Event)

	timer := time.NewTicker(1 * time.Second)

	viewClosed := make(chan struct{})
	//On receive dry is rendered
	renderChan := make(chan struct{}, 1)

	keyboardQueueForView := make(chan termbox.Event)
	dryOutputChan := dry.OuputChannel()
	statusBar := ui.NewStatusBar(0, screen.Width, appui.DryTheme)
	eventHandlerFactory := &eventHandlerFactory{
		dry:                  dry,
		screen:               screen,
		keyboardQueueForView: keyboardQueueForView,
		viewClosed:           viewClosed,
		renderChan:           renderChan}

	defer timer.Stop()
	defer close(done)
	defer close(keyboardQueue)
	defer close(keyboardQueueForView)
	defer close(viewClosed)
	defer close(renderChan)

	//tracks if the main loop has the focus (and responds to events),
	//or if events have to be delegated.
	//creation belongs outside the loop
	focus := &focusTracker{&sync.Mutex{}, true}

	//renders dry on message until renderChan is closed
	go func() {
		for {
			_, ok := <-renderChan
			if ok {
				screen.Clear()
				Render(dry, screen, statusBar)
			} else {
				return
			}
		}
	}()

	renderChan <- struct{}{}

	//timer and status bar are shown if the main loop has the focus
	go func(focus *focusTracker) {
		for {
			select {
			case <-timer.C:
				if focus.hasFocus() {
					timestamp := time.Now().Format(`15:04:05`)
					screen.RenderLine(0, 0, `<right><white>`+timestamp+`</></right>`)
					screen.Flush()
				}
			case dryMessage, ok := <-dryOutputChan:
				if ok {
					if focus.hasFocus() {
						statusBar.StatusMessage(dryMessage, 10*time.Second)
						if dry.Changed() {
							renderChan <- struct{}{}
						} else {
							statusBar.Render()
						}
					} else {
						//stop the status bar until the focus is retrieved
						statusBar.Stop()
					}
				} else {
					return
				}
			}
		}
	}(focus)

	go func() {
		for range viewClosed {
			focus.flip()
			dry.ShowMainView()
			renderChan <- struct{}{}
		}
	}()

	go func() {
		for event := range keyboardQueue {
			if focus.hasFocus() {
				handler := eventHandlerFactory.handlerFor(dry.viewMode())
				if handler != nil {
					handler.handle(event)
					focus.set(handler.hasFocus())
				} else {
					log.Panic("There is no event handler")
				}
			} else {
				//Whoever has the focus, handles the event
				select {
				case keyboardQueueForView <- event:
				default:
				}
			}
		}
	}()

	//main loop that handles termui events
loop:
	for event := range termuiEvents {
		switch event.Type {
		case termbox.EventInterrupt:
			break loop
		case termbox.EventKey:
			//Ctrl+C breaks the loop (and exits dry) no matter what
			if event.Key == termbox.KeyCtrlC || (focus.hasFocus() && (event.Ch == 'q' || event.Ch == 'Q')) {
				break loop
			} else {
				select {
				case keyboardQueue <- event:
				default:
				}
			}
		case termbox.EventResize:
			screen.Resize()
		}
	}

	log.Debug("something broke the loop. Time to die")
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"log/syslog"
	"math/rand"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"strconv"
	"syscall"
	"time"

	"github.com/weaveworks/scope/xfer"
)

// Set during buildtime.
var version = "unknown"

func main() {
	var (
		defaultProbes = []string{fmt.Sprintf("localhost:%d", xfer.ProbePort), fmt.Sprintf("scope.weave.local:%d", xfer.ProbePort)}
		logfile       = flag.String("log", "stderr", "stderr, syslog, or filename")
		batch         = flag.Duration("batch", 1*time.Second, "batch interval")
		window        = flag.Duration("window", 15*time.Second, "window")
		listen        = flag.String("http.address", ":"+strconv.Itoa(xfer.AppPort), "webserver listen address")
		printVersion  = flag.Bool("version", false, "print version number and exit")
	)
	flag.Parse()
	probes := append(defaultProbes, flag.Args()...)

	if *printVersion {
		fmt.Println(version)
		return
	}

	switch *logfile {
	case "stderr":
		break // by default

	case "syslog":
		w, err := syslog.New(syslog.LOG_INFO, "scope-app")
		if err != nil {
			log.Print(err)
			return
		}
		defer w.Close()
		log.SetFlags(0)
		log.SetOutput(w)

	default: // file
		f, err := os.OpenFile(*logfile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
		if err != nil {
			log.Print(err)
			return
		}
		defer f.Close()
		log.SetOutput(f)
	}

	id := strconv.FormatInt(rand.Int63(), 16)
	log.Printf("app starting, version %s, id %s", version, id)

	// Collector deals with the probes, and generates merged reports.
	c := xfer.NewCollector(*batch, id)
	defer c.Stop()

	r := newStaticResolver(probes, c.Add)
	defer r.Stop()

	lifo := NewReportLIFO(c, *window)
	defer lifo.Stop()

	http.Handle("/", Router(lifo))
	irq := interrupt()
	go func() {
		log.Printf("listening on %s", *listen)
		log.Print(http.ListenAndServe(*listen, nil))
		irq <- syscall.SIGINT
	}()
	<-irq
	log.Printf("shutting down")
}

func interrupt() chan os.Signal {
	c := make(chan os.Signal)
	signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
	return c
}
<commit_msg>app: consistent dev version with probe<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"log/syslog"
	"math/rand"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"strconv"
	"syscall"
	"time"

	"github.com/weaveworks/scope/xfer"
)

// Set during buildtime.
var version = "dev"

func main() {
	var (
		defaultProbes = []string{fmt.Sprintf("localhost:%d", xfer.ProbePort), fmt.Sprintf("scope.weave.local:%d", xfer.ProbePort)}
		logfile       = flag.String("log", "stderr", "stderr, syslog, or filename")
		batch         = flag.Duration("batch", 1*time.Second, "batch interval")
		window        = flag.Duration("window", 15*time.Second, "window")
		listen        = flag.String("http.address", ":"+strconv.Itoa(xfer.AppPort), "webserver listen address")
		printVersion  = flag.Bool("version", false, "print version number and exit")
	)
	flag.Parse()
	probes := append(defaultProbes, flag.Args()...)

	if *printVersion {
		fmt.Println(version)
		return
	}

	switch *logfile {
	case "stderr":
		break // by default

	case "syslog":
		w, err := syslog.New(syslog.LOG_INFO, "scope-app")
		if err != nil {
			log.Print(err)
			return
		}
		defer w.Close()
		log.SetFlags(0)
		log.SetOutput(w)

	default: // file
		f, err := os.OpenFile(*logfile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
		if err != nil {
			log.Print(err)
			return
		}
		defer f.Close()
		log.SetOutput(f)
	}

	id := strconv.FormatInt(rand.Int63(), 16)
	log.Printf("app starting, version %s, id %s", version, id)

	// Collector deals with the probes, and generates merged reports.
	c := xfer.NewCollector(*batch, id)
	defer c.Stop()

	r := newStaticResolver(probes, c.Add)
	defer r.Stop()

	lifo := NewReportLIFO(c, *window)
	defer lifo.Stop()

	http.Handle("/", Router(lifo))
	irq := interrupt()
	go func() {
		log.Printf("listening on %s", *listen)
		log.Print(http.ListenAndServe(*listen, nil))
		irq <- syscall.SIGINT
	}()
	<-irq
	log.Printf("shutting down")
}

func interrupt() chan os.Signal {
	c := make(chan os.Signal)
	signal.Notify(c, syscall.SIGINT, syscall.SIGTERM)
	return c
}
<|endoftext|>