<commit_before>// +-------------------------------------------------------------------------
// | Copyright (C) 2016 Yunify, Inc.
// +-------------------------------------------------------------------------
// | Licensed under the Apache License, Version 2.0 (the "License");
// | you may not use this work except in compliance with the License.
// | You may obtain a copy of the License in the LICENSE file, or at:
// |
// | http://www.apache.org/licenses/LICENSE-2.0
// |
// | Unless required by applicable law or agreed to in writing, software
// | distributed under the License is distributed on an "AS IS" BASIS,
// | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// | See the License for the specific language governing permissions and
// | limitations under the License.
// +-------------------------------------------------------------------------

package specs

import (
	"errors"
	"strings"

	"github.com/imdario/mergo"

	"github.com/go-openapi/loads"
	"github.com/go-openapi/spec"

	"github.com/yunify/snips/capsules"
)

// Swagger holds the data that to parse swagger spec.
type Swagger struct {
	FilePath string
	Data     *capsules.Data
}

// Parse parses swagger spec to data.
func (s *Swagger) Parse(version string) error {
	switch version {
	case "v2.0":
		document, err := loads.Spec(s.FilePath)
		if err != nil {
			return err
		}

		document, err = document.Expanded()
		if err != nil {
			return err
		}

		allProperties := map[string]*capsules.Property{}
		allOperations := map[string]map[string]*capsules.Operation{}
		s.parseOperations(document.Spec(), allProperties, allOperations)

		s.loadData(document.Spec())
		s.loadService(document.Spec(), allProperties, allOperations)
		s.loadSubService(document.Spec(), allProperties, allOperations)
		s.loadCustomizedTypes(document.Spec())
	default:
		return errors.New("Swagger version not supported: " + version)
	}

	return nil
}

func (s *Swagger) parseOperations(
	swagger *spec.Swagger,
	allProperties map[string]*capsules.Property,
	allOperations map[string]map[string]*capsules.Operation) {

	parseOperation := func(uri string, method string,
		specOperation *spec.Operation, property *capsules.Property) {
		if specOperation.ID == "PostObject" {
			return
		}

		sections := []string{}

		if len(specOperation.Tags) > 0 {
			for _, subServiceName := range specOperation.Tags {
				sections = append(sections, subServiceName+"SubService")
			}
		} else {
			serviceName := swagger.Info.Title
			sections = append(sections, serviceName+"Service")
		}

		for _, sectionName := range sections {
			if allProperties[sectionName] == nil {
				allProperties[sectionName] = property
			} else {
				mergo.Merge(allProperties[sectionName], property)
			}

			operation := s.parseOperation(uri, method, property, specOperation, swagger)
			if allOperations[sectionName] == nil {
				allOperations[sectionName] = map[string]*capsules.Operation{}
			}
			allOperations[sectionName][specOperation.ID] = operation
		}
	}

	for requestURI, pathItem := range swagger.Paths.Paths {
		property := &capsules.Property{
			Properties: map[string]*capsules.Property{},
		}

		for _, param := range pathItem.Parameters {
			paramProperty := s.parseParameter(&param, &swagger.Parameters)
			property.Properties[paramProperty.Name] = paramProperty
		}

		if pathItem.Get != nil {
			s.parseOperationsItems(parseOperation, requestURI, &pathItem, property)
		}
	}
}

func (s *Swagger) parseOperationsItems(
	parse func(uri string, method string, specOperation *spec.Operation, property *capsules.Property),
	requestURI string, item *spec.PathItem, property *capsules.Property,
) {
	if item.Get != nil {
		parse(requestURI, "GET", item.Get, property)
	}
	if item.Put != nil {
		parse(requestURI, "PUT", item.Put, property)
	}
	if item.Post != nil {
		parse(requestURI, "POST", item.Post, property)
	}
	if item.Delete != nil {
		parse(requestURI, "DELETE", item.Delete, property)
	}
	if item.Options != nil {
		parse(requestURI, "OPTIONS", item.Options, property)
	}
	if item.Head != nil {
		parse(requestURI, "HEAD", item.Head, property)
	}
	if item.Patch != nil {
		parse(requestURI, "PATCH", item.Patch, property)
	}
}

func (s *Swagger) loadData(swagger *spec.Swagger) {
	if s.Data == nil {
		s.Data = &capsules.Data{}
	}
	s.Data.Service = nil
	s.Data.SubServices = map[string]*capsules.SubService{}
	s.Data.CustomizedTypes = map[string]*capsules.Property{}
}

func (s *Swagger) loadService(
	swagger *spec.Swagger,
	allProperties map[string]*capsules.Property,
	allOperations map[string]map[string]*capsules.Operation) {
	serviceName := swagger.Info.Title

	property := &capsules.Property{}
	if allProperties[swagger.Info.Title] != nil {
		property = allProperties[serviceName]
	}

	s.Data.Service = &capsules.Service{
		APIVersion:  swagger.Info.Version,
		Name:        serviceName,
		BasePath:    swagger.BasePath,
		Description: swagger.Info.Description,
		Properties:  property,
		Operations:  allOperations[serviceName+"Service"],
	}

	// Be compatible with QingCloud IaaS Services
	if strings.Contains(s.Data.Service.Name, "QingCloud") {
		for _, o := range s.Data.Service.Operations {
			o.Request.Query, o.Request.Elements = o.Request.Elements, o.Request.Query
		}
	}
}

func (s *Swagger) loadSubService(
	swagger *spec.Swagger,
	allProperties map[string]*capsules.Property,
	allOperations map[string]map[string]*capsules.Operation) {

	for subService, operations := range allOperations {
		if strings.Contains(subService, "SubService") {
			subServiceName := strings.Replace(subService, "SubService", "", -1)
			s.Data.SubServices[subServiceName] = &capsules.SubService{
				ID:         subServiceName,
				Name:       subServiceName,
				Properties: allProperties[subService],
				Operations: operations,
			}

			// Be compatible with QingCloud IaaS Services
			if strings.Contains(s.Data.Service.Name, "QingCloud") {
				for _, o := range s.Data.SubServices[subServiceName].Operations {
					o.Request.Query, o.Request.Elements = o.Request.Elements, o.Request.Query
				}
			}
		}
	}
}

func (s *Swagger) loadCustomizedTypes(swagger *spec.Swagger) {
	for name, definition := range swagger.Definitions {
		s.Data.CustomizedTypes[name] = s.parseSchema(name, &definition)
		s.Data.CustomizedTypes[name].ID = name
		s.Data.CustomizedTypes[name].Name = name

		for _, schemaKey := range definition.SchemaProps.Required {
			s.Data.CustomizedTypes[name].Properties[schemaKey].IsRequired = true
		}
	}
}
<commit_msg>specs: Fix typo<commit_after>// +-------------------------------------------------------------------------
// | Copyright (C) 2016 Yunify, Inc.
// +-------------------------------------------------------------------------
// | Licensed under the Apache License, Version 2.0 (the "License");
// | you may not use this work except in compliance with the License.
// | You may obtain a copy of the License in the LICENSE file, or at:
// |
// | http://www.apache.org/licenses/LICENSE-2.0
// |
// | Unless required by applicable law or agreed to in writing, software
// | distributed under the License is distributed on an "AS IS" BASIS,
// | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// | See the License for the specific language governing permissions and
// | limitations under the License.
// +-------------------------------------------------------------------------

package specs

import (
	"errors"
	"strings"

	"github.com/imdario/mergo"

	"github.com/go-openapi/loads"
	"github.com/go-openapi/spec"

	"github.com/yunify/snips/capsules"
)

// Swagger holds the data that to parse swagger spec.
type Swagger struct {
	FilePath string
	Data     *capsules.Data
}

// Parse parses swagger spec to data.
func (s *Swagger) Parse(version string) error {
	switch version {
	case "v2.0":
		document, err := loads.Spec(s.FilePath)
		if err != nil {
			return err
		}

		document, err = document.Expanded()
		if err != nil {
			return err
		}

		allProperties := map[string]*capsules.Property{}
		allOperations := map[string]map[string]*capsules.Operation{}
		s.parseOperations(document.Spec(), allProperties, allOperations)

		s.loadData(document.Spec())
		s.loadService(document.Spec(), allProperties, allOperations)
		s.loadSubService(document.Spec(), allProperties, allOperations)
		s.loadCustomizedTypes(document.Spec())
	default:
		return errors.New("Swagger version not supported: " + version)
	}

	return nil
}

func (s *Swagger) parseOperations(
	swagger *spec.Swagger,
	allProperties map[string]*capsules.Property,
	allOperations map[string]map[string]*capsules.Operation) {

	parseOperation := func(uri string, method string,
		specOperation *spec.Operation, property *capsules.Property) {
		if specOperation.ID == "PostObject" {
			return
		}

		sections := []string{}

		if len(specOperation.Tags) > 0 {
			for _, subServiceName := range specOperation.Tags {
				sections = append(sections, subServiceName+"SubService")
			}
		} else {
			serviceName := swagger.Info.Title
			sections = append(sections, serviceName+"Service")
		}

		for _, sectionName := range sections {
			if allProperties[sectionName] == nil {
				allProperties[sectionName] = property
			} else {
				mergo.Merge(allProperties[sectionName], property)
			}

			operation := s.parseOperation(uri, method, property, specOperation, swagger)
			if allOperations[sectionName] == nil {
				allOperations[sectionName] = map[string]*capsules.Operation{}
			}
			allOperations[sectionName][specOperation.ID] = operation
		}
	}

	for requestURI, pathItem := range swagger.Paths.Paths {
		property := &capsules.Property{
			Properties: map[string]*capsules.Property{},
		}

		for _, param := range pathItem.Parameters {
			paramProperty := s.parseParameter(&param, &swagger.Parameters)
			property.Properties[paramProperty.Name] = paramProperty
		}

		s.parseOperationsItems(parseOperation, requestURI, &pathItem, property)
	}
}

func (s *Swagger) parseOperationsItems(
	parse func(uri string, method string, specOperation *spec.Operation, property *capsules.Property),
	requestURI string, item *spec.PathItem, property *capsules.Property,
) {
	if item.Get != nil {
		parse(requestURI, "GET", item.Get, property)
	}
	if item.Put != nil {
		parse(requestURI, "PUT", item.Put, property)
	}
	if item.Post != nil {
		parse(requestURI, "POST", item.Post, property)
	}
	if item.Delete != nil {
		parse(requestURI, "DELETE", item.Delete, property)
	}
	if item.Options != nil {
		parse(requestURI, "OPTIONS", item.Options, property)
	}
	if item.Head != nil {
		parse(requestURI, "HEAD", item.Head, property)
	}
	if item.Patch != nil {
		parse(requestURI, "PATCH", item.Patch, property)
	}
}

func (s *Swagger) loadData(swagger *spec.Swagger) {
	if s.Data == nil {
		s.Data = &capsules.Data{}
	}
	s.Data.Service = nil
	s.Data.SubServices = map[string]*capsules.SubService{}
	s.Data.CustomizedTypes = map[string]*capsules.Property{}
}

func (s *Swagger) loadService(
	swagger *spec.Swagger,
	allProperties map[string]*capsules.Property,
	allOperations map[string]map[string]*capsules.Operation) {
	serviceName := swagger.Info.Title

	property := &capsules.Property{}
	if allProperties[swagger.Info.Title] != nil {
		property = allProperties[serviceName]
	}

	s.Data.Service = &capsules.Service{
		APIVersion:  swagger.Info.Version,
		Name:        serviceName,
		BasePath:    swagger.BasePath,
		Description: swagger.Info.Description,
		Properties:  property,
		Operations:  allOperations[serviceName+"Service"],
	}

	// Be compatible with QingCloud IaaS Services
	if strings.Contains(s.Data.Service.Name, "QingCloud") {
		for _, o := range s.Data.Service.Operations {
			o.Request.Query, o.Request.Elements = o.Request.Elements, o.Request.Query
		}
	}
}

func (s *Swagger) loadSubService(
	swagger *spec.Swagger,
	allProperties map[string]*capsules.Property,
	allOperations map[string]map[string]*capsules.Operation) {

	for subService, operations := range allOperations {
		if strings.Contains(subService, "SubService") {
			subServiceName := strings.Replace(subService, "SubService", "", -1)
			s.Data.SubServices[subServiceName] = &capsules.SubService{
				ID:         subServiceName,
				Name:       subServiceName,
				Properties: allProperties[subService],
				Operations: operations,
			}

			// Be compatible with QingCloud IaaS Services
			if strings.Contains(s.Data.Service.Name, "QingCloud") {
				for _, o := range s.Data.SubServices[subServiceName].Operations {
					o.Request.Query, o.Request.Elements = o.Request.Elements, o.Request.Query
				}
			}
		}
	}
}

func (s *Swagger) loadCustomizedTypes(swagger *spec.Swagger) {
	for name, definition := range swagger.Definitions {
		s.Data.CustomizedTypes[name] = s.parseSchema(name, &definition)
		s.Data.CustomizedTypes[name].ID = name
		s.Data.CustomizedTypes[name].Name = name

		for _, schemaKey := range definition.SchemaProps.Required {
			s.Data.CustomizedTypes[name].Properties[schemaKey].IsRequired = true
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"path"
	"path/filepath"
	"time"
)

var running bool = true
var maxlogsize uint

func exists(filename string) bool {
	_, err := os.Stat(filename); if err != nil {
		return false
	}
	return true
}

func runsvdir(dirname string) {
	services := make(map[string]*Network)

	dir, err := os.Open(dirname)
	if err != nil {
		log.Fatal(err)
	}
	defer dir.Close()

	log.Printf("Starting in %s\n", dirname)
	for running {
		dn, err := dir.Readdirnames(0); if err != nil {
			log.Fatal(err)
		}

		found := make(map[string]bool)
		for _, fn := range dn {
			fpath := path.Join(dirname, fn)
			log.Print("Considering", fpath)
			if exists(path.Join(fpath, "down")) {
				continue
			}
			if _, ok := services[fpath]; ! ok {
				if ! exists(path.Join(fpath, "server")) {
					continue
				}

				log.Printf("Found new network %s", fpath)
				newnet := NewNetwork(fpath)
				services[fpath] = newnet
				go newnet.Connect()
			}
			found[fpath] = true
		}

		// If anything vanished, disconnect it
		for fpath, nw := range services {
			if _, ok := found[fpath]; ! ok {
				log.Printf("Removing vanished network %s", fpath)
				nw.Close()
			}
		}

		_, _ = dir.Seek(0, 0)
		time.Sleep(20 * time.Second)
	}
}

func usage() {
	fmt.Fprintf(os.Stderr, "Usage: %s [OPTIONS] BASEPATH\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "\n")
	fmt.Fprintf(os.Stderr, "BASEPATH is the path to your IRC directory (see README)\n")
	fmt.Fprintf(os.Stderr, "\n")
	fmt.Fprintf(os.Stderr, "OPTIONS:\n")
	flag.PrintDefaults()
}

func main() {
	flag.Usage = usage
	flag.UintVar(&maxlogsize, "logsize", 1000, "Log entries before rotating")
	notime := flag.Bool("notime", false, "Don't timestamp log messages")
	flag.Parse()
	if flag.NArg() != 1 {
		usage()
		os.Exit(2)
	}
	basePath, err := filepath.Abs(flag.Arg(0))
	if err != nil {
		log.Fatal(err)
	}
	if *notime {
		log.SetFlags(0)
	}

	runsvdir(basePath)

	running = false
}
<commit_msg>That was a bit too much logging<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"path"
	"path/filepath"
	"time"
)

var running bool = true
var maxlogsize uint

func exists(filename string) bool {
	_, err := os.Stat(filename); if err != nil {
		return false
	}
	return true
}

func runsvdir(dirname string) {
	services := make(map[string]*Network)

	dir, err := os.Open(dirname)
	if err != nil {
		log.Fatal(err)
	}
	defer dir.Close()

	log.Printf("Starting in %s\n", dirname)
	for running {
		dn, err := dir.Readdirnames(0); if err != nil {
			log.Fatal(err)
		}

		found := make(map[string]bool)
		for _, fn := range dn {
			fpath := path.Join(dirname, fn)
			if _, ok := services[fpath]; ! ok {
				if exists(path.Join(fpath, "down")) {
					continue
				}
				if ! exists(path.Join(fpath, "server")) {
					continue
				}

				log.Printf("Found new network %s", fpath)
				newnet := NewNetwork(fpath)
				services[fpath] = newnet
				go newnet.Connect()
			}
			found[fpath] = true
		}

		// If anything vanished, disconnect it
		for fpath, nw := range services {
			if _, ok := found[fpath]; ! ok {
				log.Printf("Removing vanished network %s", fpath)
				nw.Close()
			}
		}

		_, _ = dir.Seek(0, 0)
		time.Sleep(20 * time.Second)
	}
}

func usage() {
	fmt.Fprintf(os.Stderr, "Usage: %s [OPTIONS] BASEPATH\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "\n")
	fmt.Fprintf(os.Stderr, "BASEPATH is the path to your IRC directory (see README)\n")
	fmt.Fprintf(os.Stderr, "\n")
	fmt.Fprintf(os.Stderr, "OPTIONS:\n")
	flag.PrintDefaults()
}

func main() {
	flag.Usage = usage
	flag.UintVar(&maxlogsize, "logsize", 1000, "Log entries before rotating")
	notime := flag.Bool("notime", false, "Don't timestamp log messages")
	flag.Parse()
	if flag.NArg() != 1 {
		usage()
		os.Exit(2)
	}
	basePath, err := filepath.Abs(flag.Arg(0))
	if err != nil {
		log.Fatal(err)
	}
	if *notime {
		log.SetFlags(0)
	}

	runsvdir(basePath)

	running = false
}
<|endoftext|>


<commit_before>package instructions

import "jvmgo/rtda"

// Shift left int
type ishl struct {NoOperandsInstruction}
func (self *ishl) execute(thread *rtda.Thread) {
    stack := thread.CurrentFrame().OperandStack()
    v1 := stack.PopInt()
    v2 := stack.PopInt()
    s := uint32(v2) & 0x1f
    result := v1 << s
    stack.PushInt(result)
}

// Arithmetic shift right int
type ishr struct {NoOperandsInstruction}
func (self *ishr) execute(thread *rtda.Thread) {
    stack := thread.CurrentFrame().OperandStack()
    v1 := stack.PopInt()
    v2 := stack.PopInt()
    s := uint32(v2) & 0x1f
    result := v1 >> s // todo
    stack.PushInt(result)
}

// Logical shift right int
type iushr struct {NoOperandsInstruction}
func (self *iushr) execute(thread *rtda.Thread) {
    stack := thread.CurrentFrame().OperandStack()
    v1 := stack.PopInt()
    v2 := stack.PopInt()
    s := uint32(v2) & 0x1f
    result := v1 >> s
    stack.PushInt(result)
}
<commit_msg>fix iushr<commit_after>package instructions

import "jvmgo/rtda"

// Shift left int
type ishl struct {NoOperandsInstruction}
func (self *ishl) execute(thread *rtda.Thread) {
    stack := thread.CurrentFrame().OperandStack()
    v1 := stack.PopInt()
    v2 := stack.PopInt()
    s := uint32(v2) & 0x1f
    result := v1 << s
    stack.PushInt(result)
}

// Arithmetic shift right int
type ishr struct {NoOperandsInstruction}
func (self *ishr) execute(thread *rtda.Thread) {
    stack := thread.CurrentFrame().OperandStack()
    v1 := stack.PopInt()
    v2 := stack.PopInt()
    s := uint32(v2) & 0x1f
    result := v1 >> s
    stack.PushInt(result)
}

// Logical shift right int
type iushr struct {NoOperandsInstruction}
func (self *iushr) execute(thread *rtda.Thread) {
    stack := thread.CurrentFrame().OperandStack()
    v1 := stack.PopInt()
    v2 := stack.PopInt()
    s := uint32(v2) & 0x1f
    result := int32(uint32(v1) >> s)
    stack.PushInt(result)
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package os

import (
	"syscall";
	"os";
	"unsafe";
)

const (
	blockSize = 4096	// TODO(r): use statfs
)

func clen(n []byte) int {
	for i := 0; i < len(n); i++ {
		if n[i] == 0 {
			return i
		}
	}
	return len(n)
}

// Negative count means read until EOF.
func readdirnames(file *File, count int) (names []string, err *os.Error) {
	// If this file has no dirinfo, create one.
	if file.dirinfo == nil {
		file.dirinfo = new(dirInfo);
		// The buffer must be at least a block long.
		// TODO(r): use fstatfs to find fs block size.
		file.dirinfo.buf = make([]byte, blockSize);
	}
	d := file.dirinfo;
	size := count;
	if size < 0 {
		size = 100
	}
	names = make([]string, 0, size);	// Empty with room to grow.
	for count != 0 {
		// Refill the buffer if necessary
		if d.bufp == d.nbuf {
			var errno int64;
			dbuf := (*syscall.Dirent)(unsafe.Pointer(&d.buf[0]));
			d.nbuf, errno = syscall.Getdents(file.fd, dbuf, int64(len(d.buf)));
			if d.nbuf < 0 {
				return names, os.ErrnoToError(errno)
			}
			if d.nbuf == 0 {
				break	// EOF
			}
			d.bufp = 0;
		}
		// Drain the buffer
		for count != 0 && d.bufp < d.nbuf {
			dirent := (*syscall.Dirent)(unsafe.Pointer(&d.buf[d.bufp]));
			d.bufp += int64(dirent.Reclen);
			if dirent.Ino == 0 {	// File absent in directory.
				continue
			}
			var name = string(dirent.Name[0:clen(dirent.Namlen)]);
			if name == "." || name == ".." {	// Useless names
				continue
			}
			count--;
			if len(names) == cap(names) {
				nnames := make([]string, len(names), 2*len(names));
				for i := 0; i < len(names); i++ {
					nnames[i] = names[i]
				}
				names = nnames;
			}
			names = names[0:len(names)+1];
			names[len(names)-1] = name;
		}
	}
	return names, nil;
}
<commit_msg>fix typo breaking linux build<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package os

import (
	"syscall";
	"os";
	"unsafe";
)

const (
	blockSize = 4096	// TODO(r): use statfs
)

func clen(n []byte) int {
	for i := 0; i < len(n); i++ {
		if n[i] == 0 {
			return i
		}
	}
	return len(n)
}

// Negative count means read until EOF.
func readdirnames(file *File, count int) (names []string, err *os.Error) {
	// If this file has no dirinfo, create one.
	if file.dirinfo == nil {
		file.dirinfo = new(dirInfo);
		// The buffer must be at least a block long.
		// TODO(r): use fstatfs to find fs block size.
		file.dirinfo.buf = make([]byte, blockSize);
	}
	d := file.dirinfo;
	size := count;
	if size < 0 {
		size = 100
	}
	names = make([]string, 0, size);	// Empty with room to grow.
	for count != 0 {
		// Refill the buffer if necessary
		if d.bufp == d.nbuf {
			var errno int64;
			dbuf := (*syscall.Dirent)(unsafe.Pointer(&d.buf[0]));
			d.nbuf, errno = syscall.Getdents(file.fd, dbuf, int64(len(d.buf)));
			if d.nbuf < 0 {
				return names, os.ErrnoToError(errno)
			}
			if d.nbuf == 0 {
				break	// EOF
			}
			d.bufp = 0;
		}
		// Drain the buffer
		for count != 0 && d.bufp < d.nbuf {
			dirent := (*syscall.Dirent)(unsafe.Pointer(&d.buf[d.bufp]));
			d.bufp += int64(dirent.Reclen);
			if dirent.Ino == 0 {	// File absent in directory.
				continue
			}
			var name = string(dirent.Name[0:clen(dirent.Name)]);
			if name == "." || name == ".." {	// Useless names
				continue
			}
			count--;
			if len(names) == cap(names) {
				nnames := make([]string, len(names), 2*len(names));
				for i := 0; i < len(names); i++ {
					nnames[i] = names[i]
				}
				names = nnames;
			}
			names = names[0:len(names)+1];
			names[len(names)-1] = name;
		}
	}
	return names, nil;
}
<|endoftext|>


<commit_before>package tlock

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"sync"
	"testing"
	"time"

	. "gopkg.in/check.v1"
)

func Test(t *testing.T) {
	TestingT(t)
}

type serverTestSuite struct {
	a *App
}

var _ = Suite(&serverTestSuite{})

func (s *serverTestSuite) SetUpSuite(c *C) {
	s.a = NewApp()

	err := s.a.StartHTTP("127.0.0.1:0")
	c.Assert(err, IsNil)
}

func (s *serverTestSuite) TearDownSuite(c *C) {
	if s.a != nil {
		s.a.Close()
	}
}

func (s *serverTestSuite) getLocks(c *C) string {
	c.Assert(s.a.httpListener, NotNil)
	addr := s.a.httpListener.Addr()

	r, err := http.Get(fmt.Sprintf("http://%s/lock", addr))
	c.Assert(err, IsNil)

	defer r.Body.Close()
	c.Assert(r.StatusCode, Equals, http.StatusOK)
	b, err := ioutil.ReadAll(r.Body)
	c.Assert(err, IsNil)
	return string(b)
}

func (s *serverTestSuite) lock(c *C, names string, tp string, timeout int) uint64 {
	c.Assert(s.a.httpListener, NotNil)
	addr := s.a.httpListener.Addr()

	r, err := http.Post(fmt.Sprintf("http://%s/lock?names=%s&type=%s&timeout=%d", addr, url.QueryEscape(names), tp, timeout), "", strings.NewReader(""))
	c.Assert(err, IsNil)

	defer r.Body.Close()
	buf, err := ioutil.ReadAll(r.Body)
	c.Assert(err, IsNil)

	if timeout == 0 {
		c.Assert(r.StatusCode, Equals, http.StatusOK)
		id, err := strconv.ParseUint(string(buf), 10, 64)
		c.Assert(err, IsNil)
		return id
	} else {
		c.Assert(r.StatusCode, Equals, http.StatusRequestTimeout)
		return 0
	}
}

func (s *serverTestSuite) unlock(c *C, id uint64) {
	c.Assert(s.a.httpListener, NotNil)
	addr := s.a.HTTPAddr()

	req, _ := http.NewRequest("DELETE", fmt.Sprintf("http://%s/lock?id=%d", addr, id), nil)
	r, err := http.DefaultClient.Do(req)
	c.Assert(err, IsNil)

	defer r.Body.Close()
	ioutil.ReadAll(r.Body)

	c.Assert(r.StatusCode, Equals, http.StatusOK)
}

func (s *serverTestSuite) TestKeyLock(c *C) {
	var wg sync.WaitGroup

	wg.Add(1)

	names := "a,b"
	tp := "key"

	go func() {
		defer wg.Done()
		time.Sleep(500 * time.Millisecond)
		id := s.lock(c, names, tp, 0)
		s.unlock(c, id)
	}()

	id := s.lock(c, names, tp, 0)
	s.unlock(c, id)

	wg.Wait()
}

func (s *serverTestSuite) TestPathLock(c *C) {
	var wg sync.WaitGroup

	wg.Add(1)

	names := "a/b,a/c"
	tp := "path"

	go func() {
		defer wg.Done()
		time.Sleep(500 * time.Millisecond)
		id := s.lock(c, names, tp, 0)
		s.unlock(c, id)
	}()

	id := s.lock(c, names, tp, 0)
	s.unlock(c, id)

	wg.Wait()

	wg.Add(1)

	done := make(chan struct{})
	go func() {
		defer wg.Done()
		id := s.lock(c, names, tp, 0)
		done <- struct{}{}
		time.Sleep(2 * time.Second)
		done <- struct{}{}

		s.unlock(c, id)
	}()

	<-done
	id = s.lock(c, names, tp, 1)
	<-done

	id = s.lock(c, names, tp, 0)
	s.unlock(c, id)
	wg.Wait()
}

func (s *serverTestSuite) TestGetLock(c *C) {
	names := "a/b"
	tp := "key"

	id := s.lock(c, names, tp, 0)
	str := s.getLocks(c)
	c.Assert(strings.Contains(str, names), Equals, true)

	s.unlock(c, id)
	str = s.getLocks(c)
	c.Assert(strings.Contains(str, names), Equals, false)
}
<commit_msg>add RESP test<commit_after>package tlock

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/siddontang/goredis"
	. "gopkg.in/check.v1"
)

func Test(t *testing.T) {
	TestingT(t)
}

type serverTestSuite struct {
	a *App
}

var _ = Suite(&serverTestSuite{})

func (s *serverTestSuite) SetUpSuite(c *C) {
	s.a = NewApp()

	err := s.a.StartHTTP("127.0.0.1:0")
	c.Assert(err, IsNil)

	err = s.a.StartRESP("127.0.0.1:0")
	c.Assert(err, IsNil)
}

func (s *serverTestSuite) TearDownSuite(c *C) {
	if s.a != nil {
		s.a.Close()
	}
}

func (s *serverTestSuite) getLocks(c *C) string {
	c.Assert(s.a.httpListener, NotNil)
	addr := s.a.httpListener.Addr()

	r, err := http.Get(fmt.Sprintf("http://%s/lock", addr))
	c.Assert(err, IsNil)

	defer r.Body.Close()
	c.Assert(r.StatusCode, Equals, http.StatusOK)
	b, err := ioutil.ReadAll(r.Body)
	c.Assert(err, IsNil)
	return string(b)
}

func (s *serverTestSuite) lock(c *C, names string, tp string, timeout int) uint64 {
	c.Assert(s.a.httpListener, NotNil)
	addr := s.a.httpListener.Addr()

	r, err := http.Post(fmt.Sprintf("http://%s/lock?names=%s&type=%s&timeout=%d", addr, url.QueryEscape(names), tp, timeout), "", strings.NewReader(""))
	c.Assert(err, IsNil)

	defer r.Body.Close()
	buf, err := ioutil.ReadAll(r.Body)
	c.Assert(err, IsNil)

	if timeout == 0 {
		c.Assert(r.StatusCode, Equals, http.StatusOK)
		id, err := strconv.ParseUint(string(buf), 10, 64)
		c.Assert(err, IsNil)
		return id
	} else {
		c.Assert(r.StatusCode, Equals, http.StatusRequestTimeout)
		return 0
	}
}

func (s *serverTestSuite) unlock(c *C, id uint64) {
	c.Assert(s.a.httpListener, NotNil)
	addr := s.a.HTTPAddr()

	req, _ := http.NewRequest("DELETE", fmt.Sprintf("http://%s/lock?id=%d", addr, id), nil)
	r, err := http.DefaultClient.Do(req)
	c.Assert(err, IsNil)

	defer r.Body.Close()
	ioutil.ReadAll(r.Body)

	c.Assert(r.StatusCode, Equals, http.StatusOK)
}

func (s *serverTestSuite) TestKeyLock(c *C) {
	var wg sync.WaitGroup

	wg.Add(1)

	names := "a,b"
	tp := "key"

	go func() {
		defer wg.Done()
		time.Sleep(500 * time.Millisecond)
		id := s.lock(c, names, tp, 0)
		s.unlock(c, id)
	}()

	id := s.lock(c, names, tp, 0)
	s.unlock(c, id)

	wg.Wait()
}

func (s *serverTestSuite) TestPathLock(c *C) {
	var wg sync.WaitGroup

	wg.Add(1)

	names := "a/b,a/c"
	tp := "path"

	go func() {
		defer wg.Done()
		time.Sleep(500 * time.Millisecond)
		id := s.lock(c, names, tp, 0)
		s.unlock(c, id)
	}()

	id := s.lock(c, names, tp, 0)
	s.unlock(c, id)

	wg.Wait()

	wg.Add(1)

	done := make(chan struct{})
	go func() {
		defer wg.Done()
		id := s.lock(c, names, tp, 0)
		done <- struct{}{}
		time.Sleep(2 * time.Second)
		done <- struct{}{}

		s.unlock(c, id)
	}()

	<-done
	id = s.lock(c, names, tp, 1)
	<-done

	id = s.lock(c, names, tp, 0)
	s.unlock(c, id)
	wg.Wait()
}

func (s *serverTestSuite) TestGetLock(c *C) {
	names := "a/b"
	tp := "key"

	id := s.lock(c, names, tp, 0)
	str := s.getLocks(c)
	c.Assert(strings.Contains(str, names), Equals, true)

	s.unlock(c, id)
	str = s.getLocks(c)
	c.Assert(strings.Contains(str, names), Equals, false)
}

func (s *serverTestSuite) TestRESPLock(c *C) {
	addr := s.a.RESPAddr()
	c.Assert(addr, NotNil)

	c1, err := goredis.Connect(addr.String())
	c.Assert(addr, NotNil)
	defer c1.Close()

	c2, err := goredis.Connect(addr.String())
	c.Assert(addr, NotNil)
	defer c2.Close()

	var wg sync.WaitGroup

	wg.Add(1)

	done := make(chan struct{})
	go func() {
		defer wg.Done()
		id, err := goredis.Bytes(c2.Do("LOCK", "a", "TYPE", "KEY", "TIMEOUT", 0))
		c.Assert(err, IsNil)

		done <- struct{}{}
		time.Sleep(2 * time.Second)
		done <- struct{}{}

		_, err = c2.Do("UNLOCK", id)
		c.Assert(err, IsNil)
	}()

	<-done
	_, err = c1.Do("LOCK", "a", "TYPE", "KEY", "TIMEOUT", 1)
	<-done

	c.Assert(err, NotNil)
	c.Assert(strings.Contains(err.Error(), errLockTimeout.Error()), Equals, true)

	id, err := goredis.Bytes(c1.Do("LOCK", "a", "TYPE", "KEY", "TIMEOUT", 0))
	c.Assert(err, IsNil)
	_, err = c1.Do("UNLOCK", id)
	c.Assert(err, IsNil)

	wg.Wait()
}

func (s *serverTestSuite) TestRESPLockClose(c *C) {
	addr := s.a.RESPAddr()
	c.Assert(addr, NotNil)

	c1, err := goredis.Connect(addr.String())
	c.Assert(addr, NotNil)
	defer c1.Close()

	c2, err := goredis.Connect(addr.String())
	c.Assert(addr, NotNil)
	defer c2.Close()

	var wg sync.WaitGroup

	wg.Add(1)

	done := make(chan struct{})
	go func() {
		defer wg.Done()
		_, err := goredis.Bytes(c2.Do("LOCK", "a", "TYPE", "KEY", "TIMEOUT", 0))
		c.Assert(err, IsNil)

		done <- struct{}{}
		time.Sleep(2 * time.Second)
		done <- struct{}{}

		c2.Close()
	}()

	<-done
	_, err = c1.Do("LOCK", "a", "TYPE", "KEY", "TIMEOUT", 1)
	<-done

	c.Assert(err, NotNil)
	c.Assert(strings.Contains(err.Error(), errLockTimeout.Error()), Equals, true)

	id, err := goredis.Bytes(c1.Do("LOCK", "a", "TYPE", "KEY", "TIMEOUT", 0))
	c.Assert(err, IsNil)
	_, err = c1.Do("UNLOCK", id)
	c.Assert(err, IsNil)

	wg.Wait()
}
<|endoftext|>


<commit_before>package api

const (
	// JobTypeService indicates a long-running processes
	JobTypeService = "service"

	// JobTypeBatch indicates a short-lived process
	JobTypeBatch = "batch"
)

// Jobs is used to access the job-specific endpoints.
type Jobs struct {
	client *Client
}

// Jobs returns a handle on the jobs endpoints.
func (c *Client) Jobs() *Jobs {
	return &Jobs{client: c}
}

// Register is used to register a new job. It returns the ID
// of the evaluation, along with any errors encountered.
func (j *Jobs) Register(job *Job, q *WriteOptions) (string, *WriteMeta, error) {
	var resp registerJobResponse

	req := &registerJobRequest{job}
	wm, err := j.client.write("/v1/jobs", req, &resp, q)
	if err != nil {
		return "", nil, err
	}
	return resp.EvalID, wm, nil
}

// List is used to list all of the existing jobs.
func (j *Jobs) List(q *QueryOptions) ([]*JobListStub, *QueryMeta, error) {
	var resp []*JobListStub
	qm, err := j.client.query("/v1/jobs", &resp, q)
	if err != nil {
		return nil, qm, err
	}
	return resp, qm, nil
}

// Info is used to retrieve information about a particular
// job given its unique ID.
func (j *Jobs) Info(jobID string, q *QueryOptions) (*Job, *QueryMeta, error) {
	var resp Job
	qm, err := j.client.query("/v1/job/"+jobID, &resp, q)
	if err != nil {
		return nil, nil, err
	}
	return &resp, qm, nil
}

// Allocations is used to return the allocs for a given job ID.
func (j *Jobs) Allocations(jobID string, q *QueryOptions) ([]*AllocationListStub, *QueryMeta, error) {
	var resp []*AllocationListStub
	qm, err := j.client.query("/v1/job/"+jobID+"/allocations", &resp, q)
	if err != nil {
		return nil, nil, err
	}
	return resp, qm, nil
}

// Evaluations is used to query the evaluations associated with
// the given job ID.
func (j *Jobs) Evaluations(jobID string, q *QueryOptions) ([]*Evaluation, *QueryMeta, error) {
	var resp []*Evaluation
	qm, err := j.client.query("/v1/job/"+jobID+"/evaluations", &resp, q)
	if err != nil {
		return nil, nil, err
	}
	return resp, qm, nil
}

// Delete is used to remove an existing job.
func (j *Jobs) Delete(jobID string, q *WriteOptions) (*WriteMeta, error) {
	wm, err := j.client.delete("/v1/job/"+jobID, nil, q)
	if err != nil {
		return nil, err
	}
	return wm, nil
}

// ForceEvaluate is used to force-evaluate an existing job.
func (j *Jobs) ForceEvaluate(jobID string, q *WriteOptions) (string, *WriteMeta, error) {
	var resp registerJobResponse
	wm, err := j.client.write("/v1/job/"+jobID+"/evaluate", nil, &resp, q)
	if err != nil {
		return "", nil, err
	}
	return resp.EvalID, wm, nil
}

// Job is used to serialize a job.
type Job struct {
	ID                string
	Name              string
	Type              string
	Priority          int
	AllAtOnce         bool
	Datacenters       []string
	Constraints       []*Constraint
	TaskGroups        []*TaskGroup
	Meta              map[string]string
	Status            string
	StatusDescription string
	CreateIndex       uint64
	ModifyIndex       uint64
}

// JobListStub is used to return a subset of information about
// jobs during list operations.
type JobListStub struct {
	ID                string
	Name              string
	Type              string
	Priority          int
	Status            string
	StatusDescription string
	CreateIndex       uint64
	ModifyIndex       uint64
}

// NewServiceJob creates and returns a new service-style job
// for long-lived processes using the provided name, ID, and
// relative job priority.
func NewServiceJob(id, name string, pri int) *Job {
	return newJob(id, name, JobTypeService, pri)
}

// NewBatchJob creates and returns a new batch-style job for
// short-lived processes using the provided name and ID along
// with the relative job priority.
func NewBatchJob(id, name string, pri int) *Job {
	return newJob(id, name, JobTypeBatch, pri)
}

// newJob is used to create a new Job struct.
func newJob(jobID, jobName, jobType string, pri int) *Job {
	return &Job{
		ID:       jobID,
		Name:     jobName,
		Type:     jobType,
		Priority: pri,
	}
}

// SetMeta is used to set arbitrary k/v pairs of metadata on a job.
func (j *Job) SetMeta(key, val string) *Job {
	if j.Meta == nil {
		j.Meta = make(map[string]string)
	}
	j.Meta[key] = val
	return j
}

// AddDatacenter is used to add a datacenter to a job.
func (j *Job) AddDatacenter(dc string) *Job {
	j.Datacenters = append(j.Datacenters, dc)
	return j
}

// Constrain is used to add a constraint to a job.
func (j *Job) Constrain(c *Constraint) *Job {
	j.Constraints = append(j.Constraints, c)
	return j
}

// AddTaskGroup adds a task group to an existing job.
func (j *Job) AddTaskGroup(grp *TaskGroup) *Job {
	j.TaskGroups = append(j.TaskGroups, grp)
	return j
}

// registerJobRequest is used to serialize a job registration
type registerJobRequest struct {
	Job *Job
}

// registerJobResponse is used to deserialize a job response
type registerJobResponse struct {
	EvalID string
}
<commit_msg>api: add region to jobs<commit_after>package api

const (
	// JobTypeService indicates a long-running processes
	JobTypeService = "service"

	// JobTypeBatch indicates a short-lived process
	JobTypeBatch = "batch"
)

// Jobs is used to access the job-specific endpoints.
type Jobs struct {
	client *Client
}

// Jobs returns a handle on the jobs endpoints.
func (c *Client) Jobs() *Jobs {
	return &Jobs{client: c}
}

// Register is used to register a new job. It returns the ID
// of the evaluation, along with any errors encountered.
func (j *Jobs) Register(job *Job, q *WriteOptions) (string, *WriteMeta, error) {
	var resp registerJobResponse

	req := &registerJobRequest{job}
	wm, err := j.client.write("/v1/jobs", req, &resp, q)
	if err != nil {
		return "", nil, err
	}
	return resp.EvalID, wm, nil
}

// List is used to list all of the existing jobs.
func (j *Jobs) List(q *QueryOptions) ([]*JobListStub, *QueryMeta, error) {
	var resp []*JobListStub
	qm, err := j.client.query("/v1/jobs", &resp, q)
	if err != nil {
		return nil, qm, err
	}
	return resp, qm, nil
}

// Info is used to retrieve information about a particular
// job given its unique ID.
func (j *Jobs) Info(jobID string, q *QueryOptions) (*Job, *QueryMeta, error) {
	var resp Job
	qm, err := j.client.query("/v1/job/"+jobID, &resp, q)
	if err != nil {
		return nil, nil, err
	}
	return &resp, qm, nil
}

// Allocations is used to return the allocs for a given job ID.
func (j *Jobs) Allocations(jobID string, q *QueryOptions) ([]*AllocationListStub, *QueryMeta, error) {
	var resp []*AllocationListStub
	qm, err := j.client.query("/v1/job/"+jobID+"/allocations", &resp, q)
	if err != nil {
		return nil, nil, err
	}
	return resp, qm, nil
}

// Evaluations is used to query the evaluations associated with
// the given job ID.
func (j *Jobs) Evaluations(jobID string, q *QueryOptions) ([]*Evaluation, *QueryMeta, error) {
	var resp []*Evaluation
	qm, err := j.client.query("/v1/job/"+jobID+"/evaluations", &resp, q)
	if err != nil {
		return nil, nil, err
	}
	return resp, qm, nil
}

// Delete is used to remove an existing job.
func (j *Jobs) Delete(jobID string, q *WriteOptions) (*WriteMeta, error) {
	wm, err := j.client.delete("/v1/job/"+jobID, nil, q)
	if err != nil {
		return nil, err
	}
	return wm, nil
}

// ForceEvaluate is used to force-evaluate an existing job.
func (j *Jobs) ForceEvaluate(jobID string, q *WriteOptions) (string, *WriteMeta, error) {
	var resp registerJobResponse
	wm, err := j.client.write("/v1/job/"+jobID+"/evaluate", nil, &resp, q)
	if err != nil {
		return "", nil, err
	}
	return resp.EvalID, wm, nil
}

// Job is used to serialize a job.
type Job struct {
	Region            string
	ID                string
	Name              string
	Type              string
	Priority          int
	AllAtOnce         bool
	Datacenters       []string
	Constraints       []*Constraint
	TaskGroups        []*TaskGroup
	Meta              map[string]string
	Status            string
	StatusDescription string
	CreateIndex       uint64
	ModifyIndex       uint64
}

// JobListStub is used to return a subset of information about
// jobs during list operations.
type JobListStub struct {
	ID                string
	Name              string
	Type              string
	Priority          int
	Status            string
	StatusDescription string
	CreateIndex       uint64
	ModifyIndex       uint64
}

// NewServiceJob creates and returns a new service-style job
// for long-lived processes using the provided name, ID, and
// relative job priority.
func NewServiceJob(id, name string, pri int) *Job {
	return newJob(id, name, JobTypeService, pri)
}

// NewBatchJob creates and returns a new batch-style job for
// short-lived processes using the provided name and ID along
// with the relative job priority.
func NewBatchJob(id, name string, pri int) *Job {
	return newJob(id, name, JobTypeBatch, pri)
}

// newJob is used to create a new Job struct.
func newJob(jobID, jobName, jobType string, pri int) *Job {
	return &Job{
		ID:       jobID,
		Name:     jobName,
		Type:     jobType,
		Priority: pri,
	}
}

// SetMeta is used to set arbitrary k/v pairs of metadata on a job.
func (j *Job) SetMeta(key, val string) *Job {
	if j.Meta == nil {
		j.Meta = make(map[string]string)
	}
	j.Meta[key] = val
	return j
}

// AddDatacenter is used to add a datacenter to a job.
func (j *Job) AddDatacenter(dc string) *Job {
	j.Datacenters = append(j.Datacenters, dc)
	return j
}

// Constrain is used to add a constraint to a job.
func (j *Job) Constrain(c *Constraint) *Job {
	j.Constraints = append(j.Constraints, c)
	return j
}

// AddTaskGroup adds a task group to an existing job.
func (j *Job) AddTaskGroup(grp *TaskGroup) *Job {
	j.TaskGroups = append(j.TaskGroups, grp)
	return j
}

// registerJobRequest is used to serialize a job registration
type registerJobRequest struct {
	Job *Job
}

// registerJobResponse is used to deserialize a job response
type registerJobResponse struct {
	EvalID string
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"net/http"
	"os"
	"os/signal"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"

	"github.com/carlpett/stream_exporter/input"
	"github.com/carlpett/stream_exporter/linemetrics"
)

var (
	lineProcessingTime = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: "stream_exporter",
			Subsystem: "line_processing",
			Name:      "duration_seconds",
			Help:      "Observed duration, in seconds, of processing a single line per registered metric",
			Buckets:   prometheus.ExponentialBuckets(time.Microsecond.Seconds(), 3.981072, 5),
			// This results in 5 buckets from 1 us to approx 1 ms (3.98...^5 ~= 1000)
		},
		[]string{"metric"},
	)
	totalLines = prometheus.NewCounter(
		prometheus.CounterOpts{
			Namespace: "stream_exporter",
			Subsystem: "line_processing",
			Name:      "lines_total",
			Help:      "Number of lines processed",
		},
	)
)

var (
	configFilePath = flag.String("config", "stream_exporter.yaml", "Path to config file")

	inputType      = flag.String("input.type", "", "What input module to use")
	listInputTypes = flag.Bool("input.print", false, "Print available input modules and exit")

	metricsListenAddr = flag.String("web.listen-address", ":9178", "Address on which to expose metrics")
	metricsPath       = flag.String("web.metrics-path", "/metrics", "Path under which the metrics are available")
)

func main() {
	flag.Parse()

	if *listInputTypes {
		fmt.Println(input.GetAvailableInputs())
		os.Exit(0)
	}
	if *inputType == "" {
		fmt.Printf("-input.type is required. The following input types are available:\n%v", input.GetAvailableInputs())
		os.Exit(1)
	}

	metricsConfig, err := linemetrics.ReadPatternConfig(*configFilePath)
	if err != nil {
		log.Fatalf("Could not read pattern config: %v", err)
	}

	// Define metrics
	metrics := make([]linemetrics.LineMetric, 0, len(metricsConfig))
	for _, definition := range metricsConfig {
		lineMetric, collector := linemetrics.NewLineMetric(definition)
		metrics = append(metrics, lineMetric)
		prometheus.MustRegister(collector)
	}

	prometheus.MustRegister(lineProcessingTime)
	prometheus.MustRegister(totalLines)

	// Setup signal handling
	quitSig := make(chan os.Signal, 1)
	signal.Notify(quitSig, os.Interrupt)

	// Configure input
	inputReader, err := input.NewInput(*inputType)
	if err != nil {
		log.Fatalf("Could not initialize input: %v", err)
	}

	inputChannel := make(chan string)
	go inputReader.StartStream(inputChannel)

	// Setup http server
	http.Handle(*metricsPath, promhttp.Handler())
	go http.ListenAndServe(*metricsListenAddr, nil)

	// Main loop
	done := false
	for !done {
		select {
		case line, ok := <-inputChannel:
			if !ok {
				done = true
				break
			}
			for _, m := range metrics {
				t := time.Now()
				m.MatchLine(line)
				lineProcessingTime.WithLabelValues(m.Name()).Observe(time.Since(t).Seconds())
			}
			totalLines.Inc()
		case <-quitSig:
			log.Info("Received quit signal, shutting down...")
			done = true
			break
		}
	}
}
<commit_msg>Add log message when endpoint is started<commit_after>package main

import (
	"flag"
	"fmt"
	"net/http"
	"os"
	"os/signal"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"

	"github.com/carlpett/stream_exporter/input"
	"github.com/carlpett/stream_exporter/linemetrics"
)

var (
	lineProcessingTime = prometheus.NewHistogramVec(
		prometheus.HistogramOpts{
			Namespace: "stream_exporter",
			Subsystem: "line_processing",
			Name:      "duration_seconds",
			Help:      "Observed duration, in seconds, of processing a single line per registered metric",
			Buckets:   prometheus.ExponentialBuckets(time.Microsecond.Seconds(), 3.981072, 5),
			// This results in 5 buckets from 1 us to approx 1 ms (3.98...^5 ~= 1000)
		},
		[]string{"metric"},
	)
	totalLines = prometheus.NewCounter(
		prometheus.CounterOpts{
			Namespace: "stream_exporter",
			Subsystem: "line_processing",
			Name:      "lines_total",
			Help:      "Number of lines processed",
		},
	)
)

var (
	configFilePath = flag.String("config", "stream_exporter.yaml", "Path to config file")

	inputType      = flag.String("input.type", "", "What input module to use")
	listInputTypes = flag.Bool("input.print", false, "Print available input modules and exit")

	metricsListenAddr = flag.String("web.listen-address", ":9178", "Address on which to expose metrics")
	metricsPath       = flag.String("web.metrics-path", "/metrics", "Path under which the metrics are available")
)

func main() {
	flag.Parse()

	if *listInputTypes {
		fmt.Println(input.GetAvailableInputs())
		os.Exit(0)
	}
	if *inputType == "" {
		fmt.Printf("-input.type is required. The following input types are available:\n%v", input.GetAvailableInputs())
		os.Exit(1)
	}

	metricsConfig, err := linemetrics.ReadPatternConfig(*configFilePath)
	if err != nil {
		log.Fatalf("Could not read pattern config: %v", err)
	}

	// Define metrics
	metrics := make([]linemetrics.LineMetric, 0, len(metricsConfig))
	for _, definition := range metricsConfig {
		lineMetric, collector := linemetrics.NewLineMetric(definition)
		metrics = append(metrics, lineMetric)
		prometheus.MustRegister(collector)
	}

	prometheus.MustRegister(lineProcessingTime)
	prometheus.MustRegister(totalLines)

	// Setup signal handling
	quitSig := make(chan os.Signal, 1)
	signal.Notify(quitSig, os.Interrupt)

	// Configure input
	inputReader, err := input.NewInput(*inputType)
	if err != nil {
		log.Fatalf("Could not initialize input: %v", err)
	}

	inputChannel := make(chan string)
	go inputReader.StartStream(inputChannel)

	// Setup http server
	http.Handle(*metricsPath, promhttp.Handler())
	go http.ListenAndServe(*metricsListenAddr, nil)
	log.Infof("Serving metrics on %s%s", *metricsListenAddr, *metricsPath)

	// Main loop
	done := false
	for !done {
		select {
		case line, ok := <-inputChannel:
			if !ok {
				done = true
				break
			}
			for _, m := range metrics {
				t := time.Now()
				m.MatchLine(line)
				lineProcessingTime.WithLabelValues(m.Name()).Observe(time.Since(t).Seconds())
			}
			totalLines.Inc()
		case <-quitSig:
			log.Info("Received quit signal, shutting down...")
			done = true
			break
		}
	}
}
<|endoftext|>


<commit_before>package ext4

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"io/ioutil"
)

func (er Reader) Root() (Directory, error) {
	inode, err := er.GetInode(2)
	if err != nil {
		return Directory{}, err
	}
	return Directory{
		r:     er,
		inode: inode,
		path:  "/",
	}, nil
}

type Directory struct {
	r     Reader
	inode Inode
	path  string
}

func (d Directory) Entries() ([]DirEntry, error) {
	var entries []DirEntry
	b, err := d.r.GetInodeContent(d.inode)
	if err != nil {
		return entries, err
	}

	entries = make([]DirEntry, 0, d.r.super.blockSize()/12) // min dir_entry2 rec_len seems to be 12
	r := bytes.NewReader(b)
	for {
		de, err := ReadDirectoryEntry(r)
		if err == io.EOF {
			break
		}
		if err != nil {
			return []DirEntry{}, err
		}
		entries = append(entries, de)
	}
	return entries, nil
}

func (er Reader) ListPath(path string) ([]DirEntry, error) {
	if path == "" || path[0] != '/' {
		return []DirEntry{}, fmt.Errorf("path must start with '/': %q", path)
	}

	rootNode, err := er.GetInode(2)
	if err != nil {
		return []DirEntry{}, err
	}

	return er.traversePath(rootNode, path)
}

func (er Reader) traversePath(current Inode, path string) (entries []DirEntry, err error) {
	b, err := er.GetInodeContent(current)
	if err != nil {
		return
	}

	entries = make([]DirEntry, 0, er.super.blockSize()/12) // min dir_entry2 rec_len seems to be 12
	r := bytes.NewReader(b)
	for {
		de, err := ReadDirectoryEntry(r)
		if err == io.EOF {
			break
		}
		if err != nil {
			return []DirEntry{}, err
		}
		entries = append(entries, de)
	}

	for path != "" && path[0] == '/' {
		path = path[1:]
	}
	if path == "" {
		return
	}

	i := 0
	for ; i < len(path) && path[i] != '/'; i++ {
	}
	dirname := path[:i]

	for _, e := range entries {
		if e.Name.String() == dirname {
			inode, err := er.GetInode(e.Inode)
			if err != nil {
				return []DirEntry{}, err
			}
			if e.FileType == FileTypeDir {
				return er.traversePath(inode, path[i:])
			} else if e.FileType == FileTypeSymlink {
				if inode.Size() < 60 {
					l := string(inode.Data[:inode.Size()])
					//fmt.Printf("=== following symlink %s\n", l)
					return er.traversePath(current, l)
				} else {
					b, err := ioutil.ReadAll(inode.GetDataReader())
					if err != nil {
						return []DirEntry{}, err
					}
					l := string(b)
					//fmt.Printf("=== following symlink %s\n", l)
					return er.traversePath(current, l)
				}
			}
		}
	}

	return []DirEntry{}, ErrNotFound
}

func ReadDirectoryEntry(r io.Reader) (entry DirEntry, err error) {
	err = binary.Read(r, binary.LittleEndian, &entry.DirEntryHeader)
	if err != nil {
		return
	}
	entry.Name = make(charArray, entry.NameLen, entry.NameLen)
	err = binary.Read(r, binary.LittleEndian, &entry.Name)
	if err != nil {
		return
	}
	buf := make([]byte, entry.RecLen-8-uint16(entry.NameLen))
	_, err = r.Read(buf)
	return
}

type DirEntry struct {
	DirEntryHeader
	Name charArray // File name.
}

type charArray []byte

func (c charArray) String() string {
	return string([]byte(c))
}

type DirEntryHeader struct {
	Inode   uint32 // Number of the inode that this directory entry points to.
	RecLen  uint16 // Length of this directory entry.
	NameLen byte   // Length of the file name.
	// File type code, one of:
	FileType FileType
}

type FileType byte

const (
	FileTypeUnknown  FileType = 0x0 // Unknown.
	FileTypeFile     FileType = 0x1 // Regular file.
	FileTypeDir      FileType = 0x2 // Directory.
	FileTypeChardev  FileType = 0x3 // Character device file.
	FileTypeBlockdev FileType = 0x4 // Block device file.
	FileTypeFIFO     FileType = 0x5 // FIFO.
	FileTypeSocket   FileType = 0x6 // Socket.
	FileTypeSymlink  FileType = 0x7 // Symbolic link.
)

func (t FileType) String() string {
	switch t {
	case FileTypeUnknown:
		return "Unknown"
	case FileTypeFile:
		return "File"
	case FileTypeDir:
		return "Dir"
	case FileTypeChardev:
		return "Chardev"
	case FileTypeBlockdev:
		return "Blockdev"
	case FileTypeFIFO:
		return "FIFO"
	case FileTypeSocket:
		return "Socket"
	case FileTypeSymlink:
		return "Symlink"
	default:
		return fmt.Sprintf("FileType(0x%x)", t)
	}
}
<commit_msg>Add Directory.ChangeDir()<commit_after>package ext4

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"io/ioutil"
	"regexp"
	"strings"
)

func (er Reader) Root() (Directory, error) {
	inode, err := er.GetInode(2)
	if err != nil {
		return Directory{}, err
	}
	return Directory{
		r:     er,
		inode: inode,
		path:  "/",
	}, nil
}

type Directory struct {
	r     Reader
	inode Inode
	path  string
}

func (d Directory) Entries() ([]DirEntry, error) {
	var entries []DirEntry
	b, err := d.r.GetInodeContent(d.inode)
	if err != nil {
		return entries, err
	}

	entries = make([]DirEntry, 0, d.r.super.blockSize()/12) // min dir_entry2 rec_len seems to be 12
	r := bytes.NewReader(b)
	for {
		de, err := readDirEntry(r)
		if err == io.EOF {
			break
		}
		if err != nil {
			return []DirEntry{}, err
		}
		entries = append(entries, de)
	}
	return entries, nil
}

func (d Directory) findEntry(name string) (entry DirEntry, err error) {
	entries, err := d.Entries()
	if err != nil {
		return DirEntry{}, err
	}
	for _, e := range entries {
		if string(e.Name) == name {
			return e, nil
		}
	}
	return DirEntry{}, ErrNotFound
}

var slashes = regexp.MustCompile("/+")

func (d Directory) ChangeDir(path string) (Directory, error) {
	path = slashes.ReplaceAllLiteralString(path, "/")
	s := strings.Split(path, "/")
	for len(s) > 0 && s[0] == "" {
		s = s[1:]
	}
	if len(s) == 0 {
		return Directory{}, fmt.Errorf("invalid path")
	}
	e, err := d.findEntry(s[0])
	if err != nil {
		return Directory{}, err
	}
	if e.FileType == FileTypeDir {
		inode, err := d.r.GetInode(e.Inode)
		if err != nil {
			return Directory{}, err
		}
		dir := Directory{
			r:     d.r,
			inode: inode,
			path:  d.path + s[0] + "/",
		}
		if len(s) == 1 {
			return dir, nil
		}
		return dir.ChangeDir(strings.Join(s[1:], "/"))
	}
	if e.FileType == FileTypeSymlink {
		inode, err := d.r.GetInode(e.Inode)
		if err != nil {
			return Directory{}, err
		}
		link, err := ioutil.ReadAll(inode.GetDataReader())
		if err != nil {
			return Directory{}, err
		}
		path = string(link) + "/" + strings.Join(s[1:], "/")
		return d.ChangeDir(path)
	}
	return Directory{}, fmt.Errorf("Not a directory or symlink: ", d.path+s[0])
}

func readDirEntry(r io.Reader) (entry DirEntry, err error) {
	err = binary.Read(r, binary.LittleEndian, &entry.DirEntryHeader)
	if err != nil {
		return
	}
	entry.Name = make(charArray, entry.NameLen, entry.NameLen)
	err = binary.Read(r, binary.LittleEndian, &entry.Name)
	if err != nil {
		return
	}
	buf := make([]byte, entry.RecLen-8-uint16(entry.NameLen))
	_, err = r.Read(buf)
	return
}

type DirEntry struct {
	DirEntryHeader
	Name charArray // File name.
}

type charArray []byte

func (c charArray) String() string {
	return string([]byte(c))
}

type DirEntryHeader struct {
	Inode   uint32 // Number of the inode that this directory entry points to.
	RecLen  uint16 // Length of this directory entry.
	NameLen byte   // Length of the file name.
	// File type code, one of:
	FileType FileType
}

type FileType byte

const (
	FileTypeUnknown  FileType = 0x0 // Unknown.
	FileTypeFile     FileType = 0x1 // Regular file.
	FileTypeDir      FileType = 0x2 // Directory.
	FileTypeChardev  FileType = 0x3 // Character device file.
	FileTypeBlockdev FileType = 0x4 // Block device file.
	FileTypeFIFO     FileType = 0x5 // FIFO.
	FileTypeSocket   FileType = 0x6 // Socket.
	FileTypeSymlink  FileType = 0x7 // Symbolic link.
)

func (t FileType) String() string {
	switch t {
	case FileTypeUnknown:
		return "Unknown"
	case FileTypeFile:
		return "File"
	case FileTypeDir:
		return "Dir"
	case FileTypeChardev:
		return "Chardev"
	case FileTypeBlockdev:
		return "Blockdev"
	case FileTypeFIFO:
		return "FIFO"
	case FileTypeSocket:
		return "Socket"
	case FileTypeSymlink:
		return "Symlink"
	default:
		return fmt.Sprintf("FileType(0x%x)", t)
	}
}
<|endoftext|>


<commit_before>package httpmatchers

import (
	"testing"

	"github.com/onsi/gomega/types"
)

type matchTest struct {
	matchFn     func() types.GomegaMatcher
	passingCode int
	failingCode int
}

var matchTests = []matchTest{
	{BeHTTPStatusOK, 200, 404},
}

func TestMatch(t *testing.T) {
	for i, test := range matchTests {
		matcher := test.matchFn()

		passSuccess, err := matcher.Match(test.passingCode)
		if !passSuccess {
			t.Errorf("Test %d: Match(%d): Expected true, actual false", i, test.passingCode)
		}
		if err != nil {
			t.Errorf("Test %d: Match(%d): Expected err to be nil, was %v", i, test.passingCode, err)
		}

		failSuccess, err := matcher.Match(test.failingCode)
		if failSuccess {
			t.Errorf("Test %d: Match(%d): Expected false, actual true", i, test.failingCode)
		}
		if err != nil {
			t.Errorf("Test %d: Match(%d): Expected err to be nil, was %v", i, test.failingCode, err)
		}

		invalidMatch, err := matcher.Match("banana")
		if invalidMatch {
			t.Errorf("Test %d: Match(%q): Expected false, actual true", i, "banana")
		}
		if err == nil {
			t.Errorf("Test %d: Match(%q): Expected err to not be nil", i, "banana")
		}
	}
}
<commit_msg>Simplify testing<commit_after>package httpmatchers

import (
	"testing"

	"github.com/onsi/gomega/types"
)

type matchTest struct {
	matchFn     func() types.GomegaMatcher
	passingCode int
}

var matchTests = []matchTest{
	{BeHTTPStatusOK, 200},
}

func TestMatch(t *testing.T) {
	failCode := 0
	invalidCode := "banana"

	for i, test := range matchTests {
		matcher := test.matchFn()

		passSuccess, err := matcher.Match(test.passingCode)
		if !passSuccess {
			t.Errorf("Test %d: Match(%d): Expected true, actual false", i, test.passingCode)
		}
		if err != nil {
			t.Errorf("Test %d: Match(%d): Expected err to be nil, was %v", i, test.passingCode, err)
		}

		failSuccess, err := matcher.Match(failCode)
		if failSuccess {
			t.Errorf("Test %d: Match(%d): Expected false, actual true", i, failCode)
		}
		if err != nil {
			t.Errorf("Test %d: Match(%d): Expected err to be nil, was %v", i, failCode, err)
		}

		invalidMatch, err := matcher.Match(invalidCode)
		if invalidMatch {
			t.Errorf("Test %d: Match(%q): Expected false, actual true", i, invalidCode)
		}
		if err == nil {
			t.Errorf("Test %d: Match(%q): Expected err to not be nil", i, invalidCode)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2011 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package oglematchers_test

import (
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
)

////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////

type AnyTest struct {
}

func init() { RegisterTestSuite(&AnyTest{}) }

////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////

func (t *AnyTest) Description() {
	m := Any()
	ExpectEq("is anything", m.Description())
}

func (t *AnyTest) Matches() {
	var res bool
	var err error
	m := Any()

	err = m.Matches(nil)
	ExpectEq(nil, err)

	err = m.Matches(17)
	ExpectEq(nil, err)

	err = m.Matches("taco")
	ExpectEq(nil, err)
}
<commit_msg>Fixed an error.<commit_after>// Copyright 2011 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package oglematchers_test

import (
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
)

////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////

type AnyTest struct {
}

func init() { RegisterTestSuite(&AnyTest{}) }

////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////

func (t *AnyTest) Description() {
	m := Any()
	ExpectEq("is anything", m.Description())
}

func (t *AnyTest) Matches() {
	var err error
	m := Any()

	err = m.Matches(nil)
	ExpectEq(nil, err)

	err = m.Matches(17)
	ExpectEq(nil, err)

	err = m.Matches("taco")
	ExpectEq(nil, err)
}
<|endoftext|>


<commit_before>package libkbfs

import (
	"bytes"
	"fmt"
	"runtime"
	"sync"
	"testing"

	keybase1 "github.com/keybase/client/protocol/go"
	"golang.org/x/net/context"
)

type FakeBServerClient struct {
	blocks     map[keybase1.GetBlockArg]keybase1.GetBlockRes
	blocksLock sync.Mutex
	readyChan  chan<- struct{}
	goChan     <-chan struct{}
	finishChan chan<- struct{}
}

func NewFakeBServerClient(
	readyChan chan<- struct{},
	goChan <-chan struct{},
	finishChan chan<- struct{}) *FakeBServerClient {
	return &FakeBServerClient{
		blocks:    make(map[keybase1.GetBlockArg]keybase1.GetBlockRes),
		readyChan: readyChan,
		goChan:    goChan,
	}
}

func (fc *FakeBServerClient) maybeWaitOnChannel() {
	if fc.readyChan != nil {
		// say we're ready, and wait for the signal to proceed
		fc.readyChan <- struct{}{}
		<-fc.goChan
	}
}

func (fc *FakeBServerClient) maybeFinishOnChannel() {
	if fc.finishChan != nil {
		fc.finishChan <- struct{}{}
	}
}

func (fc *FakeBServerClient) Call(s string, args interface{},
	res interface{}) error {
	switch s {
	case "keybase.1.block.establishSession":
		// no need to do anything
		return nil

	case "keybase.1.block.putBlock":
		fc.maybeWaitOnChannel()
		defer fc.maybeFinishOnChannel()
		putArgs := args.([]interface{})[0].(keybase1.PutBlockArg)
		fc.blocksLock.Lock()
		defer fc.blocksLock.Unlock()
		fc.blocks[keybase1.GetBlockArg{Bid: putArgs.Bid}] =
			keybase1.GetBlockRes{BlockKey: putArgs.BlockKey, Buf: putArgs.Buf}
		return nil

	case "keybase.1.block.getBlock":
		fc.maybeWaitOnChannel()
		defer fc.maybeFinishOnChannel()
		getArgs := args.([]interface{})[0].(keybase1.GetBlockArg)
		getRes := res.(*keybase1.GetBlockRes)
		fc.blocksLock.Lock()
		defer fc.blocksLock.Unlock()
		getRes2, ok := fc.blocks[getArgs]
		*getRes = getRes2
		if !ok {
			return fmt.Errorf("No such block: %v", getArgs)
		}
		return nil

	default:
		return fmt.Errorf("Unknown call: %s %v %v", s, args, res)
	}
}

func (fc *FakeBServerClient) numBlocks() int {
	fc.blocksLock.Lock()
	defer fc.blocksLock.Unlock()
	return len(fc.blocks)
}

// Test that putting a block, and getting it back, works
func TestBServerRemotePutAndGet(t *testing.T) {
	codec := NewCodecMsgpack()
	localUsers := MakeLocalUsers([]string{"testuser"})
	loggedInUser := localUsers[0]
	kbpki := NewKBPKIMemory(loggedInUser.UID, localUsers)
	config := &ConfigLocal{codec: codec, kbpki: kbpki}
	fc := NewFakeBServerClient(nil, nil, nil)
	ctx := context.Background()
	b := newBlockServerRemoteWithClient(ctx, config, fc)

	bID := fakeBlockID(1)
	tlfID := FakeTlfID(2, false)
	bCtx := BlockPointer{bID, 1, 1, kbpki.LoggedIn, "", zeroBlockRefNonce}
	data := []byte{1, 2, 3, 4}
	crypto := &CryptoCommon{codec}
	serverHalf, err := crypto.MakeRandomBlockCryptKeyServerHalf()
	if err != nil {
		t.Errorf("Couldn't make block server key half: %v", err)
	}
	err = b.Put(ctx, bID, tlfID, bCtx, data, serverHalf)
	if err != nil {
		t.Fatalf("Put got error: %v", err)
	}

	// make sure it actually got to the db
	nb := fc.numBlocks()
	if nb != 1 {
		t.Errorf("There are %d blocks in the db, not 1 as expected", nb)
	}

	// Now get the same block back
	buf, key, err := b.Get(ctx, bID, bCtx)
	if err != nil {
		t.Fatalf("Get returned an error: %v", err)
	}
	if !bytes.Equal(buf, data) {
		t.Errorf("Got bad data -- got %v, expected %v", buf, data)
	}
	if key != serverHalf {
		t.Errorf("Got bad key -- got %v, expected %v", key, serverHalf)
	}
}

// If we cancel the RPC before the RPC returns, the call should error quickly.
func TestBServerRemotePutCanceled(t *testing.T) {
	codec := NewCodecMsgpack()
	localUsers := MakeLocalUsers([]string{"testuser"})
	loggedInUser := localUsers[0]
	kbpki := NewKBPKIMemory(loggedInUser.UID, localUsers)
	config := &ConfigLocal{codec: codec, kbpki: kbpki}
	readyChan := make(chan struct{})
	goChan := make(chan struct{})
	fc := NewFakeBServerClient(readyChan, goChan, nil)

	f := func(ctx context.Context) error {
		b := newBlockServerRemoteWithClient(ctx, config, fc)

		bID := fakeBlockID(1)
		tlfID := FakeTlfID(2, false)
		bCtx := BlockPointer{bID, 1, 1, kbpki.LoggedIn, "", zeroBlockRefNonce}
		data := []byte{1, 2, 3, 4}
		crypto := &CryptoCommon{codec}
		serverHalf, err := crypto.MakeRandomBlockCryptKeyServerHalf()
		if err != nil {
			t.Errorf("Couldn't make block server key half: %v", err)
		}
		err = b.Put(ctx, bID, tlfID, bCtx, data, serverHalf)
		return err
	}
	testWithCanceledContext(t, context.Background(), readyChan, goChan, f)
}

// Test that RPCs wait for the bserver to connect to the backend
func TestBServerRemoteWaitForReconnect(t *testing.T) {
	codec := NewCodecMsgpack()
	localUsers := MakeLocalUsers([]string{"testuser"})
	loggedInUser := localUsers[0]
	kbpki := NewKBPKIMemory(loggedInUser.UID, localUsers)
	config := &ConfigLocal{codec: codec, kbpki: kbpki}
	fc := NewFakeBServerClient(nil, nil, nil)
	ctx := context.Background()

	b := newBlockServerRemoteWithClient(ctx, config, fc)

	putChan := make(chan error)
	go func() {
		bID := fakeBlockID(1)
		tlfID := FakeTlfID(2, false)
		bCtx := BlockPointer{bID, 1, 1, kbpki.LoggedIn, "", zeroBlockRefNonce}
		data := []byte{1, 2, 3, 4}
		crypto := &CryptoCommon{codec}
		serverHalf, err := crypto.MakeRandomBlockCryptKeyServerHalf()
		if err != nil {
			t.Errorf("Couldn't make block server key half: %v", err)
		}
		// wait til the test says to start
		<-putChan
		putChan <- b.Put(ctx, bID, tlfID, bCtx, data, serverHalf)
	}()

	// tell the put to start
	putChan <- nil
	// give the goroutine a chance to run
	runtime.Gosched()

	// Make sure there's no answer yet. Still a little racy (i.e.,
	// we're not 100% guaranteed the Put is waiting for the connect)
	// but that's ok.
	select {
	case <-putChan:
		t.Fatal("Got an answer from put before we connected!")
	default:
		// fall through to connecting
	}

	// now there should be an answer waiting for us
	err := <-putChan
	if err != nil {
		t.Fatalf("Put got an error: %v", err)
	}

	// make sure it actually got to the db
	nb := fc.numBlocks()
	if nb != 1 {
		t.Errorf("There are %d blocks in the db, not 1 as expected", nb)
	}
}
<commit_msg>flip the order of signaling putChan in BServerRemoteWaitForReconnect test<commit_after>package libkbfs

import (
	"bytes"
	"fmt"
	"runtime"
	"sync"
	"testing"

	keybase1 "github.com/keybase/client/protocol/go"
	"golang.org/x/net/context"
)

type FakeBServerClient struct {
	blocks     map[keybase1.GetBlockArg]keybase1.GetBlockRes
	blocksLock sync.Mutex
	readyChan  chan<- struct{}
	goChan     <-chan struct{}
	finishChan chan<- struct{}
}

func NewFakeBServerClient(
	readyChan chan<- struct{},
	goChan <-chan struct{},
	finishChan chan<- struct{}) *FakeBServerClient {
	return &FakeBServerClient{
		blocks:    make(map[keybase1.GetBlockArg]keybase1.GetBlockRes),
		readyChan: readyChan,
		goChan:    goChan,
	}
}

func (fc *FakeBServerClient) maybeWaitOnChannel() {
	if fc.readyChan != nil {
		// say we're ready, and wait for the signal to proceed
		fc.readyChan <- struct{}{}
		<-fc.goChan
	}
}

func (fc *FakeBServerClient) maybeFinishOnChannel() {
	if fc.finishChan != nil {
		fc.finishChan <- struct{}{}
	}
}

func (fc *FakeBServerClient) Call(s string, args interface{},
	res interface{}) error {
	switch s {
	case "keybase.1.block.establishSession":
		// no need to do anything
		return nil

	case "keybase.1.block.putBlock":
		fc.maybeWaitOnChannel()
		defer fc.maybeFinishOnChannel()
		putArgs := args.([]interface{})[0].(keybase1.PutBlockArg)
		fc.blocksLock.Lock()
		defer fc.blocksLock.Unlock()
		fc.blocks[keybase1.GetBlockArg{Bid: putArgs.Bid}] =
			keybase1.GetBlockRes{BlockKey: putArgs.BlockKey, Buf: putArgs.Buf}
		return nil

	case "keybase.1.block.getBlock":
		fc.maybeWaitOnChannel()
		defer fc.maybeFinishOnChannel()
		getArgs := args.([]interface{})[0].(keybase1.GetBlockArg)
		getRes := res.(*keybase1.GetBlockRes)
		fc.blocksLock.Lock()
		defer fc.blocksLock.Unlock()
		getRes2, ok := fc.blocks[getArgs]
		*getRes = getRes2
		if !ok {
			return fmt.Errorf("No such block: %v", getArgs)
		}
		return nil

	default:
		return fmt.Errorf("Unknown call: %s %v %v", s, args, res)
	}
}

func (fc *FakeBServerClient) numBlocks() int {
	fc.blocksLock.Lock()
	defer fc.blocksLock.Unlock()
	return len(fc.blocks)
}

// Test that putting a block, and getting it back, works
func TestBServerRemotePutAndGet(t *testing.T) {
	codec := NewCodecMsgpack()
	localUsers := MakeLocalUsers([]string{"testuser"})
	loggedInUser := localUsers[0]
	kbpki := NewKBPKIMemory(loggedInUser.UID, localUsers)
	config := &ConfigLocal{codec: codec, kbpki: kbpki}
	fc := NewFakeBServerClient(nil, nil, nil)
	ctx := context.Background()
	b := newBlockServerRemoteWithClient(ctx, config, fc)

	bID := fakeBlockID(1)
	tlfID := FakeTlfID(2, false)
	bCtx := BlockPointer{bID, 1, 1, kbpki.LoggedIn, "", zeroBlockRefNonce}
	data := []byte{1, 2, 3, 4}
	crypto := &CryptoCommon{codec}
	serverHalf, err := crypto.MakeRandomBlockCryptKeyServerHalf()
	if err != nil {
		t.Errorf("Couldn't make block server key half: %v", err)
	}
	err = b.Put(ctx, bID, tlfID, bCtx, data, serverHalf)
	if err != nil {
		t.Fatalf("Put got error: %v", err)
	}

	// make sure it actually got to the db
	nb := fc.numBlocks()
	if nb != 1 {
		t.Errorf("There are %d blocks in the db, not 1 as expected", nb)
	}

	// Now get the same block back
	buf, key, err := b.Get(ctx, bID, bCtx)
	if err != nil {
		t.Fatalf("Get returned an error: %v", err)
	}
	if !bytes.Equal(buf, data) {
		t.Errorf("Got bad data -- got %v, expected %v", buf, data)
	}
	if key != serverHalf {
		t.Errorf("Got bad key -- got %v, expected %v", key, serverHalf)
	}
}

// If we cancel the RPC before the RPC returns, the call should error quickly.
func TestBServerRemotePutCanceled(t *testing.T) {
	codec := NewCodecMsgpack()
	localUsers := MakeLocalUsers([]string{"testuser"})
	loggedInUser := localUsers[0]
	kbpki := NewKBPKIMemory(loggedInUser.UID, localUsers)
	config := &ConfigLocal{codec: codec, kbpki: kbpki}
	readyChan := make(chan struct{})
	goChan := make(chan struct{})
	fc := NewFakeBServerClient(readyChan, goChan, nil)

	f := func(ctx context.Context) error {
		b := newBlockServerRemoteWithClient(ctx, config, fc)

		bID := fakeBlockID(1)
		tlfID := FakeTlfID(2, false)
		bCtx := BlockPointer{bID, 1, 1, kbpki.LoggedIn, "", zeroBlockRefNonce}
		data := []byte{1, 2, 3, 4}
		crypto := &CryptoCommon{codec}
		serverHalf, err := crypto.MakeRandomBlockCryptKeyServerHalf()
		if err != nil {
			t.Errorf("Couldn't make block server key half: %v", err)
		}
		err = b.Put(ctx, bID, tlfID, bCtx, data, serverHalf)
		return err
	}
	testWithCanceledContext(t, context.Background(), readyChan, goChan, f)
}

// Test that RPCs wait for the bserver to connect to the backend
func TestBServerRemoteWaitForReconnect(t *testing.T) {
	codec := NewCodecMsgpack()
	localUsers := MakeLocalUsers([]string{"testuser"})
	loggedInUser := localUsers[0]
	kbpki := NewKBPKIMemory(loggedInUser.UID, localUsers)
	config := &ConfigLocal{codec: codec, kbpki: kbpki}
	fc := NewFakeBServerClient(nil, nil, nil)
	ctx := context.Background()

	b := newBlockServerRemoteWithClient(ctx, config, fc)

	putChan := make(chan error)
	go func() {
		bID := fakeBlockID(1)
		tlfID := FakeTlfID(2, false)
		bCtx := BlockPointer{bID, 1, 1, kbpki.LoggedIn, "", zeroBlockRefNonce}
		data := []byte{1, 2, 3, 4}
		crypto := &CryptoCommon{codec}
		serverHalf, err := crypto.MakeRandomBlockCryptKeyServerHalf()
		if err != nil {
			t.Errorf("Couldn't make block server key half: %v", err)
		}
		// wait til the test says to start
		<-putChan
		putChan <- b.Put(ctx, bID, tlfID, bCtx, data, serverHalf)
	}()

	// give the goroutine a chance to run
	runtime.Gosched()

	// Make sure there's no answer yet. Still a little racy (i.e.,
	// we're not 100% guaranteed the Put is waiting for the connect)
	// but that's ok.
	select {
	case <-putChan:
		t.Fatal("Got an answer from put before we connected!")
	default:
		// fall through to connecting
	}

	// tell the put to start
	putChan <- nil

	// now there should be an answer waiting for us
	err := <-putChan
	if err != nil {
		t.Fatalf("Put got an error: %v", err)
	}

	// make sure it actually got to the db
	nb := fc.numBlocks()
	if nb != 1 {
		t.Errorf("There are %d blocks in the db, not 1 as expected", nb)
	}
}
<|endoftext|>


<commit_before>package project

import (
	"bytes"
	"errors"
	"strings"

	log "github.com/Sirupsen/logrus"
	"gopkg.in/yaml.v2"
)

type ServiceState string

var (
	EXECUTED   ServiceState = ServiceState("executed")
	UNKNOWN    ServiceState = ServiceState("unknown")
	ErrRestart error        = errors.New("Restart execution")
)

type ProjectEvent struct {
	Event       Event
	ServiceName string
	Data        map[string]string
}

func NewProject(name string, factory ServiceFactory) *Project {
	return &Project{
		Name:    name,
		configs: make(map[string]*ServiceConfig),
		factory: factory,
	}
}

func (p *Project) CreateService(name string, config ServiceConfig) (Service, error) {
	if p.EnvironmentLookup != nil {
		parsedEnv := make([]string, 0, len(config.Environment))

		for _, env := range config.Environment {
			if strings.IndexRune(env, '=') != -1 {
				parsedEnv = append(parsedEnv, env)
				continue
			}

			for _, value := range p.EnvironmentLookup.Lookup(env, name, &config) {
				parsedEnv = append(parsedEnv, value)
			}
		}

		config.Environment = parsedEnv
	}

	return p.factory.Create(p, name, &config)
}

func (p *Project) AddConfig(name string, config *ServiceConfig) error {
	p.Notify(SERVICE_ADD, name, nil)

	p.configs[name] = config

	return nil
}

func (p *Project) Load(bytes []byte) error {
	configs := make(map[string]*ServiceConfig)
	err := yaml.Unmarshal(bytes, configs)
	if err != nil {
		log.Fatalf("Could not parse config for project %s : %v", p.Name, err)
	}

	for name, config := range configs {
		err := p.AddConfig(name, config)
		if err != nil {
			return err
		}
	}

	return nil
}

func (p *Project) Up() error {
	wrappers := make(map[string]*serviceWrapper)

	for name, _ := range p.configs {
		wrapper, err := newServiceWrapper(name, p)
		if err != nil {
			return err
		}
		wrappers[name] = wrapper
	}

	p.Notify(PROJECT_UP_START, "", nil)

	err := p.startAll(wrappers, 0)

	if err == nil {
		p.Notify(PROJECT_UP_DONE, "", nil)
	}

	return err
}

func (p *Project) startAll(wrappers map[string]*serviceWrapper, level int) error {
	restart := false

	if level > 0 {
		for _, wrapper := range wrappers {
			if err := wrapper.Reset(); err != nil {
				return err
			}
		}
	}

	for _, wrapper := range wrappers {
		go wrapper.Start(wrappers)
	}

	var firstError error

	for _, wrapper := range wrappers {
		err := wrapper.Wait()
		if err == ErrRestart {
			restart = true
		} else if err != nil {
			log.Errorf("Failed to start: %s : %v", wrapper.name, err)
			if firstError == nil {
				firstError = err
			}
		}
	}

	if restart {
		if p.ReloadCallback != nil {
			if err := p.ReloadCallback(); err != nil {
				log.Errorf("Failed calling callback: %v", err)
			}
		}
		return p.startAll(wrappers, level+1)
	} else {
		return firstError
	}
}

func (p *Project) AddListener(c chan<- ProjectEvent) {
	p.listeners = append(p.listeners, c)
}

func (p *Project) Notify(event Event, serviceName string, data map[string]string) {
	buffer := bytes.NewBuffer(nil)
	if data != nil {
		for k, v := range data {
			if buffer.Len() > 0 {
				buffer.WriteString(", ")
			}
			buffer.WriteString(k)
			buffer.WriteString("=")
			buffer.WriteString(v)
		}
	}

	if event == SERVICE_UP {
		p.upCount++
	}

	logf := log.Debugf

	if SERVICE_UP == event {
		logf = log.Infof
	}

	if serviceName == "" {
		logf("Project [%s]: %s %s", p.Name, event, buffer.Bytes())
	} else {
		logf("[%d/%d] [%s]: %s %s", p.upCount, len(p.configs), serviceName, event, buffer.Bytes())
	}

	for _, l := range p.listeners {
		projectEvent := ProjectEvent{
			Event:       event,
			ServiceName: serviceName,
			Data:        data,
		}
		// Don't ever block
		select {
		case l <- projectEvent:
		default:
		}
	}
}
<commit_msg>Fix reloading config and adding new services<commit_after>package project

import (
	"bytes"
	"errors"
	"strings"

	"gopkg.in/yaml.v2"

	log "github.com/Sirupsen/logrus"
)

type ServiceState string

var (
	EXECUTED   ServiceState = ServiceState("executed")
	UNKNOWN    ServiceState = ServiceState("unknown")
	ErrRestart error        = errors.New("Restart execution")
)

type ProjectEvent struct {
	Event       Event
	ServiceName string
	Data        map[string]string
}

func NewProject(name string, factory ServiceFactory) *Project {
	return &Project{
		Name:    name,
		configs: make(map[string]*ServiceConfig),
		factory: factory,
	}
}

func (p *Project) CreateService(name string, config ServiceConfig) (Service, error) {
	if p.EnvironmentLookup != nil {
		parsedEnv := make([]string, 0, len(config.Environment))

		for _, env := range config.Environment {
			if strings.IndexRune(env, '=') != -1 {
				parsedEnv = append(parsedEnv, env)
				continue
			}

			for _, value := range p.EnvironmentLookup.Lookup(env, name, &config) {
				parsedEnv = append(parsedEnv, value)
			}
		}

		config.Environment = parsedEnv
	}

	return p.factory.Create(p, name, &config)
}

func (p *Project) AddConfig(name string, config *ServiceConfig) error {
	p.Notify(SERVICE_ADD, name, nil)

	p.configs[name] = config
	p.reload = append(p.reload, name)

	return nil
}

func (p *Project) Load(bytes []byte) error {
	configs := make(map[string]*ServiceConfig)
	err := yaml.Unmarshal(bytes, configs)
	if err != nil {
		log.Fatalf("Could not parse config for project %s : %v", p.Name, err)
	}

	for name, config := range configs {
		err := p.AddConfig(name, config)
		if err != nil {
			return err
		}
	}

	return nil
}

func (p *Project) loadWrappers(wrappers map[string]*serviceWrapper) error {
	for _, name := range p.reload {
		wrapper, err := newServiceWrapper(name, p)
		if err != nil {
			return err
		}
		wrappers[name] = wrapper
	}

	p.reload = []string{}

	return nil
}

func (p *Project) Up() error {
	wrappers := make(map[string]*serviceWrapper)

	p.Notify(PROJECT_UP_START, "", nil)

	err := p.startAll(wrappers)

	if err == nil {
		p.Notify(PROJECT_UP_DONE, "", nil)
	}

	return err
}

func (p *Project) startAll(wrappers map[string]*serviceWrapper) error {
	restart := false

	for _, wrapper := range wrappers {
		if err := wrapper.Reset(); err != nil {
			return err
		}
	}

	p.loadWrappers(wrappers)

	for _, wrapper := range wrappers {
		go wrapper.Start(wrappers)
	}

	var firstError error

	for _, wrapper := range wrappers {
		err := wrapper.Wait()
		if err == ErrRestart {
			restart = true
		} else if err != nil {
			log.Errorf("Failed to start: %s : %v", wrapper.name, err)
			if firstError == nil {
				firstError = err
			}
		}
	}

	if restart {
		if p.ReloadCallback != nil {
			if err := p.ReloadCallback(); err != nil {
				log.Errorf("Failed calling callback: %v", err)
			}
		}
		return p.startAll(wrappers)
	} else {
		return firstError
	}
}

func (p *Project) AddListener(c chan<- ProjectEvent) {
	p.listeners = append(p.listeners, c)
}

func (p *Project) Notify(event Event, serviceName string, data map[string]string) {
	buffer := bytes.NewBuffer(nil)
	if data != nil {
		for k, v := range data {
			if buffer.Len() > 0 {
				buffer.WriteString(", ")
			}
			buffer.WriteString(k)
			buffer.WriteString("=")
			buffer.WriteString(v)
		}
	}

	if event == SERVICE_UP {
		p.upCount++
	}

	logf := log.Debugf

	if SERVICE_UP == event {
		logf = log.Infof
	}

	if serviceName == "" {
		logf("Project [%s]: %s %s", p.Name, event, buffer.Bytes())
	} else {
		logf("[%d/%d] [%s]: %s %s", p.upCount, len(p.configs), serviceName, event, buffer.Bytes())
	}

	for _, l := range p.listeners {
		projectEvent := ProjectEvent{
			Event:       event,
			ServiceName: serviceName,
			Data:        data,
		}
		// Don't ever block
		select {
		case l <- projectEvent:
		default:
		}
	}
}
<|endoftext|>