<commit_before>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package hbasekv

import (
	"fmt"
	"math/rand"
	"strings"
	"sync"
	"time"

	"github.com/juju/errors"
	"github.com/ngaut/log"
	"github.com/pingcap/go-hbase"
	"github.com/pingcap/go-themis"
	"github.com/pingcap/go-themis/oracle"
	"github.com/pingcap/go-themis/oracle/oracles"
	"github.com/pingcap/tidb/kv"
)

const (
	// hbaseColFamily is the hbase column family name.
	hbaseColFamily = "f"
	// hbaseQualifier is the hbase column name.
	hbaseQualifier = "q"
	// hbaseFmlAndQual is a shortcut.
	hbaseFmlAndQual = hbaseColFamily + ":" + hbaseQualifier
	// fix length conn pool
	hbaseConnPoolSize = 10
)

var (
	hbaseColFamilyBytes = []byte(hbaseColFamily)
	hbaseQualifierBytes = []byte(hbaseQualifier)
)

var (
	_ kv.Storage = (*hbaseStore)(nil)
)

var (
	// ErrInvalidDSN is returned when store dsn is invalid.
	ErrInvalidDSN = errors.New("invalid dsn")
)

type storeCache struct {
	mu    sync.Mutex
	cache map[string]*hbaseStore
}

var mc storeCache

func init() {
	mc.cache = make(map[string]*hbaseStore)
	rand.Seed(time.Now().UnixNano())
}

type hbaseStore struct {
	mu        sync.Mutex
	dsn       string
	storeName string
	oracle    oracle.Oracle
	conns     []hbase.HBaseClient
}

func (s *hbaseStore) getHBaseClient() hbase.HBaseClient {
	// return hbase connection randomly
	return s.conns[rand.Intn(hbaseConnPoolSize)]
}

func (s *hbaseStore) Begin() (kv.Transaction, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	hbaseCli := s.getHBaseClient()
	t, err := themis.NewTxn(hbaseCli, s.oracle)
	if err != nil {
		return nil, errors.Trace(err)
	}
	txn := newHbaseTxn(t, s.storeName)
	return txn, nil
}

func (s *hbaseStore) GetSnapshot(ver kv.Version) (kv.Snapshot, error) {
	hbaseCli := s.getHBaseClient()
	t, err := themis.NewTxn(hbaseCli, s.oracle)
	if err != nil {
		return nil, errors.Trace(err)
	}
	return newHbaseSnapshot(t, s.storeName), nil
}

func (s *hbaseStore) Close() error {
	mc.mu.Lock()
	defer mc.mu.Unlock()

	delete(mc.cache, s.dsn)

	var err error
	for _, conn := range s.conns {
		err = conn.Close()
		if err != nil {
			log.Error(err)
		}
	}
	// return last error
	return err
}

func (s *hbaseStore) UUID() string {
	return fmt.Sprintf("hbase.%s.%s", s.storeName, s.dsn)
}

func (s *hbaseStore) CurrentVersion() (kv.Version, error) {
	hbaseCli := s.getHBaseClient()
	t, err := themis.NewTxn(hbaseCli, s.oracle)
	if err != nil {
		return kv.Version{Ver: 0}, errors.Trace(err)
	}
	defer t.Release()

	return kv.Version{Ver: t.GetStartTS()}, nil
}

// Driver implements engine Driver.
type Driver struct {
}

// Open opens or creates an HBase storage with given dsn, format should be 'zk1,zk2,zk3|tsoaddr:port/tblName'.
// If tsoAddr is not provided, it will use a local oracle instead.
func (d Driver) Open(dsn string) (kv.Storage, error) {
	mc.mu.Lock()
	defer mc.mu.Unlock()

	if store, ok := mc.cache[dsn]; ok {
		// TODO: check the cache store has the same engine with this Driver.
		return store, nil
	}

	zks, oracleAddr, tableName, err := parseDSN(dsn)
	if err != nil {
		return nil, errors.Trace(err)
	}

	// create buffered HBase connections, HBaseClient is goroutine-safe, so
	// it's OK to redistribute to transactions.
	conns := make([]hbase.HBaseClient, 0, hbaseConnPoolSize)
	for i := 0; i < hbaseConnPoolSize; i++ {
		c, err := hbase.NewClient(zks, "/hbase")
		if err != nil {
			return nil, errors.Trace(err)
		}
		conns = append(conns, c)
	}

	c := conns[0]
	if !c.TableExists(tableName) {
		// Create new hbase table for store.
		t := hbase.NewTableDesciptor(hbase.NewTableNameWithDefaultNS(tableName))
		cf := hbase.NewColumnFamilyDescriptor(hbaseColFamily)
		cf.AddStrAddr("THEMIS_ENABLE", "true")
		t.AddColumnDesc(cf)
		//TODO: specify split?
		if err := c.CreateTable(t, nil); err != nil {
			return nil, errors.Trace(err)
		}
	}

	var ora oracle.Oracle
	if len(oracleAddr) == 0 {
		ora = oracles.NewLocalOracle()
	} else {
		ora = oracles.NewRemoteOracle(oracleAddr)
	}

	s := &hbaseStore{
		dsn:       dsn,
		storeName: tableName,
		oracle:    ora,
		conns:     conns,
	}
	mc.cache[dsn] = s
	return s, nil
}

func parseDSN(dsn string) (zks []string, oracleAddr, tableName string, err error) {
	pos := strings.LastIndex(dsn, "/")
	if pos == -1 {
		err = errors.Trace(ErrInvalidDSN)
		return
	}
	tableName = dsn[pos+1:]
	addrs := dsn[:pos]

	pos = strings.LastIndex(addrs, "|")
	if pos != -1 {
		oracleAddr = addrs[pos+1:]
		addrs = addrs[:pos]
	}
	zks = strings.Split(addrs, ",")
	return
}
<commit_msg>hbase/kv: fit for hbase refactor<commit_after>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package hbasekv

import (
	"fmt"
	"math/rand"
	"strings"
	"sync"
	"time"

	"github.com/juju/errors"
	"github.com/ngaut/log"
	"github.com/pingcap/go-hbase"
	"github.com/pingcap/go-themis"
	"github.com/pingcap/go-themis/oracle"
	"github.com/pingcap/go-themis/oracle/oracles"
	"github.com/pingcap/tidb/kv"
)

const (
	// hbaseColFamily is the hbase column family name.
	hbaseColFamily = "f"
	// hbaseQualifier is the hbase column name.
	hbaseQualifier = "q"
	// hbaseFmlAndQual is a shortcut.
	hbaseFmlAndQual = hbaseColFamily + ":" + hbaseQualifier
	// fix length conn pool
	hbaseConnPoolSize = 10
)

var (
	hbaseColFamilyBytes = []byte(hbaseColFamily)
	hbaseQualifierBytes = []byte(hbaseQualifier)
)

var (
	_ kv.Storage = (*hbaseStore)(nil)
)

var (
	// ErrInvalidDSN is returned when store dsn is invalid.
	ErrInvalidDSN = errors.New("invalid dsn")
)

type storeCache struct {
	mu    sync.Mutex
	cache map[string]*hbaseStore
}

var mc storeCache

func init() {
	mc.cache = make(map[string]*hbaseStore)
	rand.Seed(time.Now().UnixNano())
}

type hbaseStore struct {
	mu        sync.Mutex
	dsn       string
	storeName string
	oracle    oracle.Oracle
	conns     []hbase.HBaseClient
}

func (s *hbaseStore) getHBaseClient() hbase.HBaseClient {
	// return hbase connection randomly
	return s.conns[rand.Intn(hbaseConnPoolSize)]
}

func (s *hbaseStore) Begin() (kv.Transaction, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	hbaseCli := s.getHBaseClient()
	t, err := themis.NewTxn(hbaseCli, s.oracle)
	if err != nil {
		return nil, errors.Trace(err)
	}
	txn := newHbaseTxn(t, s.storeName)
	return txn, nil
}

func (s *hbaseStore) GetSnapshot(ver kv.Version) (kv.Snapshot, error) {
	hbaseCli := s.getHBaseClient()
	t, err := themis.NewTxn(hbaseCli, s.oracle)
	if err != nil {
		return nil, errors.Trace(err)
	}
	return newHbaseSnapshot(t, s.storeName), nil
}

func (s *hbaseStore) Close() error {
	mc.mu.Lock()
	defer mc.mu.Unlock()

	delete(mc.cache, s.dsn)

	var err error
	for _, conn := range s.conns {
		err = conn.Close()
		if err != nil {
			log.Error(err)
		}
	}
	// return last error
	return err
}

func (s *hbaseStore) UUID() string {
	return fmt.Sprintf("hbase.%s.%s", s.storeName, s.dsn)
}

func (s *hbaseStore) CurrentVersion() (kv.Version, error) {
	hbaseCli := s.getHBaseClient()
	t, err := themis.NewTxn(hbaseCli, s.oracle)
	if err != nil {
		return kv.Version{Ver: 0}, errors.Trace(err)
	}
	defer t.Release()

	return kv.Version{Ver: t.GetStartTS()}, nil
}

// Driver implements engine Driver.
type Driver struct {
}

// Open opens or creates an HBase storage with given dsn, format should be 'zk1,zk2,zk3|tsoaddr:port/tblName'.
// If tsoAddr is not provided, it will use a local oracle instead.
func (d Driver) Open(dsn string) (kv.Storage, error) {
	mc.mu.Lock()
	defer mc.mu.Unlock()

	if store, ok := mc.cache[dsn]; ok {
		// TODO: check the cache store has the same engine with this Driver.
		return store, nil
	}

	zks, oracleAddr, tableName, err := parseDSN(dsn)
	if err != nil {
		return nil, errors.Trace(err)
	}

	// create buffered HBase connections, HBaseClient is goroutine-safe, so
	// it's OK to redistribute to transactions.
	conns := make([]hbase.HBaseClient, 0, hbaseConnPoolSize)
	for i := 0; i < hbaseConnPoolSize; i++ {
		var c hbase.HBaseClient
		c, err = hbase.NewClient(zks, "/hbase")
		if err != nil {
			return nil, errors.Trace(err)
		}
		conns = append(conns, c)
	}

	c := conns[0]
	var b bool
	b, err = c.TableExists(tableName)
	if err != nil {
		return nil, errors.Trace(err)
	}
	if !b {
		// Create new hbase table for store.
		t := hbase.NewTableDesciptor(hbase.NewTableNameWithDefaultNS(tableName))
		cf := hbase.NewColumnFamilyDescriptor(hbaseColFamily)
		cf.AddStrAddr("THEMIS_ENABLE", "true")
		t.AddColumnDesc(cf)
		//TODO: specify split?
		if err := c.CreateTable(t, nil); err != nil {
			return nil, errors.Trace(err)
		}
	}

	var ora oracle.Oracle
	if len(oracleAddr) == 0 {
		ora = oracles.NewLocalOracle()
	} else {
		ora = oracles.NewRemoteOracle(oracleAddr)
	}

	s := &hbaseStore{
		dsn:       dsn,
		storeName: tableName,
		oracle:    ora,
		conns:     conns,
	}
	mc.cache[dsn] = s
	return s, nil
}

func parseDSN(dsn string) (zks []string, oracleAddr, tableName string, err error) {
	pos := strings.LastIndex(dsn, "/")
	if pos == -1 {
		err = errors.Trace(ErrInvalidDSN)
		return
	}
	tableName = dsn[pos+1:]
	addrs := dsn[:pos]

	pos = strings.LastIndex(addrs, "|")
	if pos != -1 {
		oracleAddr = addrs[pos+1:]
		addrs = addrs[:pos]
	}
	zks = strings.Split(addrs, ",")
	return
}
<|endoftext|>


<commit_before>// Package websockets implements a WebSocket server by executing
// a command and piping its input and output through the WebSocket
// connection.
package websockets

import (
	"net/http"

	"github.com/mholt/caddy/middleware"
	"golang.org/x/net/websocket"
)

type (
	// WebSockets is a type that holds configuration for the
	// websocket middleware generally, like a list of all the
	// websocket endpoints.
	WebSockets struct {
		// Next is the next HTTP handler in the chain for when the path doesn't match
		Next middleware.Handler

		// Sockets holds all the web socket endpoint configurations
		Sockets []Config
	}

	// WSConfig holds the configuration for a single websocket
	// endpoint which may serve multiple websocket connections.
	Config struct {
		Path      string
		Command   string
		Arguments []string
		Respawn   bool // TODO: Not used, but parser supports it until we decide on it
	}
)

// ServeHTTP converts the HTTP request to a WebSocket connection and serves it up.
func (ws WebSockets) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) {
	for _, sockconfig := range ws.Sockets {
		if middleware.Path(r.URL.Path).Matches(sockconfig.Path) {
			socket := WebSocket{
				Config:  sockconfig,
				Request: r,
			}
			websocket.Handler(socket.Handle).ServeHTTP(w, r)
			return 0, nil
		}
	}

	// Didn't match a websocket path, so pass-thru
	return ws.Next.ServeHTTP(w, r)
}

var (
	// See CGI spec, 4.1.4
	GatewayInterface string

	// See CGI spec, 4.1.17
	ServerSoftware string
)
<commit_msg>Fix lint warnings for middleware/websockets<commit_after>// Package websockets implements a WebSocket server by executing
// a command and piping its input and output through the WebSocket
// connection.
package websockets

import (
	"net/http"

	"github.com/mholt/caddy/middleware"
	"golang.org/x/net/websocket"
)

type (
	// WebSockets is a type that holds configuration for the
	// websocket middleware generally, like a list of all the
	// websocket endpoints.
	WebSockets struct {
		// Next is the next HTTP handler in the chain for when the path doesn't match
		Next middleware.Handler

		// Sockets holds all the web socket endpoint configurations
		Sockets []Config
	}

	// Config holds the configuration for a single websocket
	// endpoint which may serve multiple websocket connections.
	Config struct {
		Path      string
		Command   string
		Arguments []string
		Respawn   bool // TODO: Not used, but parser supports it until we decide on it
	}
)

// ServeHTTP converts the HTTP request to a WebSocket connection and serves it up.
func (ws WebSockets) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) {
	for _, sockconfig := range ws.Sockets {
		if middleware.Path(r.URL.Path).Matches(sockconfig.Path) {
			socket := WebSocket{
				Config:  sockconfig,
				Request: r,
			}
			websocket.Handler(socket.Handle).ServeHTTP(w, r)
			return 0, nil
		}
	}

	// Didn't match a websocket path, so pass-thru
	return ws.Next.ServeHTTP(w, r)
}

var (
	// GatewayInterface is the dialect of CGI being used by the server
	// to communicate with the script.  See CGI spec, 4.1.4
	GatewayInterface string

	// ServerSoftware is the name and version of the information server
	// software making the CGI request.  See CGI spec, 4.1.17
	ServerSoftware string
)
<|endoftext|>


<commit_before>package streams

import (
	"fmt"
	"sync"
	"time"

	"github.com/FederationOfFathers/dashboard/db"
	"github.com/uber-go/zap"
)

var Streams = []*db.Stream{}
var lock sync.Mutex
var logger = zap.New(zap.NewJSONEncoder()).With(zap.String("module", "streams"))
var channel string

var DB *db.DB

func Init(notifySlackChannel string) error {
	channel = notifySlackChannel
	updated()
	return nil
}

func updated() {
	if s, err := DB.Streams(); err != nil {
		logger.Error("Error updating streams", zap.Error(err))
	} else {
		Streams = s
	}
}

func Mind() {
	go mind()
}

func MindList() {
	go func() {
		uptimer := time.Tick(1 * time.Minute)
		for {
			select {
			case <-uptimer:
				updated()
			}
		}
	}()
}

func mind() {
	mindYoutube()
	mindTwitch()
	uptimer := time.Tick(1 * time.Minute)
	twtimer := time.Tick(1 * time.Minute)
	yttimer := time.Tick(1 * time.Minute)
	bptimer := time.Tick(1 * time.Minute)
	for {
		select {
		case <-uptimer:
			updated()
		case <-twtimer:
			mindTwitch()
		case <-yttimer:
			mindYoutube()
		case <-bptimer:
			mindBeam()
		}
	}
}

func Owner(s *db.Stream) (*db.Member, error) {
	return DB.MemberByID(s.MemberID)
}

func Add(kind, identifier, userID string) error {
	member, err := DB.MemberBySlackID(userID)
	if err != nil {
		return err
	}
	switch kind {
	case "twitch":
		err := DB.Exec(
			"INSERT INTO `streams` (`member_id`,`twitch`) VALUES (?,?) ON DUPLICATE KEY UPDATE `twitch`=?",
			member.ID,
			identifier,
			identifier,
		).Error
		updated()
		return err
	case "beam":
		err := DB.Exec(
			"INSERT INTO `streams` (`member_id`,`beam`) VALUES (?,?) ON DUPLICATE KEY UPDATE `beam`=?",
			member.ID,
			identifier,
			identifier,
		).Error
		updated()
		return err
	case "youtube":
		err := DB.Exec(
			"INSERT INTO `streams` (`member_id`,`youtube`) VALUES (?,?) ON DUPLICATE KEY UPDATE `youtube`=?",
			member.ID,
			identifier,
			identifier,
		).Error
		updated()
		return err
	}
	return fmt.Errorf("unknown kind!")
}

func Remove(memberID int, kind string) error {
	switch kind {
	case "twitch":
		err := DB.Exec("UPDATE `streams` SET `twitch` = '' WHERE `id` = ?", memberID).Error
		updated()
		return err
	case "beam":
		err := DB.Exec("UPDATE `streams` SET `beam` = '' WHERE `id` = ?", memberID).Error
		updated()
		return err
	case "youtube":
		err := DB.Exec("UPDATE `streams` SET `youtube` = '' WHERE `id` = ?", memberID).Error
		updated()
		return err
	}
	return fmt.Errorf("unknown kind!")
}
<commit_msg>minBeam() on initial mind()<commit_after>package streams

import (
	"fmt"
	"sync"
	"time"

	"github.com/FederationOfFathers/dashboard/db"
	"github.com/uber-go/zap"
)

var Streams = []*db.Stream{}
var lock sync.Mutex
var logger = zap.New(zap.NewJSONEncoder()).With(zap.String("module", "streams"))
var channel string

var DB *db.DB

func Init(notifySlackChannel string) error {
	channel = notifySlackChannel
	updated()
	return nil
}

func updated() {
	if s, err := DB.Streams(); err != nil {
		logger.Error("Error updating streams", zap.Error(err))
	} else {
		Streams = s
	}
}

func Mind() {
	go mind()
}

func MindList() {
	go func() {
		uptimer := time.Tick(1 * time.Minute)
		for {
			select {
			case <-uptimer:
				updated()
			}
		}
	}()
}

func mind() {
	mindYoutube()
	mindTwitch()
	mindBeam()
	uptimer := time.Tick(1 * time.Minute)
	twtimer := time.Tick(1 * time.Minute)
	yttimer := time.Tick(1 * time.Minute)
	bptimer := time.Tick(1 * time.Minute)
	for {
		select {
		case <-uptimer:
			updated()
		case <-twtimer:
			mindTwitch()
		case <-yttimer:
			mindYoutube()
		case <-bptimer:
			mindBeam()
		}
	}
}

func Owner(s *db.Stream) (*db.Member, error) {
	return DB.MemberByID(s.MemberID)
}

func Add(kind, identifier, userID string) error {
	member, err := DB.MemberBySlackID(userID)
	if err != nil {
		return err
	}
	switch kind {
	case "twitch":
		err := DB.Exec(
			"INSERT INTO `streams` (`member_id`,`twitch`) VALUES (?,?) ON DUPLICATE KEY UPDATE `twitch`=?",
			member.ID,
			identifier,
			identifier,
		).Error
		updated()
		return err
	case "beam":
		err := DB.Exec(
			"INSERT INTO `streams` (`member_id`,`beam`) VALUES (?,?) ON DUPLICATE KEY UPDATE `beam`=?",
			member.ID,
			identifier,
			identifier,
		).Error
		updated()
		return err
	case "youtube":
		err := DB.Exec(
			"INSERT INTO `streams` (`member_id`,`youtube`) VALUES (?,?) ON DUPLICATE KEY UPDATE `youtube`=?",
			member.ID,
			identifier,
			identifier,
		).Error
		updated()
		return err
	}
	return fmt.Errorf("unknown kind!")
}

func Remove(memberID int, kind string) error {
	switch kind {
	case "twitch":
		err := DB.Exec("UPDATE `streams` SET `twitch` = '' WHERE `id` = ?", memberID).Error
		updated()
		return err
	case "beam":
		err := DB.Exec("UPDATE `streams` SET `beam` = '' WHERE `id` = ?", memberID).Error
		updated()
		return err
	case "youtube":
		err := DB.Exec("UPDATE `streams` SET `youtube` = '' WHERE `id` = ?", memberID).Error
		updated()
		return err
	}
	return fmt.Errorf("unknown kind!")
}
<|endoftext|>


<commit_before>package gengateway

import (
	"errors"
	"fmt"
	"go/format"
	"path"
	"path/filepath"
	"strings"

	"github.com/golang/glog"
	"github.com/golang/protobuf/proto"
	plugin "github.com/golang/protobuf/protoc-gen-go/plugin"
	"github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway/descriptor"
	gen "github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway/generator"
)

var (
	errNoTargetService = errors.New("no target service defined in the file")
)

type generator struct {
	reg               *descriptor.Registry
	baseImports       []descriptor.GoPackage
	useRequestContext bool
}

// New returns a new generator which generates grpc gateway files.
func New(reg *descriptor.Registry, useRequestContext bool) gen.Generator {
	var imports []descriptor.GoPackage
	for _, pkgpath := range []string{
		"io",
		"net/http",
		"github.com/grpc-ecosystem/grpc-gateway/runtime",
		"github.com/grpc-ecosystem/grpc-gateway/utilities",
		"github.com/golang/protobuf/proto",
		"golang.org/x/net/context",
		"google.golang.org/grpc",
		"google.golang.org/grpc/codes",
		"google.golang.org/grpc/grpclog",
	} {
		pkg := descriptor.GoPackage{
			Path: pkgpath,
			Name: path.Base(pkgpath),
		}
		if err := reg.ReserveGoPackageAlias(pkg.Name, pkg.Path); err != nil {
			for i := 0; ; i++ {
				alias := fmt.Sprintf("%s_%d", pkg.Name, i)
				if err := reg.ReserveGoPackageAlias(alias, pkg.Path); err != nil {
					continue
				}
				pkg.Alias = alias
				break
			}
		}
		imports = append(imports, pkg)
	}
	return &generator{reg: reg, baseImports: imports, useRequestContext: useRequestContext}
}

func (g *generator) Generate(targets []*descriptor.File) ([]*plugin.CodeGeneratorResponse_File, error) {
	var files []*plugin.CodeGeneratorResponse_File
	for _, file := range targets {
		glog.V(1).Infof("Processing %s", file.GetName())
		code, err := g.generate(file)
		if err == errNoTargetService {
			glog.V(1).Infof("%s: %v", file.GetName(), err)
			continue
		}
		if err != nil {
			return nil, err
		}
		formatted, err := format.Source([]byte(code))
		if err != nil {
			glog.Errorf("%v: %s", err, code)
			return nil, err
		}
		name := file.GetName()
		ext := filepath.Ext(name)
		base := strings.TrimSuffix(name, ext)
		output := fmt.Sprintf("%s.pb.gw.go", base)
		files = append(files, &plugin.CodeGeneratorResponse_File{
			Name:    proto.String(output),
			Content: proto.String(string(formatted)),
		})
		glog.V(1).Infof("Will emit %s", output)
	}
	return files, nil
}

func (g *generator) generate(file *descriptor.File) (string, error) {
	pkgSeen := make(map[string]bool)
	var imports []descriptor.GoPackage
	for _, pkg := range g.baseImports {
		pkgSeen[pkg.Path] = true
		imports = append(imports, pkg)
	}
	for _, svc := range file.Services {
		for _, m := range svc.Methods {
			pkg := m.RequestType.File.GoPkg
			if pkg == file.GoPkg {
				continue
			}
			if pkgSeen[pkg.Path] {
				continue
			}
			pkgSeen[pkg.Path] = true
			imports = append(imports, pkg)
		}
	}
	return applyTemplate(param{File: file, Imports: imports, UseRequestContext: g.useRequestContext})
}
<commit_msg>Do not add imports for a method when this latter is not exposed by the gateway.<commit_after>package gengateway

import (
	"errors"
	"fmt"
	"go/format"
	"path"
	"path/filepath"
	"strings"

	"github.com/golang/glog"
	"github.com/golang/protobuf/proto"
	plugin "github.com/golang/protobuf/protoc-gen-go/plugin"
	"github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway/descriptor"
	gen "github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway/generator"
	options "github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis/google/api"
)

var (
	errNoTargetService = errors.New("no target service defined in the file")
)

type generator struct {
	reg               *descriptor.Registry
	baseImports       []descriptor.GoPackage
	useRequestContext bool
}

// New returns a new generator which generates grpc gateway files.
func New(reg *descriptor.Registry, useRequestContext bool) gen.Generator {
	var imports []descriptor.GoPackage
	for _, pkgpath := range []string{
		"io",
		"net/http",
		"github.com/grpc-ecosystem/grpc-gateway/runtime",
		"github.com/grpc-ecosystem/grpc-gateway/utilities",
		"github.com/golang/protobuf/proto",
		"golang.org/x/net/context",
		"google.golang.org/grpc",
		"google.golang.org/grpc/codes",
		"google.golang.org/grpc/grpclog",
	} {
		pkg := descriptor.GoPackage{
			Path: pkgpath,
			Name: path.Base(pkgpath),
		}
		if err := reg.ReserveGoPackageAlias(pkg.Name, pkg.Path); err != nil {
			for i := 0; ; i++ {
				alias := fmt.Sprintf("%s_%d", pkg.Name, i)
				if err := reg.ReserveGoPackageAlias(alias, pkg.Path); err != nil {
					continue
				}
				pkg.Alias = alias
				break
			}
		}
		imports = append(imports, pkg)
	}
	return &generator{reg: reg, baseImports: imports, useRequestContext: useRequestContext}
}

func (g *generator) Generate(targets []*descriptor.File) ([]*plugin.CodeGeneratorResponse_File, error) {
	var files []*plugin.CodeGeneratorResponse_File
	for _, file := range targets {
		glog.V(1).Infof("Processing %s", file.GetName())
		code, err := g.generate(file)
		if err == errNoTargetService {
			glog.V(1).Infof("%s: %v", file.GetName(), err)
			continue
		}
		if err != nil {
			return nil, err
		}
		formatted, err := format.Source([]byte(code))
		if err != nil {
			glog.Errorf("%v: %s", err, code)
			return nil, err
		}
		name := file.GetName()
		ext := filepath.Ext(name)
		base := strings.TrimSuffix(name, ext)
		output := fmt.Sprintf("%s.pb.gw.go", base)
		files = append(files, &plugin.CodeGeneratorResponse_File{
			Name:    proto.String(output),
			Content: proto.String(string(formatted)),
		})
		glog.V(1).Infof("Will emit %s", output)
	}
	return files, nil
}

func (g *generator) generate(file *descriptor.File) (string, error) {
	pkgSeen := make(map[string]bool)
	var imports []descriptor.GoPackage
	for _, pkg := range g.baseImports {
		pkgSeen[pkg.Path] = true
		imports = append(imports, pkg)
	}
	for _, svc := range file.Services {
		for _, m := range svc.Methods {
			pkg := m.RequestType.File.GoPkg
			if m.Options == nil || !proto.HasExtension(m.Options, options.E_Http) ||
				pkg == file.GoPkg || pkgSeen[pkg.Path] {
				continue
			}
			pkgSeen[pkg.Path] = true
			imports = append(imports, pkg)
		}
	}
	return applyTemplate(param{File: file, Imports: imports, UseRequestContext: g.useRequestContext})
}
<|endoftext|>


<commit_before>package srt

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"regexp"
	"strconv"
	"strings"
	"time"
)

func dropCR(data []byte) []byte {
	if len(data) > 0 && data[len(data)-1] == '\r' {
		return data[0 : len(data)-1]
	}
	return data
}

// A bufio.Scanner-function to read a string until there is a double-newline (one empty line).
// Supports reading both LF and CRLF
func scanDoubleNewline(data []byte, atEOF bool) (advance int, token []byte, err error) {
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}
	if i := bytes.Index(data, []byte{'\n', '\n'}); i >= 0 {
		// We have a full double newline-terminated line.
		return i + 2, dropCR(data[0:i]), nil
	} else if i := bytes.Index(data, []byte{'\n', '\r', '\n'}); i >= 0 {
		// We have a full double newline-terminated line.
		return i + 3, dropCR(data[0:i]), nil
	}
	// If we're at EOF, we have a final, non-terminated line. Return it.
	if atEOF {
		return len(data), dropCR(data), nil
	}
	// Request more data.
	return 0, nil, nil
}

// A struct containing the state for scanning
// a .srt-file
type SubtitleScanner struct {
	scanner *bufio.Scanner
	nextSub Subtitle
	err     error
}

// Creates a new SubtitleScanner from the given io.Reader.
func NewScanner(r io.Reader) SubtitleScanner {
	s := bufio.NewScanner(r)
	s.Split(scanDoubleNewline)
	return SubtitleScanner{s, Subtitle{}, nil}
}

// Parse a time formatted as hours:minutes:seconds,milliseconds, strictly formatted as 00:00:00,000
func parseTime(input string) (time.Duration, error) {
	regex := regexp.MustCompile(`(\d{2}):(\d{2}):(\d{2}),(\d{3})`)
	matches := regex.FindStringSubmatch(input)

	if len(matches) < 4 {
		return time.Duration(0), errors.New("Invalid time format")
	}

	hour, err := strconv.Atoi(matches[1])
	if err != nil {
		return time.Duration(0), err
	}
	minute, err := strconv.Atoi(matches[2])
	if err != nil {
		return time.Duration(0), err
	}
	second, err := strconv.Atoi(matches[3])
	if err != nil {
		return time.Duration(0), err
	}
	millisecond, err := strconv.Atoi(matches[4])
	if err != nil {
		return time.Duration(0), err
	}

	return time.Duration(time.Duration(hour)*time.Hour + time.Duration(minute)*time.Minute + time.Duration(second)*time.Second + time.Duration(millisecond)*time.Millisecond), nil
}

// Parse a bounding rectangle definition
// (X1:left X2:right Y1:top Y2:bottom)
func parseRect(input string) (Rectangle, error) {
	regex := regexp.MustCompile(`X1:(\d+) X2:(\d+) Y1:(\d+) Y2:(\d+)`)
	matches := regex.FindStringSubmatch(input)

	if len(matches) < 4 {
		return Rectangle{0, 0, 0, 0}, errors.New("Invalid bounding format")
	}

	left, err := strconv.Atoi(matches[1])
	if err != nil {
		return Rectangle{0, 0, 0, 0}, err
	}
	right, err := strconv.Atoi(matches[2])
	if err != nil {
		return Rectangle{0, 0, 0, 0}, err
	}
	top, err := strconv.Atoi(matches[3])
	if err != nil {
		return Rectangle{0, 0, 0, 0}, err
	}
	bottom, err := strconv.Atoi(matches[4])
	if err != nil {
		return Rectangle{0, 0, 0, 0}, err
	}

	return Rectangle{left, right, top, bottom}, nil
}

// Advances the SubtitleScanner-state, reading a new
// Subtitle-object. Returns true if an object was read
// or false if an error ocurred
func (s *SubtitleScanner) Scan() bool {
	if s.scanner.Scan() {
		var (
			nextnum           int
			start             time.Duration
			end               time.Duration
			subtitletext      string
			subtitleRectangle Rectangle
		)

		str := strings.Split(s.scanner.Text(), "\n")

		for i := 0; i < len(str); i++ {
			text := strings.TrimRight(str[i], "\r")
			switch i {
			case 0:
				num, err := strconv.Atoi(text)
				if err != nil {
					s.err = err
					return false
				}
				nextnum = num
			case 1:
				elements := strings.Split(text, " ")
				if len(elements) >= 3 {
					startTime, err := parseTime(elements[0])
					if err != nil {
						s.err = err
						return false
					}
					endTime, err := parseTime(elements[2])
					if err != nil {
						s.err = err
						return false
					}
					start = startTime
					end = endTime

					if len(elements) >= 7 {
						rect, err := parseRect(strings.Join(elements[3:7], " "))
						if err != nil {
							s.err = err
							return false
						}

						subtitleRectangle = rect
					} else {
						subtitleRectangle = Rectangle{0, 0, 0, 0}
					}
				} else {
					s.err = fmt.Errorf("srt: Invalid timestamp on row: %s", text)
					return false
				}
			default:
				if len(subtitletext) > 0 {
					subtitletext += "\n"
				}
				subtitletext += text
			}
		}

		s.nextSub = Subtitle{nextnum, start, end, subtitletext, subtitleRectangle}

		return true
	} else {
		return false
	}
}

// Gets the error of the SubtitleScanner.
// Returns nil if the last error was EOF
func (s *SubtitleScanner) Err() error {
	if s.err != nil {
		return s.err
	}
	return s.scanner.Err()
}

// Get the last read subtitle-object
func (s *SubtitleScanner) Subtitle() Subtitle {
	return s.nextSub
}
<commit_msg>Add variable for controlling validation<commit_after>package srt

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"io"
	"regexp"
	"strconv"
	"strings"
	"time"
)

type ValidationStrictness int

const (
        StrictValidation ValidationStrictness = iota
	LenientValidation
	SkipInvalid
)

var InputValidationStrictness ValidationStrictness = StrictValidation

func dropCR(data []byte) []byte {
	if len(data) > 0 && data[len(data)-1] == '\r' {
		return data[0 : len(data)-1]
	}
	return data
}

// A bufio.Scanner-function to read a string until there is a double-newline (one empty line).
// Supports reading both LF and CRLF
func scanDoubleNewline(data []byte, atEOF bool) (advance int, token []byte, err error) {
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}
	if i := bytes.Index(data, []byte{'\n', '\n'}); i >= 0 {
		// We have a full double newline-terminated line.
		return i + 2, dropCR(data[0:i]), nil
	} else if i := bytes.Index(data, []byte{'\n', '\r', '\n'}); i >= 0 {
		// We have a full double newline-terminated line.
		return i + 3, dropCR(data[0:i]), nil
	}
	// If we're at EOF, we have a final, non-terminated line. Return it.
	if atEOF {
		return len(data), dropCR(data), nil
	}
	// Request more data.
	return 0, nil, nil
}

// A struct containing the state for scanning
// a .srt-file
type SubtitleScanner struct {
	scanner *bufio.Scanner
	nextSub Subtitle
	err     error
}

// Creates a new SubtitleScanner from the given io.Reader.
func NewScanner(r io.Reader) SubtitleScanner {
	s := bufio.NewScanner(r)
	s.Split(scanDoubleNewline)
	return SubtitleScanner{s, Subtitle{}, nil}
}

// Parse a time formatted as hours:minutes:seconds,milliseconds, strictly formatted as 00:00:00,000
func parseTime(input string) (time.Duration, error) {
	regex := regexp.MustCompile(`(\d{2}):(\d{2}):(\d{2}),(\d{3})`)
	matches := regex.FindStringSubmatch(input)

	if len(matches) < 4 {
		return time.Duration(0), errors.New("Invalid time format")
	}

	hour, err := strconv.Atoi(matches[1])
	if err != nil {
		return time.Duration(0), err
	}
	minute, err := strconv.Atoi(matches[2])
	if err != nil {
		return time.Duration(0), err
	}
	second, err := strconv.Atoi(matches[3])
	if err != nil {
		return time.Duration(0), err
	}
	millisecond, err := strconv.Atoi(matches[4])
	if err != nil {
		return time.Duration(0), err
	}

	return time.Duration(time.Duration(hour)*time.Hour + time.Duration(minute)*time.Minute + time.Duration(second)*time.Second + time.Duration(millisecond)*time.Millisecond), nil
}

// Parse a bounding rectangle definition
// (X1:left X2:right Y1:top Y2:bottom)
func parseRect(input string) (result Rectangle, errResult error) {
	regex := regexp.MustCompile(`X1:(\d+) X2:(\d+) Y1:(\d+) Y2:(\d+)`)
	matches := regex.FindStringSubmatch(input)

	// If validation is set to lenient, set this optional
	// element be ignored if the format is invalid
	if InputValidationStrictness == LenientValidation {
		defer func() {
			if errResult != nil {
				result = Rectangle{0,0,0,0}
				errResult = nil
			}
		}()
	}

	if len(matches) < 4 {
		return Rectangle{0, 0, 0, 0}, errors.New("Invalid bounding format")
	}

	left, err := strconv.Atoi(matches[1])
	if err != nil {
		return Rectangle{0, 0, 0, 0}, err
	}
	right, err := strconv.Atoi(matches[2])
	if err != nil {
		return Rectangle{0, 0, 0, 0}, err
	}
	top, err := strconv.Atoi(matches[3])
	if err != nil {
		return Rectangle{0, 0, 0, 0}, err
	}
	bottom, err := strconv.Atoi(matches[4])
	if err != nil {
		return Rectangle{0, 0, 0, 0}, err
	}

	return Rectangle{left, right, top, bottom}, nil
}

// Advances the SubtitleScanner-state, reading a new
// Subtitle-object. Returns true if an object was read
// or false if an error ocurred
func (s *SubtitleScanner) Scan() (wasRead bool) {
	if s.scanner.Scan() {
		var (
			nextnum           int
			start             time.Duration
			end               time.Duration
			subtitletext      string
			subtitleRectangle Rectangle
		)

		// If we are reckless, ignore invalid Subtitles and just
		// find the next one
		if InputValidationStrictness == SkipInvalid {
			defer func() {
				s.err = nil
				if !wasRead {
					wasRead = s.Scan()
					// If we dont' return true here, then
					// the underlying scanner returned false.
					// This means that we either had a read error
					// or the reader is empty
				}
			}()
		}

		str := strings.Split(s.scanner.Text(), "\n")

		for i := 0; i < len(str); i++ {
			text := strings.TrimRight(str[i], "\r")
			switch i {
			case 0:
				num, err := strconv.Atoi(text)
				if err != nil {
					s.err = err
					return false
				}
				nextnum = num
			case 1:
				elements := strings.Split(text, " ")
				if len(elements) >= 3 {
					startTime, err := parseTime(elements[0])
					if err != nil {
						s.err = err
						return false
					}
					endTime, err := parseTime(elements[2])
					if err != nil {
						s.err = err
						return false
					}
					start = startTime
					end = endTime

					if len(elements) >= 7 {
						rect, err := parseRect(strings.Join(elements[3:7], " "))
						if err != nil {
							s.err = err
							return false
						}

						subtitleRectangle = rect
					} else {
						subtitleRectangle = Rectangle{0, 0, 0, 0}
					}
				} else {
					s.err = fmt.Errorf("srt: Invalid timestamp on row: %s", text)
					return false
				}
			default:
				if len(subtitletext) > 0 {
					subtitletext += "\n"
				}
				subtitletext += text
			}
		}

		s.nextSub = Subtitle{nextnum, start, end, subtitletext, subtitleRectangle}

		return true
	} else {
		return false
	}
}

// Gets the error of the SubtitleScanner.
// Returns nil if the last error was EOF
func (s *SubtitleScanner) Err() error {
	if s.err != nil {
		return s.err
	}
	return s.scanner.Err()
}

// Get the last read subtitle-object
func (s *SubtitleScanner) Subtitle() Subtitle {
	return s.nextSub
}
<|endoftext|>


<commit_before>package taskmanager

import (
	"fmt"
	"testing"
)

func TestNewJobManager(t *testing.T) {
	var jm interface{}

	t.Run("Initialises and returns a JobManager", func(t *testing.T) {
		jm = NewJobManager()

		switch jm.(type) {
		case JobManager:
		default:
			t.Errorf("NewJobManager() error = Received %T, expected JobManager", jm)
		}
	})

	list := jm.(JobManager).JobList
	t.Run("Initialises with a list of Jobs", func(t *testing.T) {
		if len(list) == 0 {
			t.Errorf("NewJobManager().JobList error = list is empty")
		}
	})

	// test default job functions
	for _, j := range []string{"post-to-web", "get-from-web", "log"} {
		t.Run(fmt.Sprintf("Initialises JobList and contains %s", j), func(t *testing.T) {
			_, ok := list[j]
			if !ok {
				t.Errorf("NewJobManager().JobList error = no such entry %s", j)
			}
		})
	}

}

func simpleJob(jn JobNotification) (output map[string]interface{}, err error) { return }
func TestJobManager_AddJob(t *testing.T) {
	jm := NewJobManager()
	jm.AddJob("simple-job", simpleJob)

	t.Run("adds jobs to JobList", func(t *testing.T) {
		_, ok := jm.JobList["simple-job"]
		if !ok {
			t.Errorf("NewJobManager().JobList error = no such entry 'simple-job'")
		}
	})
}

func TestJobManager_Consume(t *testing.T) {
	jm := NewJobManager()
	jm.AddJob("simple-job", simpleJob)

	o, err := jm.Consume(`
{
  "UUID": "abcd-efg-1234-foo",
  "Register": "name",
  "Type": "simple-job"
}
`)

	t.Run("consumes and runs a job", func(t *testing.T) {
		if err != nil {
			t.Errorf("Consume() error = %v", err)
		}
	})

	t.Run("returned job data has the correct UUID", func(t *testing.T) {
		if o["UUID"].(string) != "abcd-efg-1234-foo" {
			t.Errorf("Consume()[uuid] error = received %q, expected abcd-efg-1234-foo", o["UUID"].(string))
		}
	})

	t.Run("returned job data has the correct Register value", func(t *testing.T) {
		if o["Register"].(string) != "name" {
			t.Errorf("Consume()[register] error = received %q, expected name", o["Register"].(string))
		}
	})

	t.Run("returned job data has the correct Failed state", func(t *testing.T) {
		if o["Failed"].(bool) != false {
			t.Errorf("Consume()[failed] error = received %b, expected false", o["Failed"].(bool))
		}
	})

	t.Run("returned job data has the correct Data map", func(t *testing.T) {
		switch o["Data"].(type) {
		case map[string]interface{}:
		default:
			t.Errorf("Consume()[data] error = received %T, expected map[string]interface{}", o["Data"])
		}
	})

	t.Run("returned job data has a Start time", func(t *testing.T) {
		if o["Start"].(string) == "" {
			t.Errorf("Consume()[start] error = received %q, expected something useful", o["Start"].(string))
		}
	})

	t.Run("returned job data has an End time", func(t *testing.T) {
		if o["End"].(string) == "" {
			t.Errorf("Consume()[end] error = received %q, expected something useful", o["End"].(string))
		}
	})

	t.Run("returned job data has a Duration", func(t *testing.T) {
		if o["End"].(string) == " ms" {
			t.Errorf("Consume()[duration] error = received %q, expected something useful", o["Duraction"].(string))
		}
	})

}
<commit_msg>Format string with bool correctly<commit_after>package taskmanager

import (
	"fmt"
	"testing"
)

func TestNewJobManager(t *testing.T) {
	var jm interface{}

	t.Run("Initialises and returns a JobManager", func(t *testing.T) {
		jm = NewJobManager()

		switch jm.(type) {
		case JobManager:
		default:
			t.Errorf("NewJobManager() error = Received %T, expected JobManager", jm)
		}
	})

	list := jm.(JobManager).JobList
	t.Run("Initialises with a list of Jobs", func(t *testing.T) {
		if len(list) == 0 {
			t.Errorf("NewJobManager().JobList error = list is empty")
		}
	})

	// test default job functions
	for _, j := range []string{"post-to-web", "get-from-web", "log"} {
		t.Run(fmt.Sprintf("Initialises JobList and contains %s", j), func(t *testing.T) {
			_, ok := list[j]
			if !ok {
				t.Errorf("NewJobManager().JobList error = no such entry %s", j)
			}
		})
	}

}

func simpleJob(jn JobNotification) (output map[string]interface{}, err error) { return }
func TestJobManager_AddJob(t *testing.T) {
	jm := NewJobManager()
	jm.AddJob("simple-job", simpleJob)

	t.Run("adds jobs to JobList", func(t *testing.T) {
		_, ok := jm.JobList["simple-job"]
		if !ok {
			t.Errorf("NewJobManager().JobList error = no such entry 'simple-job'")
		}
	})
}

func TestJobManager_Consume(t *testing.T) {
	jm := NewJobManager()
	jm.AddJob("simple-job", simpleJob)

	o, err := jm.Consume(`
{
  "UUID": "abcd-efg-1234-foo",
  "Register": "name",
  "Type": "simple-job"
}
`)

	t.Run("consumes and runs a job", func(t *testing.T) {
		if err != nil {
			t.Errorf("Consume() error = %v", err)
		}
	})

	t.Run("returned job data has the correct UUID", func(t *testing.T) {
		if o["UUID"].(string) != "abcd-efg-1234-foo" {
			t.Errorf("Consume()[uuid] error = received %q, expected abcd-efg-1234-foo", o["UUID"].(string))
		}
	})

	t.Run("returned job data has the correct Register value", func(t *testing.T) {
		if o["Register"].(string) != "name" {
			t.Errorf("Consume()[register] error = received %q, expected name", o["Register"].(string))
		}
	})

	t.Run("returned job data has the correct Failed state", func(t *testing.T) {
		if o["Failed"].(bool) != false {
			t.Errorf("Consume()[failed] error = received %v, expected false", o["Failed"].(bool))
		}
	})

	t.Run("returned job data has the correct Data map", func(t *testing.T) {
		switch o["Data"].(type) {
		case map[string]interface{}:
		default:
			t.Errorf("Consume()[data] error = received %T, expected map[string]interface{}", o["Data"])
		}
	})

	t.Run("returned job data has a Start time", func(t *testing.T) {
		if o["Start"].(string) == "" {
			t.Errorf("Consume()[start] error = received %q, expected something useful", o["Start"].(string))
		}
	})

	t.Run("returned job data has an End time", func(t *testing.T) {
		if o["End"].(string) == "" {
			t.Errorf("Consume()[end] error = received %q, expected something useful", o["End"].(string))
		}
	})

	t.Run("returned job data has a Duration", func(t *testing.T) {
		if o["End"].(string) == " ms" {
			t.Errorf("Consume()[duration] error = received %q, expected something useful", o["Duraction"].(string))
		}
	})

}
<|endoftext|>


<commit_before>package helpers

import (
	"encoding/json"
	"os"
	"path/filepath"
)

type Config struct {
	ApiEndpoint string `json:"api"`
	AppsDomain  string `json:"apps_domain"`

	AdminUser     string `json:"admin_user"`
	AdminPassword string `json:"admin_password"`

	PersistentAppHost      string `json:"persistent_app_host"`
	PersistentAppSpace     string `json:"persistent_app_space"`
	PersistentAppOrg       string `json:"persistent_app_org"`
	PersistentAppQuotaName string `json:"persistent_app_quota_name"`

	SkipSSLValidation bool `json:"skip_ssl_validation"`

	ArtifactsDirectory string `json:"artifacts_directory"`
}

var loadedConfig *Config

func LoadConfig() Config {
	if loadedConfig == nil {
		loadedConfig = loadConfigJsonFromPath()
	}

	if loadedConfig.ApiEndpoint == "" {
		panic("missing configuration 'api'")
	}

	if loadedConfig.AdminUser == "" {
		panic("missing configuration 'admin_user'")
	}

	if loadedConfig.ApiEndpoint == "" {
		panic("missing configuration 'admin_password'")
	}

	return *loadedConfig
}

func loadConfigJsonFromPath() *Config {
	var config *Config = &Config{
		PersistentAppHost:      "CATS-persistent-app",
		PersistentAppSpace:     "CATS-persistent-space",
		PersistentAppOrg:       "CATS-persistent-org",
		PersistentAppQuotaName: "CATS-persistent-quota",

		ArtifactsDirectory: filepath.Join("..", "results"),
	}

	path := configPath()

	configFile, err := os.Open(path)
	if err != nil {
		panic(err)
	}

	decoder := json.NewDecoder(configFile)
	err = decoder.Decode(config)
	if err != nil {
		panic(err)
	}

	return config
}

func configPath() string {
	path := os.Getenv("CONFIG")
	if path == "" {
		panic("Must set $CONFIG to point to an integration config .json file.")
	}

	return path
}
<commit_msg>add new field for overriding default timeout<commit_after>package helpers

import (
	"encoding/json"
	"os"
	"path/filepath"
	"time"
)

type Config struct {
	ApiEndpoint string `json:"api"`
	AppsDomain  string `json:"apps_domain"`

	AdminUser     string `json:"admin_user"`
	AdminPassword string `json:"admin_password"`

	PersistentAppHost      string `json:"persistent_app_host"`
	PersistentAppSpace     string `json:"persistent_app_space"`
	PersistentAppOrg       string `json:"persistent_app_org"`
	PersistentAppQuotaName string `json:"persistent_app_quota_name"`

	SkipSSLValidation bool `json:"skip_ssl_validation"`

	ArtifactsDirectory string `json:"artifacts_directory"`

	DefaultTimeout  time.Duration `json:"default_timeout"`
	CfPushTimeout   time.Duration `json:"cf_push_timeout"`
	LongCurlTimeout time.Duration `json:"long_curl_timeout"`
}

var loadedConfig *Config

func LoadConfig() Config {
	if loadedConfig == nil {
		loadedConfig = loadConfigJsonFromPath()
	}

	if loadedConfig.ApiEndpoint == "" {
		panic("missing configuration 'api'")
	}

	if loadedConfig.AdminUser == "" {
		panic("missing configuration 'admin_user'")
	}

	if loadedConfig.ApiEndpoint == "" {
		panic("missing configuration 'admin_password'")
	}

	return *loadedConfig
}

func loadConfigJsonFromPath() *Config {
	var config *Config = &Config{
		PersistentAppHost:      "CATS-persistent-app",
		PersistentAppSpace:     "CATS-persistent-space",
		PersistentAppOrg:       "CATS-persistent-org",
		PersistentAppQuotaName: "CATS-persistent-quota",

		ArtifactsDirectory: filepath.Join("..", "results"),
	}

	path := configPath()

	configFile, err := os.Open(path)
	if err != nil {
		panic(err)
	}

	decoder := json.NewDecoder(configFile)
	err = decoder.Decode(config)
	if err != nil {
		panic(err)
	}

	return config
}

func configPath() string {
	path := os.Getenv("CONFIG")
	if path == "" {
		panic("Must set $CONFIG to point to an integration config .json file.")
	}

	return path
}
<|endoftext|>


<commit_before>package svfs

import (
	"fmt"
	"os"
	"regexp"
	"strings"
	"time"

	"github.com/xlucas/swift"

	"bazil.org/fuse"
	"bazil.org/fuse/fs"
	"golang.org/x/net/context"
)

const (
	dirContentType  = "application/directory"
	linkContentType = "application/link"
)

var (
	folderRegex = regexp.MustCompile("^.+/$")
	subdirRegex = regexp.MustCompile(".*/.*$")
)

// Directory represents a standard directory entry.
type Directory struct {
	apex bool
	name string
	path string
	so   *swift.Object
	sh   swift.Headers
	c    *swift.Container
	cs   *swift.Container
}

// Attr fills file attributes of a directory within the current context.
func (d *Directory) Attr(ctx context.Context, a *fuse.Attr) error {
	a.Mode = os.ModeDir | os.FileMode(DefaultMode)
	a.Gid = uint32(DefaultGID)
	a.Uid = uint32(DefaultUID)
	a.Size = uint64(BlockSize)

	if d.so != nil {
		a.Atime = time.Now()
		a.Mtime = getMtime(d.so, d.sh)
		a.Ctime = a.Mtime
		a.Crtime = a.Mtime
	}

	return nil
}

// Create makes a new object node represented by a file. It returns
// an object node and an opened file handle.
func (d *Directory) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) {
	// Create an empty object in swift
	path := d.path + req.Name

	// New node
	node := &Object{name: req.Name, path: path, c: d.c, cs: d.cs}

	err := SwiftConnection.ObjectPutBytes(node.c.Name, node.path, nil, "")
	if err != nil {
		return nil, nil, err
	}

	// Get object handler
	fh, err := node.open(req.Flags, &resp.Flags)
	if err != nil {
		return nil, nil, err
	}

	// Get object info
	obj := &swift.Object{
		Name:         path,
		Bytes:        0,
		LastModified: time.Now(),
	}

	node.so = obj
	node.sh = map[string]string{}

	// Cache it
	directoryCache.Set(d.c.Name, d.path, req.Name, node)

	return node, fh, nil
}

// Export gives a direntry for the current directory node.
func (d *Directory) Export() fuse.Dirent {
	return fuse.Dirent{
		Name: d.name,
		Type: fuse.DT_Dir,
	}
}

// ReadDirAll reads the content of a directory and returns a
// list of children nodes as direntries, using/filling the
// cache of nodes.
func (d *Directory) ReadDirAll(ctx context.Context) (direntries []fuse.Dirent, err error) {
	var (
		dirs  = make(map[string]bool)
		tasks = make(chan Node, ListerConcurrency)
		count = 0
	)

	defer close(tasks)

	// Cache check
	if _, nodes := directoryCache.GetAll(d.c.Name, d.path); nodes != nil {
		for _, node := range nodes {
			direntries = append(direntries, node.Export())
		}
		return direntries, nil
	}

	// Fetch objects
	objects, err := SwiftConnection.ObjectsAll(d.c.Name, &swift.ObjectsOpts{
		Delimiter: '/',
		Prefix:    d.path,
	})
	if err != nil {
		return nil, err
	}

	var children = make(map[string]Node)

	// Fill cache
	for _, object := range objects {
		var (
			child    Node
			o        = object
			path     = object.Name
			fileName = strings.TrimSuffix(strings.TrimPrefix(o.Name, d.path), "/")
		)

		// This is a symlink
		if isSymlink(o, d.path) {
			child = &Symlink{path: path, name: fileName, c: d.c, so: &o, sh: swift.Headers{}, p: d}
			directoryLister.AddTask(child, tasks)
			child = nil
			count++
			goto finish
		}

		// This is a standard directory
		if isDirectory(o, d.path) {
			if !strings.HasSuffix(o.Name, "/") {
				path += "/"
			}
			child = &Directory{c: d.c, cs: d.cs, so: &o, sh: swift.Headers{}, path: path, name: fileName}
			dirs[fileName] = true
			goto finish
		}

		// This is a pseudo directory. Add it only if the real directory is missing
		if isPseudoDirectory(o, d.path) && !dirs[fileName] {
			child = &Directory{c: d.c, cs: d.cs, so: &o, sh: swift.Headers{}, path: path, name: fileName}
			dirs[fileName] = true
			goto finish
		}

		// This is a pure swift object
		if !strings.HasSuffix(o.Name, "/") {
			child = &Object{path: path, name: fileName, c: d.c, cs: d.cs, so: &o, sh: swift.Headers{}, p: d}

			// If we are writing to this object at the moment
			// we don't want to update the cache with this.
			if changeCache.Exist(d.c.Name, path) {
				child = changeCache.Get(d.c.Name, path)
				goto export
			}

			// Large objects needs extra information
			if isLargeObject(&o) {
				directoryLister.AddTask(child, tasks)
				child = nil
				count++
			}
		}

	finish:
		// Always fetch extra info if asked
		if child != nil && ExtraAttr {
			directoryLister.AddTask(child, tasks)
			child = nil
			count++
		}

	export:
		// Add nodes not requiring extra info
		if child != nil {
			direntries = append(direntries, child.Export())
			children[child.Name()] = child
		}

	}

	// Wait for directory lister to finish
	if count > 0 {
		done := 0
		for task := range tasks {
			done++
			direntries = append(direntries, task.Export())
			children[task.Name()] = task
			if done == count {
				break
			}
		}
	}

	directoryCache.AddAll(d.c.Name, d.path, d, children)

	return direntries, nil
}

// Link creates a hard link between two nodes.
func (d *Directory) Link(ctx context.Context, req *fuse.LinkRequest, old fs.Node) (node fs.Node, err error) {
	if object, ok := old.(*Object); ok {
		return object.copy(d, req.NewName)
	}
	if symlink, ok := old.(*Symlink); ok {
		return symlink.copy(d, req.NewName)
	}
	return nil, fuse.ENOTSUP
}

// Lookup gets a children node if its name matches the requested direntry name.
// If the cache is empty for the current directory, it will fill it and try to
// match the requested direnty after this operation.
// It returns ENOENT if not found.
func (d *Directory) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (fs.Node, error) {
	if _, found := directoryCache.Peek(d.c.Name, d.path); !found {
		d.ReadDirAll(ctx)
	}

	// Find matching child
	if item := directoryCache.Get(d.c.Name, d.path, req.Name); item != nil {
		if n, ok := item.(fs.Node); ok {
			return n, nil
		}
	}

	return nil, fuse.ENOENT
}

// Mkdir creates a new directory node within the current directory. It is represented
// by an empty object ending with a slash in the Swift container.
func (d *Directory) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) {
	absPath := d.path + req.Name

	// Create the file in swift
	if err := SwiftConnection.ObjectPutBytes(d.c.Name, absPath, nil, dirContentType); err != nil {
		return nil, fuse.EIO
	}

	// Directory object
	node := &Directory{
		c:    d.c,
		cs:   d.cs,
		name: req.Name,
		path: absPath + "/",
		sh:   swift.Headers{},
		so: &swift.Object{
			Name:         absPath,
			ContentType:  dirContentType,
			LastModified: time.Now(),
		},
	}

	// Cache eviction
	directoryCache.Set(d.c.Name, d.path, req.Name, node)

	return node, nil
}

// Name gets the direntry name
func (d *Directory) Name() string {
	return d.name
}

// Remove deletes a direntry and relevant node. It is not supported on container
// nodes. It handles standard and segmented object deletion.
func (d *Directory) Remove(ctx context.Context, req *fuse.RemoveRequest) error {
	var (
		path = d.path + req.Name
		node = directoryCache.Get(d.c.Name, d.path, req.Name)
	)

	if directory, ok := node.(*Directory); ok {
		return d.removeDirectory(directory, req.Name)
	}
	if object, ok := node.(*Object); ok {
		return d.removeObject(object, req.Name, path)
	}
	if symlink, ok := node.(*Symlink); ok {
		return d.removeSymlink(symlink, req.Name, path)
	}

	return fuse.ENOTSUP
}

// Setattr changes file attributes on the current object. Not supported on directories.
func (d *Directory) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error {
	return nil
}

func (d *Directory) move(oldContainer, oldPath, oldName, newContainer, newPath, newName string) error {
	// Get the old node from cache

	return fuse.ENOTSUP
}

func (d *Directory) moveObject(oldContainer, oldPath, oldName, newContainer, newPath, newName string, o *Object, manifest bool) error {
	if manifest {
		err := SwiftConnection.ObjectMove(oldContainer, oldPath+oldName, newContainer, newPath+newName)
		if err != nil {
			return err
		}
	} else {
		_, err := SwiftConnection.ManifestCopy(oldContainer, oldPath+oldName, newContainer, newPath+newName, nil)
		if err != nil {
			return err
		}
		err = SwiftConnection.ObjectDelete(oldContainer, oldPath+oldName)
		if err != nil {
			return err
		}
	}

	o.name = newName
	o.path = newPath + newName

	directoryCache.Delete(oldContainer, oldPath, oldName)
	directoryCache.Set(newContainer, newPath, newName, o)

	return nil
}

func (d *Directory) removeDirectory(directory *Directory, name string) error {
	SwiftConnection.ObjectDelete(directory.c.Name, directory.so.Name)
	if _, found := directoryCache.Peek(directory.c.Name, directory.path); found {
		directoryCache.DeleteAll(directory.c.Name, directory.path)
	}

	directoryCache.Delete(directory.c.Name, d.path, directory.name)

	return nil
}

func (d *Directory) removeObject(object *Object, name, path string) error {
	if object.segmented {
		_, h, err := SwiftConnection.Object(d.c.Name, path)
		if err != nil {
			return err
		}
		if !segmentPathRegex.Match([]byte(h[manifestHeader])) {
			return fmt.Errorf("Invalid segment path for manifest %s", name)
		}
		if err := deleteSegments(d.cs.Name, h[manifestHeader]); err != nil {
			return err
		}
	}

	SwiftConnection.ObjectDelete(d.c.Name, path)
	directoryCache.Delete(d.c.Name, d.path, name)

	return nil
}

func (d *Directory) removeSymlink(symlink *Symlink, name, path string) error {
	err := SwiftConnection.ObjectDelete(d.c.Name, path)
	if err != nil {
		return err
	}

	directoryCache.Delete(d.c.Name, d.path, name)
	return nil
}

// Rename moves a node from its current directory to a new directory and updates the cache.
func (d *Directory) Rename(ctx context.Context, req *fuse.RenameRequest, newDir fs.Node) error {
	if t, ok := newDir.(*Directory); ok && (t.c.Name == d.c.Name) {
		// Get object from cache
		oldNode := directoryCache.Get(d.c.Name, d.path, req.OldName)

		// Rename it
		if oldObject, ok := oldNode.(*Object); ok {
			return oldObject.rename(t, req.NewName)
		}

		if oldSymlink, ok := oldNode.(*Symlink); ok {
			return oldSymlink.rename(t, req.NewName)
		}
	}
	return fuse.ENOTSUP
}

// Symlink creates a new symbolic link to the specified target in the current directory.
func (d *Directory) Symlink(ctx context.Context, req *fuse.SymlinkRequest) (fs.Node, error) {
	var (
		absPath = d.path + req.NewName
		headers = map[string]string{objectSymlinkHeader: req.Target}
	)

	// Create the file in swift
	w, err := SwiftConnection.ObjectCreate(d.c.Name, absPath, false, "", linkContentType, headers)
	if err != nil {
		return nil, err
	}

	w.Close()

	link := &Symlink{
		c:    d.c,
		p:    d,
		name: req.NewName,
		path: absPath,
		sh:   headers,
		so: &swift.Object{
			ContentType: linkContentType,
			Name:        absPath,
			Bytes:       0,
		},
	}

	directoryCache.Set(d.c.Name, d.path, req.NewName, link)

	return link, nil
}

var (
	_ Node             = (*Directory)(nil)
	_ fs.Node          = (*Directory)(nil)
	_ fs.NodeCreater   = (*Directory)(nil)
	_ fs.NodeLinker    = (*Directory)(nil)
	_ fs.NodeRemover   = (*Directory)(nil)
	_ fs.NodeMkdirer   = (*Directory)(nil)
	_ fs.NodeRenamer   = (*Directory)(nil)
	_ fs.NodeSetattrer = (*Directory)(nil)
	_ fs.NodeSymlinker = (*Directory)(nil)
)
<commit_msg>Subdirectory handling for horizon fixes #72<commit_after>package svfs

import (
	"fmt"
	"os"
	"regexp"
	"strings"
	"time"

	"github.com/xlucas/swift"

	"bazil.org/fuse"
	"bazil.org/fuse/fs"
	"golang.org/x/net/context"
)

const (
	dirContentType  = "application/directory"
	linkContentType = "application/link"
)

var (
	folderRegex = regexp.MustCompile("^.+/$")
	subdirRegex = regexp.MustCompile(".*/.*$")
)

// Directory represents a standard directory entry.
type Directory struct {
	apex bool
	name string
	path string
	so   *swift.Object
	sh   swift.Headers
	c    *swift.Container
	cs   *swift.Container
}

// Attr fills file attributes of a directory within the current context.
func (d *Directory) Attr(ctx context.Context, a *fuse.Attr) error {
	a.Mode = os.ModeDir | os.FileMode(DefaultMode)
	a.Gid = uint32(DefaultGID)
	a.Uid = uint32(DefaultUID)
	a.Size = uint64(BlockSize)

	if d.so != nil {
		a.Atime = time.Now()
		a.Mtime = getMtime(d.so, d.sh)
		a.Ctime = a.Mtime
		a.Crtime = a.Mtime
	}

	return nil
}

// Create makes a new object node represented by a file. It returns
// an object node and an opened file handle.
func (d *Directory) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) {
	// Create an empty object in swift
	path := d.path + req.Name

	// New node
	node := &Object{name: req.Name, path: path, c: d.c, cs: d.cs}

	err := SwiftConnection.ObjectPutBytes(node.c.Name, node.path, nil, "")
	if err != nil {
		return nil, nil, err
	}

	// Get object handler
	fh, err := node.open(req.Flags, &resp.Flags)
	if err != nil {
		return nil, nil, err
	}

	// Get object info
	obj := &swift.Object{
		Name:         path,
		Bytes:        0,
		LastModified: time.Now(),
	}

	node.so = obj
	node.sh = map[string]string{}

	// Cache it
	directoryCache.Set(d.c.Name, d.path, req.Name, node)

	return node, fh, nil
}

// Export gives a direntry for the current directory node.
func (d *Directory) Export() fuse.Dirent {
	return fuse.Dirent{
		Name: d.name,
		Type: fuse.DT_Dir,
	}
}

// ReadDirAll reads the content of a directory and returns a
// list of children nodes as direntries, using/filling the
// cache of nodes.
func (d *Directory) ReadDirAll(ctx context.Context) (direntries []fuse.Dirent, err error) {
	var (
		dirs  = make(map[string]bool)
		tasks = make(chan Node, ListerConcurrency)
		count = 0
	)

	defer close(tasks)

	// Cache check
	if _, nodes := directoryCache.GetAll(d.c.Name, d.path); nodes != nil {
		for _, node := range nodes {
			direntries = append(direntries, node.Export())
		}
		return direntries, nil
	}

	// Fetch objects
	objects, err := SwiftConnection.ObjectsAll(d.c.Name, &swift.ObjectsOpts{
		Delimiter: '/',
		Prefix:    d.path,
	})
	if err != nil {
		return nil, err
	}

	var children = make(map[string]Node)

	// Fill cache
	for _, object := range objects {
		var (
			child    Node
			o        = object
			path     = object.Name
			fileName = strings.TrimSuffix(strings.TrimPrefix(o.Name, d.path), "/")
		)

		// This is a symlink
		if isSymlink(o, d.path) {
			child = &Symlink{path: path, name: fileName, c: d.c, so: &o, sh: swift.Headers{}, p: d}
			directoryLister.AddTask(child, tasks)
			child = nil
			count++
			goto finish
		}

		// This is a standard directory
		if isDirectory(o, d.path) {
			if !strings.HasSuffix(o.Name, "/") {
				path += "/"
			}
			child = &Directory{c: d.c, cs: d.cs, so: &o, sh: swift.Headers{}, path: path, name: fileName}
			dirs[fileName] = true
			goto finish
		}

		// This is a pseudo directory. Add it only if the real directory is missing
		if isPseudoDirectory(o, d.path) && !dirs[fileName] {
			child = &Directory{c: d.c, cs: d.cs, so: &o, sh: swift.Headers{}, path: path, name: fileName}
			dirs[fileName] = true
			goto finish
		}

		// This is a pure swift object
		if !strings.HasSuffix(o.Name, "/") {
			child = &Object{path: path, name: fileName, c: d.c, cs: d.cs, so: &o, sh: swift.Headers{}, p: d}

			// If we are writing to this object at the moment
			// we don't want to update the cache with this.
			if changeCache.Exist(d.c.Name, path) {
				child = changeCache.Get(d.c.Name, path)
				goto export
			}

			// Large objects needs extra information
			if isLargeObject(&o) {
				directoryLister.AddTask(child, tasks)
				child = nil
				count++
			}
		}

	finish:
		// Always fetch extra info if asked
		if child != nil && ExtraAttr {
			directoryLister.AddTask(child, tasks)
			child = nil
			count++
		}

	export:
		// Add nodes not requiring extra info
		if child != nil {
			direntries = append(direntries, child.Export())
			children[child.Name()] = child
		}

	}

	// Wait for directory lister to finish
	if count > 0 {
		done := 0
		for task := range tasks {
			done++
			direntries = append(direntries, task.Export())
			children[task.Name()] = task
			if done == count {
				break
			}
		}
	}

	directoryCache.AddAll(d.c.Name, d.path, d, children)

	return direntries, nil
}

// Link creates a hard link between two nodes.
func (d *Directory) Link(ctx context.Context, req *fuse.LinkRequest, old fs.Node) (node fs.Node, err error) {
	if object, ok := old.(*Object); ok {
		return object.copy(d, req.NewName)
	}
	if symlink, ok := old.(*Symlink); ok {
		return symlink.copy(d, req.NewName)
	}
	return nil, fuse.ENOTSUP
}

// Lookup gets a children node if its name matches the requested direntry name.
// If the cache is empty for the current directory, it will fill it and try to
// match the requested direnty after this operation.
// It returns ENOENT if not found.
func (d *Directory) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (fs.Node, error) {
	if _, found := directoryCache.Peek(d.c.Name, d.path); !found {
		d.ReadDirAll(ctx)
	}

	// Find matching child
	if item := directoryCache.Get(d.c.Name, d.path, req.Name); item != nil {
		if n, ok := item.(fs.Node); ok {
			return n, nil
		}
	}

	return nil, fuse.ENOENT
}

// Mkdir creates a new directory node within the current directory. It is represented
// by an empty object ending with a slash in the Swift container.
func (d *Directory) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) {
	absPath := d.path + req.Name + "/"

	// Create the file in swift
	if err := SwiftConnection.ObjectPutBytes(d.c.Name, absPath, nil, dirContentType); err != nil {
		return nil, fuse.EIO
	}

	// Directory object
	node := &Directory{
		c:    d.c,
		cs:   d.cs,
		name: req.Name,
		path: absPath,
		sh:   swift.Headers{},
		so: &swift.Object{
			Name:         absPath,
			ContentType:  dirContentType,
			LastModified: time.Now(),
		},
	}

	// Cache eviction
	directoryCache.Set(d.c.Name, d.path, req.Name, node)

	return node, nil
}

// Name gets the direntry name
func (d *Directory) Name() string {
	return d.name
}

// Remove deletes a direntry and relevant node. It is not supported on container
// nodes. It handles standard and segmented object deletion.
func (d *Directory) Remove(ctx context.Context, req *fuse.RemoveRequest) error {
	var (
		path = d.path + req.Name
		node = directoryCache.Get(d.c.Name, d.path, req.Name)
	)

	if directory, ok := node.(*Directory); ok {
		return d.removeDirectory(directory, req.Name)
	}
	if object, ok := node.(*Object); ok {
		return d.removeObject(object, req.Name, path)
	}
	if symlink, ok := node.(*Symlink); ok {
		return d.removeSymlink(symlink, req.Name, path)
	}

	return fuse.ENOTSUP
}

// Setattr changes file attributes on the current object. Not supported on directories.
func (d *Directory) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error {
	return nil
}

func (d *Directory) move(oldContainer, oldPath, oldName, newContainer, newPath, newName string) error {
	// Get the old node from cache

	return fuse.ENOTSUP
}

func (d *Directory) moveObject(oldContainer, oldPath, oldName, newContainer, newPath, newName string, o *Object, manifest bool) error {
	if manifest {
		err := SwiftConnection.ObjectMove(oldContainer, oldPath+oldName, newContainer, newPath+newName)
		if err != nil {
			return err
		}
	} else {
		_, err := SwiftConnection.ManifestCopy(oldContainer, oldPath+oldName, newContainer, newPath+newName, nil)
		if err != nil {
			return err
		}
		err = SwiftConnection.ObjectDelete(oldContainer, oldPath+oldName)
		if err != nil {
			return err
		}
	}

	o.name = newName
	o.path = newPath + newName

	directoryCache.Delete(oldContainer, oldPath, oldName)
	directoryCache.Set(newContainer, newPath, newName, o)

	return nil
}

func (d *Directory) removeDirectory(directory *Directory, name string) error {
	SwiftConnection.ObjectDelete(directory.c.Name, directory.so.Name)
	if _, found := directoryCache.Peek(directory.c.Name, directory.path); found {
		directoryCache.DeleteAll(directory.c.Name, directory.path)
	}

	directoryCache.Delete(directory.c.Name, d.path, directory.name)

	return nil
}

func (d *Directory) removeObject(object *Object, name, path string) error {
	if object.segmented {
		_, h, err := SwiftConnection.Object(d.c.Name, path)
		if err != nil {
			return err
		}
		if !segmentPathRegex.Match([]byte(h[manifestHeader])) {
			return fmt.Errorf("Invalid segment path for manifest %s", name)
		}
		if err := deleteSegments(d.cs.Name, h[manifestHeader]); err != nil {
			return err
		}
	}

	SwiftConnection.ObjectDelete(d.c.Name, path)
	directoryCache.Delete(d.c.Name, d.path, name)

	return nil
}

func (d *Directory) removeSymlink(symlink *Symlink, name, path string) error {
	err := SwiftConnection.ObjectDelete(d.c.Name, path)
	if err != nil {
		return err
	}

	directoryCache.Delete(d.c.Name, d.path, name)
	return nil
}

// Rename moves a node from its current directory to a new directory and updates the cache.
func (d *Directory) Rename(ctx context.Context, req *fuse.RenameRequest, newDir fs.Node) error {
	if t, ok := newDir.(*Directory); ok && (t.c.Name == d.c.Name) {
		// Get object from cache
		oldNode := directoryCache.Get(d.c.Name, d.path, req.OldName)

		// Rename it
		if oldObject, ok := oldNode.(*Object); ok {
			return oldObject.rename(t, req.NewName)
		}

		if oldSymlink, ok := oldNode.(*Symlink); ok {
			return oldSymlink.rename(t, req.NewName)
		}
	}
	return fuse.ENOTSUP
}

// Symlink creates a new symbolic link to the specified target in the current directory.
func (d *Directory) Symlink(ctx context.Context, req *fuse.SymlinkRequest) (fs.Node, error) {
	var (
		absPath = d.path + req.NewName
		headers = map[string]string{objectSymlinkHeader: req.Target}
	)

	// Create the file in swift
	w, err := SwiftConnection.ObjectCreate(d.c.Name, absPath, false, "", linkContentType, headers)
	if err != nil {
		return nil, err
	}

	w.Close()

	link := &Symlink{
		c:    d.c,
		p:    d,
		name: req.NewName,
		path: absPath,
		sh:   headers,
		so: &swift.Object{
			ContentType: linkContentType,
			Name:        absPath,
			Bytes:       0,
		},
	}

	directoryCache.Set(d.c.Name, d.path, req.NewName, link)

	return link, nil
}

var (
	_ Node             = (*Directory)(nil)
	_ fs.Node          = (*Directory)(nil)
	_ fs.NodeCreater   = (*Directory)(nil)
	_ fs.NodeLinker    = (*Directory)(nil)
	_ fs.NodeRemover   = (*Directory)(nil)
	_ fs.NodeMkdirer   = (*Directory)(nil)
	_ fs.NodeRenamer   = (*Directory)(nil)
	_ fs.NodeSetattrer = (*Directory)(nil)
	_ fs.NodeSymlinker = (*Directory)(nil)
)
<|endoftext|>


<commit_before>package metadata

import "github.com/xeipuuv/gojsonschema"

func NewTable(name string) *Table {
	return &Table{
		Name:    name,
		Indexes: make(map[string]*TableIndex),
	}
}

type Table struct {
	Name   string  `json:"name"`
	Schema *Schema `json:"schema"`

	// TODO: maintain another map of each column -> index? (so we can attempt to
	// re-work queries to align with indexes)
	// map of name -> index
	Indexes map[string]*TableIndex `json:"indexes,omitempty"`

	// So we know what the primary is, which will be used for .Get()
	PrimaryColumn string      `json:"primarykey,omitempty"`
	PrimaryIndex  *TableIndex `json:"-"`
}

func (t *Table) ListIndexes() []string {
	indexes := make([]string, 0, len(t.Indexes))
	for name, _ := range t.Indexes {
		indexes = append(indexes, name)
	}
	return indexes
}

type Schema struct {
	Name    string                 `json:"name"`
	Version int64                  `json:"version"`
	Schema  map[string]interface{} `json:"schema"`
	Gschema *gojsonschema.Schema   `json:"-"`
}

// TODO: add flags for other things (like uniqueness, etc.)
type TableIndex struct {
	Name string `json:"name"`
	// TODO: better schema-- this will be the data_json in the DB
	Columns []string `json:"columns"`
}
<commit_msg>Remove unused fields<commit_after>package metadata

import "github.com/xeipuuv/gojsonschema"

func NewTable(name string) *Table {
	return &Table{
		Name:    name,
		Indexes: make(map[string]*TableIndex),
	}
}

type Table struct {
	Name   string  `json:"name"`
	Schema *Schema `json:"schema"`

	// TODO: maintain another map of each column -> index? (so we can attempt to
	// re-work queries to align with indexes)
	// map of name -> index
	Indexes map[string]*TableIndex `json:"indexes,omitempty"`
}

func (t *Table) ListIndexes() []string {
	indexes := make([]string, 0, len(t.Indexes))
	for name, _ := range t.Indexes {
		indexes = append(indexes, name)
	}
	return indexes
}

type Schema struct {
	Name    string                 `json:"name"`
	Version int64                  `json:"version"`
	Schema  map[string]interface{} `json:"schema"`
	Gschema *gojsonschema.Schema   `json:"-"`
}

// TODO: add flags for other things (like uniqueness, etc.)
type TableIndex struct {
	Name string `json:"name"`
	// TODO: better schema-- this will be the data_json in the DB
	Columns []string `json:"columns"`
}
<|endoftext|>


<commit_before>package svnwatch

import (
	"encoding/xml"

	"github.com/pkg/errors"
)

type Watches struct {
	XMLName xml.Name `xml:"watches"`
	Watches []Watch  `xml:"watch"`
}

type Watch struct {
	XMLName  xml.Name  `xml:"watch"`
	URL      string    `xml:"url,attr"`
	Commands []Command `xml:"command"`
}

func (w Watch) Update(repositories *Repositories) error {
	repo := repositories.ForURL(w.URL)

	updated, err := repo.Update()

	if err != nil {
		return errors.Wrapf(err, "failed to update watch for %s", w.URL)
	}

	if updated {
		for _, cmd := range w.Commands {
			if err := cmd.Execute(*repo); err != nil {
				return errors.Wrapf(err, "failed to execute command")
			}
		}
	}

	return nil
}

func (w *Watch) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
	watch := struct {
		URL *string `xml:"url,attr"`
	}{}

	if err := d.DecodeElement(&watch, &start); err != nil {
		return err
	}

	if watch.URL == nil {
		return errors.New("missing URL from watch")
	}

	return nil
}
<commit_msg>Store unmarshaled URL in watch<commit_after>package svnwatch

import (
	"encoding/xml"

	"github.com/pkg/errors"
)

type Watches struct {
	XMLName xml.Name `xml:"watches"`
	Watches []Watch  `xml:"watch"`
}

type Watch struct {
	XMLName  xml.Name  `xml:"watch"`
	URL      string    `xml:"url,attr"`
	Commands []Command `xml:"command"`
}

func (w Watch) Update(repositories *Repositories) error {
	repo := repositories.ForURL(w.URL)

	updated, err := repo.Update()

	if err != nil {
		return errors.Wrapf(err, "failed to update watch for %s", w.URL)
	}

	if updated {
		for _, cmd := range w.Commands {
			if err := cmd.Execute(*repo); err != nil {
				return errors.Wrapf(err, "failed to execute command")
			}
		}
	}

	return nil
}

func (w *Watch) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
	watch := struct {
		URL *string `xml:"url,attr"`
	}{}

	if err := d.DecodeElement(&watch, &start); err != nil {
		return err
	}

	if watch.URL == nil {
		return errors.New("missing URL from watch")
	}

	w.URL = *watch.URL

	return nil
}
<|endoftext|>


<commit_before>// Package buildtsi reads an in-memory index and exports it as a TSI index.
package buildtsi

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/user"
	"path/filepath"
	"runtime"
	"runtime/pprof"
	"strconv"
	"strings"

	"github.com/influxdata/influxdb/logger"
	"github.com/influxdata/influxdb/models"
	"github.com/influxdata/influxdb/tsdb"
	"github.com/influxdata/influxdb/tsdb/engine/tsm1"
	"github.com/influxdata/influxdb/tsdb/index/tsi1"
	"go.uber.org/zap"
)

// Command represents the program execution for "influx_inspect inmem2tsi".
type Command struct {
	Stderr  io.Writer
	Stdout  io.Writer
	Verbose bool
	Logger  *zap.Logger

	databaseFilter  string
	retentionFilter string
	shardFilter     string
	maxLogFileSize  int64
}

// NewCommand returns a new instance of Command.
func NewCommand() *Command {
	return &Command{
		Stderr: os.Stderr,
		Stdout: os.Stdout,
		Logger: zap.NewNop(),
	}
}

// Run executes the command.
func (cmd *Command) Run(args ...string) error {
	fs := flag.NewFlagSet("inmem2tsi", flag.ExitOnError)
	dataDir := fs.String("datadir", "", "data directory")
	walDir := fs.String("waldir", "", "WAL directory")
	fs.StringVar(&cmd.databaseFilter, "database", "", "optional: database name")
	fs.StringVar(&cmd.retentionFilter, "retention", "", "optional: retention policy")
	fs.StringVar(&cmd.shardFilter, "shard", "", "optional: shard id")
	fs.Int64Var(&cmd.maxLogFileSize, "max-log-file-size", tsdb.DefaultMaxIndexLogFileSize, "optional: maximum log file size")
	fs.BoolVar(&cmd.Verbose, "v", false, "verbose")
	fs.SetOutput(cmd.Stdout)
	if err := fs.Parse(args); err != nil {
		return err
	} else if fs.NArg() > 0 || *dataDir == "" || *walDir == "" {
		fs.Usage()
		return nil
	}
	cmd.Logger = logger.New(cmd.Stderr)

	finish := startProfiles()
	defer finish()
	return cmd.run(*dataDir, *walDir)
}

func startProfiles() func() {
	runtime.MemProfileRate = 100 // Sample 1% of allocations.

	paths := []string{"/tmp/buildtsi.mem.pprof", "/tmp/buildtsi.cpu.pprof"}
	var files []*os.File
	for _, pth := range paths {
		f, err := os.Create(pth)
		if err != nil {
			log.Fatalf("memprofile: %v", err)
		}
		log.Printf("writing profile to: %s\n", pth)
		files = append(files, f)

	}

	closeFn := func() {
		// Write the memory profile
		if err := pprof.Lookup("heap").WriteTo(files[0], 0); err != nil {
			panic(err)
		}

		// Stop the CPU profile.
		pprof.StopCPUProfile()

		for _, fd := range files {
			if err := fd.Close(); err != nil {
				panic(err)
			}
		}
	}

	if err := pprof.StartCPUProfile(files[1]); err != nil {
		panic(err)
	}
	return closeFn
}

func (cmd *Command) run(dataDir, walDir string) error {
	// Verify the user actually wants to run as root.
	if isRoot() {
		fmt.Println("You are currently running as root. This will build your")
		fmt.Println("index files with root ownership and will be inaccessible")
		fmt.Println("if you run influxd as a non-root user. You should run")
		fmt.Println("buildtsi as the same user you are running influxd.")
		fmt.Print("Are you sure you want to continue? (y/N): ")
		var answer string
		if fmt.Scanln(&answer); !strings.HasPrefix(strings.TrimSpace(strings.ToLower(answer)), "y") {
			return fmt.Errorf("Operation aborted.")
		}
	}

	fis, err := ioutil.ReadDir(dataDir)
	if err != nil {
		return err
	}

	for _, fi := range fis {
		name := fi.Name()
		if !fi.IsDir() {
			continue
		} else if cmd.databaseFilter != "" && name != cmd.databaseFilter {
			continue
		}

		if err := cmd.processDatabase(name, filepath.Join(dataDir, name), filepath.Join(walDir, name)); err != nil {
			return err
		}
	}

	return nil
}

func (cmd *Command) processDatabase(dbName, dataDir, walDir string) error {
	cmd.Logger.Info("rebuilding database", zap.String("name", dbName))

	sfile := tsdb.NewSeriesFile(filepath.Join(dataDir, tsdb.SeriesFileDirectory))
	sfile.Logger = cmd.Logger
	if err := sfile.Open(); err != nil {
		return err
	}
	defer sfile.Close()

	fis, err := ioutil.ReadDir(dataDir)
	if err != nil {
		return err
	}

	for _, fi := range fis {
		rpName := fi.Name()
		if !fi.IsDir() {
			continue
		} else if rpName == tsdb.SeriesFileDirectory {
			continue
		} else if cmd.retentionFilter != "" && rpName != cmd.retentionFilter {
			continue
		}

		if err := cmd.processRetentionPolicy(sfile, dbName, rpName, filepath.Join(dataDir, rpName), filepath.Join(walDir, rpName)); err != nil {
			return err
		}
	}

	return nil
}

func (cmd *Command) processRetentionPolicy(sfile *tsdb.SeriesFile, dbName, rpName, dataDir, walDir string) error {
	cmd.Logger.Info("rebuilding retention policy", logger.Database(dbName), logger.RetentionPolicy(rpName))

	fis, err := ioutil.ReadDir(dataDir)
	if err != nil {
		return err
	}

	for _, fi := range fis {
		if !fi.IsDir() {
			continue
		} else if cmd.shardFilter != "" && fi.Name() != cmd.shardFilter {
			continue
		}

		shardID, err := strconv.ParseUint(fi.Name(), 10, 64)
		if err != nil {
			continue
		}

		if err := cmd.processShard(sfile, dbName, rpName, shardID, filepath.Join(dataDir, fi.Name()), filepath.Join(walDir, fi.Name())); err != nil {
			return err
		}
	}
	return nil
}

func (cmd *Command) processShard(sfile *tsdb.SeriesFile, dbName, rpName string, shardID uint64, dataDir, walDir string) error {
	cmd.Logger.Info("rebuilding shard", logger.Database(dbName), logger.RetentionPolicy(rpName), logger.Shard(shardID))

	// Check if shard already has a TSI index.
	indexPath := filepath.Join(dataDir, "index")
	cmd.Logger.Info("checking index path", zap.String("path", indexPath))
	if _, err := os.Stat(indexPath); !os.IsNotExist(err) {
		cmd.Logger.Info("tsi1 index already exists, skipping", zap.String("path", indexPath))
		return nil
	}

	cmd.Logger.Info("opening shard")

	// Find shard files.
	tsmPaths, err := cmd.collectTSMFiles(dataDir)
	if err != nil {
		return err
	}
	walPaths, err := cmd.collectWALFiles(walDir)
	if err != nil {
		return err
	}

	// Remove temporary index files if this is being re-run.
	tmpPath := filepath.Join(dataDir, ".index")
	cmd.Logger.Info("cleaning up partial index from previous run, if any")
	if err := os.RemoveAll(tmpPath); err != nil {
		return err
	}

	// Open TSI index in temporary path.
	tsiIndex := tsi1.NewIndex(sfile, dbName, tsi1.WithPath(tmpPath), tsi1.WithMaximumLogFileSize(cmd.maxLogFileSize))
	tsiIndex.WithLogger(cmd.Logger)
	cmd.Logger.Info("opening tsi index in temporary location", zap.String("path", tmpPath))
	if err := tsiIndex.Open(); err != nil {
		return err
	}
	defer tsiIndex.Close()

	// Write out tsm1 files.
	cmd.Logger.Info("iterating over tsm files")
	for _, path := range tsmPaths {
		cmd.Logger.Info("processing tsm file", zap.String("path", path))
		if err := cmd.processTSMFile(tsiIndex, path); err != nil {
			return err
		}
	}

	// Write out wal files.
	cmd.Logger.Info("building cache from wal files")
	cache := tsm1.NewCache(tsdb.DefaultCacheMaxMemorySize)
	loader := tsm1.NewCacheLoader(walPaths)
	loader.WithLogger(cmd.Logger)
	if err := loader.Load(cache); err != nil {
		return err
	}

	cmd.Logger.Info("iterating over cache")
	for _, key := range cache.Keys() {
		seriesKey, _ := tsm1.SeriesAndFieldFromCompositeKey(key)
		name, tags := models.ParseKey(seriesKey)

		if cmd.Verbose {
			cmd.Logger.Info("series", zap.String("name", string(name)), zap.String("tags", tags.String()))
		}

		if err := tsiIndex.CreateSeriesIfNotExists(seriesKey, []byte(name), tags); err != nil {
			return fmt.Errorf("cannot create series: %s %s (%s)", name, tags.String(), err)
		}
	}

	// Attempt to compact the index & wait for all compactions to complete.
	cmd.Logger.Info("compacting index")
	tsiIndex.Compact()
	tsiIndex.Wait()

	// Close TSI index.
	cmd.Logger.Info("closing tsi index")
	if err := tsiIndex.Close(); err != nil {
		return err
	}

	// Rename TSI to standard path.
	cmd.Logger.Info("moving tsi to permanent location")
	return os.Rename(tmpPath, indexPath)
}

func (cmd *Command) processTSMFile(index *tsi1.Index, path string) error {
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()

	r, err := tsm1.NewTSMReader(f)
	if err != nil {
		cmd.Logger.Warn("unable to read, skipping", zap.String("path", path), zap.Error(err))
		return nil
	}
	defer r.Close()

	for i := 0; i < r.KeyCount(); i++ {
		key, _ := r.KeyAt(i)
		seriesKey, _ := tsm1.SeriesAndFieldFromCompositeKey(key)
		name, tags := models.ParseKey(seriesKey)

		if cmd.Verbose {
			cmd.Logger.Info("series", zap.String("name", string(name)), zap.String("tags", tags.String()))
		}

		if err := index.CreateSeriesIfNotExists(seriesKey, []byte(name), tags); err != nil {
			return fmt.Errorf("cannot create series: %s %s (%s)", name, tags.String(), err)
		}
	}
	return nil
}

func (cmd *Command) collectTSMFiles(path string) ([]string, error) {
	fis, err := ioutil.ReadDir(path)
	if err != nil {
		return nil, err
	}

	var paths []string
	for _, fi := range fis {
		if filepath.Ext(fi.Name()) != "."+tsm1.TSMFileExtension {
			continue
		}
		paths = append(paths, filepath.Join(path, fi.Name()))
	}
	return paths, nil
}

func (cmd *Command) collectWALFiles(path string) ([]string, error) {
	fis, err := ioutil.ReadDir(path)
	if err != nil {
		return nil, err
	}

	var paths []string
	for _, fi := range fis {
		if filepath.Ext(fi.Name()) != "."+tsm1.WALFileExtension {
			continue
		}
		paths = append(paths, filepath.Join(path, fi.Name()))
	}
	return paths, nil
}

func isRoot() bool {
	user, _ := user.Current()
	return user != nil && user.Username == "root"
}
<commit_msg>Logging format<commit_after>// Package buildtsi reads an in-memory index and exports it as a TSI index.
package buildtsi

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/user"
	"path/filepath"
	"runtime"
	"runtime/pprof"
	"strconv"
	"strings"

	"github.com/influxdata/influxdb/logger"
	"github.com/influxdata/influxdb/models"
	"github.com/influxdata/influxdb/tsdb"
	"github.com/influxdata/influxdb/tsdb/engine/tsm1"
	"github.com/influxdata/influxdb/tsdb/index/tsi1"
	"go.uber.org/zap"
)

// Command represents the program execution for "influx_inspect buildtsi".
type Command struct {
	Stderr  io.Writer
	Stdout  io.Writer
	Verbose bool
	Logger  *zap.Logger

	databaseFilter  string
	retentionFilter string
	shardFilter     string
	maxLogFileSize  int64
}

// NewCommand returns a new instance of Command.
func NewCommand() *Command {
	return &Command{
		Stderr: os.Stderr,
		Stdout: os.Stdout,
		Logger: zap.NewNop(),
	}
}

// Run executes the command.
func (cmd *Command) Run(args ...string) error {
	fs := flag.NewFlagSet("buildtsi", flag.ExitOnError)
	dataDir := fs.String("datadir", "", "data directory")
	walDir := fs.String("waldir", "", "WAL directory")
	fs.StringVar(&cmd.databaseFilter, "database", "", "optional: database name")
	fs.StringVar(&cmd.retentionFilter, "retention", "", "optional: retention policy")
	fs.StringVar(&cmd.shardFilter, "shard", "", "optional: shard id")
	fs.Int64Var(&cmd.maxLogFileSize, "max-log-file-size", tsdb.DefaultMaxIndexLogFileSize, "optional: maximum log file size")
	fs.BoolVar(&cmd.Verbose, "v", false, "verbose")
	fs.SetOutput(cmd.Stdout)
	if err := fs.Parse(args); err != nil {
		return err
	} else if fs.NArg() > 0 || *dataDir == "" || *walDir == "" {
		fs.Usage()
		return nil
	}
	cmd.Logger = logger.New(cmd.Stderr)

	finish := startProfiles()
	defer finish()
	return cmd.run(*dataDir, *walDir)
}

func startProfiles() func() {
	runtime.MemProfileRate = 100 // Sample 1% of allocations.

	paths := []string{"/tmp/buildtsi.mem.pprof", "/tmp/buildtsi.cpu.pprof"}
	var files []*os.File
	for _, pth := range paths {
		f, err := os.Create(pth)
		if err != nil {
			log.Fatalf("memprofile: %v", err)
		}
		log.Printf("writing profile to: %s\n", pth)
		files = append(files, f)

	}

	closeFn := func() {
		// Write the memory profile
		if err := pprof.Lookup("heap").WriteTo(files[0], 0); err != nil {
			panic(err)
		}

		// Stop the CPU profile.
		pprof.StopCPUProfile()

		for _, fd := range files {
			if err := fd.Close(); err != nil {
				panic(err)
			}
		}
	}

	if err := pprof.StartCPUProfile(files[1]); err != nil {
		panic(err)
	}
	return closeFn
}

func (cmd *Command) run(dataDir, walDir string) error {
	// Verify the user actually wants to run as root.
	if isRoot() {
		fmt.Println("You are currently running as root. This will build your")
		fmt.Println("index files with root ownership and will be inaccessible")
		fmt.Println("if you run influxd as a non-root user. You should run")
		fmt.Println("buildtsi as the same user you are running influxd.")
		fmt.Print("Are you sure you want to continue? (y/N): ")
		var answer string
		if fmt.Scanln(&answer); !strings.HasPrefix(strings.TrimSpace(strings.ToLower(answer)), "y") {
			return fmt.Errorf("operation aborted")
		}
	}

	fis, err := ioutil.ReadDir(dataDir)
	if err != nil {
		return err
	}

	for _, fi := range fis {
		name := fi.Name()
		if !fi.IsDir() {
			continue
		} else if cmd.databaseFilter != "" && name != cmd.databaseFilter {
			continue
		}

		if err := cmd.processDatabase(name, filepath.Join(dataDir, name), filepath.Join(walDir, name)); err != nil {
			return err
		}
	}

	return nil
}

func (cmd *Command) processDatabase(dbName, dataDir, walDir string) error {
	cmd.Logger.Info("Rebuilding database", zap.String("name", dbName))

	sfile := tsdb.NewSeriesFile(filepath.Join(dataDir, tsdb.SeriesFileDirectory))
	sfile.Logger = cmd.Logger
	if err := sfile.Open(); err != nil {
		return err
	}
	defer sfile.Close()

	fis, err := ioutil.ReadDir(dataDir)
	if err != nil {
		return err
	}

	for _, fi := range fis {
		rpName := fi.Name()
		if !fi.IsDir() {
			continue
		} else if rpName == tsdb.SeriesFileDirectory {
			continue
		} else if cmd.retentionFilter != "" && rpName != cmd.retentionFilter {
			continue
		}

		if err := cmd.processRetentionPolicy(sfile, dbName, rpName, filepath.Join(dataDir, rpName), filepath.Join(walDir, rpName)); err != nil {
			return err
		}
	}

	return nil
}

func (cmd *Command) processRetentionPolicy(sfile *tsdb.SeriesFile, dbName, rpName, dataDir, walDir string) error {
	cmd.Logger.Info("Rebuilding retention policy", logger.Database(dbName), logger.RetentionPolicy(rpName))

	fis, err := ioutil.ReadDir(dataDir)
	if err != nil {
		return err
	}

	for _, fi := range fis {
		if !fi.IsDir() {
			continue
		} else if cmd.shardFilter != "" && fi.Name() != cmd.shardFilter {
			continue
		}

		shardID, err := strconv.ParseUint(fi.Name(), 10, 64)
		if err != nil {
			continue
		}

		if err := cmd.processShard(sfile, dbName, rpName, shardID, filepath.Join(dataDir, fi.Name()), filepath.Join(walDir, fi.Name())); err != nil {
			return err
		}
	}
	return nil
}

func (cmd *Command) processShard(sfile *tsdb.SeriesFile, dbName, rpName string, shardID uint64, dataDir, walDir string) error {
	cmd.Logger.Info("Rebuilding shard", logger.Database(dbName), logger.RetentionPolicy(rpName), logger.Shard(shardID))

	// Check if shard already has a TSI index.
	indexPath := filepath.Join(dataDir, "index")
	cmd.Logger.Info("Checking index path", zap.String("path", indexPath))
	if _, err := os.Stat(indexPath); !os.IsNotExist(err) {
		cmd.Logger.Info("tsi1 index already exists, skipping", zap.String("path", indexPath))
		return nil
	}

	cmd.Logger.Info("Opening shard")

	// Find shard files.
	tsmPaths, err := cmd.collectTSMFiles(dataDir)
	if err != nil {
		return err
	}
	walPaths, err := cmd.collectWALFiles(walDir)
	if err != nil {
		return err
	}

	// Remove temporary index files if this is being re-run.
	tmpPath := filepath.Join(dataDir, ".index")
	cmd.Logger.Info("Cleaning up partial index from previous run, if any")
	if err := os.RemoveAll(tmpPath); err != nil {
		return err
	}

	// Open TSI index in temporary path.
	tsiIndex := tsi1.NewIndex(sfile, dbName, tsi1.WithPath(tmpPath), tsi1.WithMaximumLogFileSize(cmd.maxLogFileSize))
	tsiIndex.WithLogger(cmd.Logger)
	cmd.Logger.Info("Opening tsi index in temporary location", zap.String("path", tmpPath))
	if err := tsiIndex.Open(); err != nil {
		return err
	}
	defer tsiIndex.Close()

	// Write out tsm1 files.
	cmd.Logger.Info("Iterating over tsm files")
	for _, path := range tsmPaths {
		cmd.Logger.Info("Processing tsm file", zap.String("path", path))
		if err := cmd.processTSMFile(tsiIndex, path); err != nil {
			return err
		}
	}

	// Write out wal files.
	cmd.Logger.Info("Building cache from wal files")
	cache := tsm1.NewCache(tsdb.DefaultCacheMaxMemorySize)
	loader := tsm1.NewCacheLoader(walPaths)
	loader.WithLogger(cmd.Logger)
	if err := loader.Load(cache); err != nil {
		return err
	}

	cmd.Logger.Info("Iterating over cache")
	for _, key := range cache.Keys() {
		seriesKey, _ := tsm1.SeriesAndFieldFromCompositeKey(key)
		name, tags := models.ParseKey(seriesKey)

		if cmd.Verbose {
			cmd.Logger.Info("Series", zap.String("name", string(name)), zap.String("tags", tags.String()))
		}

		if err := tsiIndex.CreateSeriesIfNotExists(seriesKey, []byte(name), tags); err != nil {
			return fmt.Errorf("cannot create series: %s %s (%s)", name, tags.String(), err)
		}
	}

	// Attempt to compact the index & wait for all compactions to complete.
	cmd.Logger.Info("compacting index")
	tsiIndex.Compact()
	tsiIndex.Wait()

	// Close TSI index.
	cmd.Logger.Info("Closing tsi index")
	if err := tsiIndex.Close(); err != nil {
		return err
	}

	// Rename TSI to standard path.
	cmd.Logger.Info("Moving tsi to permanent location")
	return os.Rename(tmpPath, indexPath)
}

func (cmd *Command) processTSMFile(index *tsi1.Index, path string) error {
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()

	r, err := tsm1.NewTSMReader(f)
	if err != nil {
		cmd.Logger.Warn("Unable to read, skipping", zap.String("path", path), zap.Error(err))
		return nil
	}
	defer r.Close()

	for i := 0; i < r.KeyCount(); i++ {
		key, _ := r.KeyAt(i)
		seriesKey, _ := tsm1.SeriesAndFieldFromCompositeKey(key)
		name, tags := models.ParseKey(seriesKey)

		if cmd.Verbose {
			cmd.Logger.Info("Series", zap.String("name", string(name)), zap.String("tags", tags.String()))
		}

		if err := index.CreateSeriesIfNotExists(seriesKey, []byte(name), tags); err != nil {
			return fmt.Errorf("cannot create series: %s %s (%s)", name, tags.String(), err)
		}
	}
	return nil
}

func (cmd *Command) collectTSMFiles(path string) ([]string, error) {
	fis, err := ioutil.ReadDir(path)
	if err != nil {
		return nil, err
	}

	var paths []string
	for _, fi := range fis {
		if filepath.Ext(fi.Name()) != "."+tsm1.TSMFileExtension {
			continue
		}
		paths = append(paths, filepath.Join(path, fi.Name()))
	}
	return paths, nil
}

func (cmd *Command) collectWALFiles(path string) ([]string, error) {
	fis, err := ioutil.ReadDir(path)
	if err != nil {
		return nil, err
	}

	var paths []string
	for _, fi := range fis {
		if filepath.Ext(fi.Name()) != "."+tsm1.WALFileExtension {
			continue
		}
		paths = append(paths, filepath.Join(path, fi.Name()))
	}
	return paths, nil
}

func isRoot() bool {
	user, _ := user.Current()
	return user != nil && user.Username == "root"
}
<|endoftext|>


<commit_before>package collectors

import (
	"bufio"
	"os"
	"strings"
	"sync"
	"time"
	"unicode"
	"unicode/utf8"

	"github.com/StackExchange/scollector/metadata"
	"github.com/StackExchange/scollector/opentsdb"
	"github.com/StackExchange/scollector/util"
	"github.com/StackExchange/slog"
)

var collectors []Collector

type Collector interface {
	Run(chan<- *opentsdb.DataPoint)
	Name() string
	Init()
}

const (
	osCPU          = "os.cpu"
	osDiskFree     = "os.disk.fs.space_free"
	osDiskPctFree  = "os.disk.fs.percent_free"
	osDiskTotal    = "os.disk.fs.space_total"
	osDiskUsed     = "os.disk.fs.space_used"
	osMemFree      = "os.mem.free"
	osMemPctFree   = "os.mem.percent_free"
	osMemTotal     = "os.mem.total"
	osMemUsed      = "os.mem.used"
	osNetBroadcast = "os.net.packets_broadcast"
	osNetBytes     = "os.net.bytes"
	osNetDropped   = "os.net.dropped"
	osNetErrors    = "os.net.errs"
	osNetPackets   = "os.net.packets"
	osNetUnicast   = "os.net.packets_unicast"
	osNetMulticast = "os.net.packets_multicast"
)

var (
	// DefaultFreq is the duration between collection intervals if none is
	// specified.
	DefaultFreq = time.Second * 15

	timestamp = time.Now().Unix()
	tlock     sync.Mutex
)

func init() {
	go func() {
		for t := range time.Tick(time.Second) {
			tlock.Lock()
			timestamp = t.Unix()
			tlock.Unlock()
		}
	}()
}

func now() (t int64) {
	tlock.Lock()
	t = timestamp
	tlock.Unlock()
	return
}

// Search returns all collectors matching the pattern s.
func Search(s string) []Collector {
	var r []Collector
	for _, c := range collectors {
		if strings.Contains(c.Name(), s) {
			r = append(r, c)
		}
	}
	return r
}

// Runs specified collectors. Use nil for all collectors.
func Run(cs []Collector) chan *opentsdb.DataPoint {
	if cs == nil {
		cs = collectors
	}
	ch := make(chan *opentsdb.DataPoint)
	for _, c := range cs {
		go c.Run(ch)
	}
	return ch
}

// AddTS is the same as Add but lets you specify the timestamp
func AddTS(md *opentsdb.MultiDataPoint, name string, ts int64, value interface{}, t opentsdb.TagSet, rate metadata.RateType, unit metadata.Unit, desc string) {
	tags := make(opentsdb.TagSet)
	for k, v := range t {
		tags[k] = v
	}
	if host, present := tags["host"]; !present {
		tags["host"] = util.Hostname
	} else if host == "" {
		delete(tags, "host")
	}
	d := opentsdb.DataPoint{
		Metric:    name,
		Timestamp: ts,
		Value:     value,
		Tags:      tags,
	}
	*md = append(*md, &d)
	if rate != metadata.Unknown {
		metadata.AddMeta(name, nil, "rate", rate, false)
	}
	if unit != metadata.None {
		metadata.AddMeta(name, nil, "unit", unit, false)
	}
	if desc != "" {
		metadata.AddMeta(name, tags, "desc", desc, false)
	}
}

// Add appends a new data point with given metric name, value, and tags. Tags
// may be nil. If tags is nil or does not contain a host key, it will be
// automatically added. If the value of the host key is the empty string, it
// will be removed (use this to prevent the normal auto-adding of the host tag).
func Add(md *opentsdb.MultiDataPoint, name string, value interface{}, t opentsdb.TagSet, rate metadata.RateType, unit metadata.Unit, desc string) {
	AddTS(md, name, now(), value, t, rate, unit, desc)
}

func readLine(fname string, line func(string) error) error {
	f, err := os.Open(fname)
	if err != nil {
		return err
	}
	defer f.Close()
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		if err := line(scanner.Text()); err != nil {
			return err
		}
	}
	if err := scanner.Err(); err != nil {
		slog.Infof("%v: %v\n", fname, err)
	}
	return nil
}

// IsDigit returns true if s consists of decimal digits.
func IsDigit(s string) bool {
	r := strings.NewReader(s)
	for {
		ch, _, err := r.ReadRune()
		if ch == 0 || err != nil {
			break
		} else if ch == utf8.RuneError {
			return false
		} else if !unicode.IsDigit(ch) {
			return false
		}
	}
	return true
}

// IsAlNum returns true if s is alphanumeric.
func IsAlNum(s string) bool {
	r := strings.NewReader(s)
	for {
		ch, _, err := r.ReadRune()
		if ch == 0 || err != nil {
			break
		} else if ch == utf8.RuneError {
			return false
		} else if !unicode.IsDigit(ch) && !unicode.IsLetter(ch) {
			return false
		}
	}
	return true
}
<commit_msg>cmd/scollector: Return, don't print, scan error<commit_after>package collectors

import (
	"bufio"
	"os"
	"strings"
	"sync"
	"time"
	"unicode"
	"unicode/utf8"

	"github.com/StackExchange/scollector/metadata"
	"github.com/StackExchange/scollector/opentsdb"
	"github.com/StackExchange/scollector/util"
)

var collectors []Collector

type Collector interface {
	Run(chan<- *opentsdb.DataPoint)
	Name() string
	Init()
}

const (
	osCPU          = "os.cpu"
	osDiskFree     = "os.disk.fs.space_free"
	osDiskPctFree  = "os.disk.fs.percent_free"
	osDiskTotal    = "os.disk.fs.space_total"
	osDiskUsed     = "os.disk.fs.space_used"
	osMemFree      = "os.mem.free"
	osMemPctFree   = "os.mem.percent_free"
	osMemTotal     = "os.mem.total"
	osMemUsed      = "os.mem.used"
	osNetBroadcast = "os.net.packets_broadcast"
	osNetBytes     = "os.net.bytes"
	osNetDropped   = "os.net.dropped"
	osNetErrors    = "os.net.errs"
	osNetPackets   = "os.net.packets"
	osNetUnicast   = "os.net.packets_unicast"
	osNetMulticast = "os.net.packets_multicast"
)

var (
	// DefaultFreq is the duration between collection intervals if none is
	// specified.
	DefaultFreq = time.Second * 15

	timestamp = time.Now().Unix()
	tlock     sync.Mutex
)

func init() {
	go func() {
		for t := range time.Tick(time.Second) {
			tlock.Lock()
			timestamp = t.Unix()
			tlock.Unlock()
		}
	}()
}

func now() (t int64) {
	tlock.Lock()
	t = timestamp
	tlock.Unlock()
	return
}

// Search returns all collectors matching the pattern s.
func Search(s string) []Collector {
	var r []Collector
	for _, c := range collectors {
		if strings.Contains(c.Name(), s) {
			r = append(r, c)
		}
	}
	return r
}

// Runs specified collectors. Use nil for all collectors.
func Run(cs []Collector) chan *opentsdb.DataPoint {
	if cs == nil {
		cs = collectors
	}
	ch := make(chan *opentsdb.DataPoint)
	for _, c := range cs {
		go c.Run(ch)
	}
	return ch
}

// AddTS is the same as Add but lets you specify the timestamp
func AddTS(md *opentsdb.MultiDataPoint, name string, ts int64, value interface{}, t opentsdb.TagSet, rate metadata.RateType, unit metadata.Unit, desc string) {
	tags := make(opentsdb.TagSet)
	for k, v := range t {
		tags[k] = v
	}
	if host, present := tags["host"]; !present {
		tags["host"] = util.Hostname
	} else if host == "" {
		delete(tags, "host")
	}
	d := opentsdb.DataPoint{
		Metric:    name,
		Timestamp: ts,
		Value:     value,
		Tags:      tags,
	}
	*md = append(*md, &d)
	if rate != metadata.Unknown {
		metadata.AddMeta(name, nil, "rate", rate, false)
	}
	if unit != metadata.None {
		metadata.AddMeta(name, nil, "unit", unit, false)
	}
	if desc != "" {
		metadata.AddMeta(name, tags, "desc", desc, false)
	}
}

// Add appends a new data point with given metric name, value, and tags. Tags
// may be nil. If tags is nil or does not contain a host key, it will be
// automatically added. If the value of the host key is the empty string, it
// will be removed (use this to prevent the normal auto-adding of the host tag).
func Add(md *opentsdb.MultiDataPoint, name string, value interface{}, t opentsdb.TagSet, rate metadata.RateType, unit metadata.Unit, desc string) {
	AddTS(md, name, now(), value, t, rate, unit, desc)
}

func readLine(fname string, line func(string) error) error {
	f, err := os.Open(fname)
	if err != nil {
		return err
	}
	defer f.Close()
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		if err := line(scanner.Text()); err != nil {
			return err
		}
	}
	return scanner.Err()
}

// IsDigit returns true if s consists of decimal digits.
func IsDigit(s string) bool {
	r := strings.NewReader(s)
	for {
		ch, _, err := r.ReadRune()
		if ch == 0 || err != nil {
			break
		} else if ch == utf8.RuneError {
			return false
		} else if !unicode.IsDigit(ch) {
			return false
		}
	}
	return true
}

// IsAlNum returns true if s is alphanumeric.
func IsAlNum(s string) bool {
	r := strings.NewReader(s)
	for {
		ch, _, err := r.ReadRune()
		if ch == 0 || err != nil {
			break
		} else if ch == utf8.RuneError {
			return false
		} else if !unicode.IsDigit(ch) && !unicode.IsLetter(ch) {
			return false
		}
	}
	return true
}
<|endoftext|>