<commit_before>// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"os"
	"sync"

	"github.com/Sirupsen/logrus"
	"github.com/urfave/cli"
)

// name is the name of the program.
const name = "localCI"

// usage is the usage of the program.
const usage = name + ` is a continuous integration system`

var ciLog = logrus.New()

func loop(repos []Repo) error {
	var wg sync.WaitGroup

	for _, r := range repos {
		if err := r.setup(); err != nil {
			ciLog.Errorf("failed to initialize repository: %s", err)
			continue
		}

		wg.Add(1)
		go func(repo Repo) {
			repo.loop()
			wg.Done()
		}(r)
	}

	wg.Wait()

	return nil
}

func testRepos(repos []Repo) error {
	for _, r := range repos {
		if err := r.setup(); err != nil {
			ciLog.Errorf("failed to initialize repository: %s", err)
			continue
		}

		if err := r.test(); err != nil {
			ciLog.Errorf("failed to test repo %+v: %s", r, err)
			return err
		}
	}

	return nil
}

func main() {
	app := cli.NewApp()
	app.Name = name
	app.Usage = usage
	app.Version = fmt.Sprintf("localCI : %s\nCommit  : %s", version, commit)

	// Override the default function to display version details to
	// ensure the "--version" option and "version" command are identical.
	cli.VersionPrinter = func(c *cli.Context) {
		fmt.Println(c.App.Version)
	}

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "config",
			Usage: "configuration file path",
		},
		cli.BoolFlag{
			Name:  "daemon",
			Usage: "run " + name + " as a daemon monitoring the repositories specified in the configuration file",
		},
		cli.BoolFlag{
			Name:  "debug",
			Usage: "enable debug output for logging",
		},
		cli.StringFlag{
			Name:  "log",
			Value: "/dev/null",
			Usage: "set the log file path where internal debug information is written",
		},
	}

	app.Before = func(context *cli.Context) error {
		var err error

		if context.GlobalBool("debug") {
			ciLog.Level = logrus.DebugLevel
		}

		if path := context.GlobalString("log"); path != "" {
			f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND|os.O_SYNC, 0640)
			if err != nil {
				return err
			}
			ciLog.Out = f
		}

		config, err := newConfig(context.GlobalString("config"))
		if err != nil {
			return err
		}
		ciLog.Debugf("configuration %+v", config)

		runTestsInParallel = config.RunTestsInParallel

		context.App.Metadata = map[string]interface{}{
			"repos": config.Repo,
		}

		return nil
	}

	app.Action = func(context *cli.Context) error {
		repos := context.App.Metadata["repos"].([]Repo)

		if len(repos) == 0 {
			ciLog.Info("There are no repos to monitor or test")
			return nil
		}

		if context.GlobalBool("daemon") {
			return loop(repos)
		}
		return testRepos(repos)
	}

	if err := app.Run(os.Args); err != nil {
		ciLog.Error(err)
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}
<commit_msg>localCI: Add loglevels msgs support<commit_after>// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"os"
	"sync"

	"github.com/Sirupsen/logrus"
	"github.com/urfave/cli"
)

// name is the name of the program.
const name = "localCI"

// usage is the usage of the program.
const usage = name + ` is a continuous integration system`

var ciLog = logrus.New()

func loop(repos []Repo) error {
	var wg sync.WaitGroup

	for _, r := range repos {
		if err := r.setup(); err != nil {
			ciLog.Errorf("failed to initialize repository: %s", err)
			continue
		}

		wg.Add(1)
		go func(repo Repo) {
			repo.loop()
			wg.Done()
		}(r)
	}

	wg.Wait()

	return nil
}

func testRepos(repos []Repo) error {
	for _, r := range repos {
		if err := r.setup(); err != nil {
			ciLog.Errorf("failed to initialize repository: %s", err)
			continue
		}

		if err := r.test(); err != nil {
			ciLog.Errorf("failed to test repo %+v: %s", r, err)
			return err
		}
	}

	return nil
}

func main() {
	app := cli.NewApp()
	app.Name = name
	app.Usage = usage
	app.Version = fmt.Sprintf("localCI : %s\nCommit  : %s", version, commit)

	// Override the default function to display version details to
	// ensure the "--version" option and "version" command are identical.
	cli.VersionPrinter = func(c *cli.Context) {
		fmt.Println(c.App.Version)
	}

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "config",
			Usage: "configuration file path",
		},
		cli.BoolFlag{
			Name:  "daemon",
			Usage: "run " + name + " as a daemon monitoring the repositories specified in the configuration file",
		},
		cli.StringFlag{
			Name:  "log",
			Value: "/dev/null",
			Usage: "set the log file path where internal debug information is written",
		},
		cli.StringFlag{
			Name: "loglevel",
			Value: "error",
			Usage: "log level messages: <debug|error|fatal|info|panic|warn>",
		},
	}

	app.Before = func(context *cli.Context) error {
		var err error

		// Set log level messages, error by default
		switch logl := context.GlobalString("loglevel"); logl {
			case "debug":
				ciLog.Level = logrus.DebugLevel
			case "error":
				ciLog.Level = logrus.ErrorLevel
			case "fatal":
				ciLog.Level = logrus.FatalLevel
			case "info":
				ciLog.Level = logrus.InfoLevel
			case "panic":
				ciLog.Level = logrus.PanicLevel
			case "warn":
				ciLog.Level = logrus.WarnLevel
			default:
				ciLog.Errorf("%s: unknown log level", logl)
				os.Exit(1)
		}

		if path := context.GlobalString("log"); path != "" {
			f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND|os.O_SYNC, 0640)
			if err != nil {
				return err
			}
			ciLog.Out = f
		}

		config, err := newConfig(context.GlobalString("config"))
		if err != nil {
			return err
		}
		ciLog.Debugf("configuration %+v", config)

		runTestsInParallel = config.RunTestsInParallel

		context.App.Metadata = map[string]interface{}{
			"repos": config.Repo,
		}

		return nil
	}

	app.Action = func(context *cli.Context) error {
		repos := context.App.Metadata["repos"].([]Repo)

		if len(repos) == 0 {
			ciLog.Info("There are no repos to monitor or test")
			return nil
		}

		if context.GlobalBool("daemon") {
			return loop(repos)
		}
		return testRepos(repos)
	}

	if err := app.Run(os.Args); err != nil {
		ciLog.Error(err)
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"

	"github.com/devict/magopie"
	"github.com/gorilla/mux"
	"github.com/justinas/alice"
	"github.com/spf13/afero"
)

var (
	httpAddr    = flag.String("http", ":8080", "HTTP service address")
	downloadDir = flag.String("dir", ".", "Directory where magopie should download .torrent files")
	apiKey      = flag.String("key", "", "Shared API key for clients (required)")
)

func init() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
}

func main() {
	flag.Parse()

	if *apiKey == "" {
		fmt.Fprintln(os.Stderr, "-key flag is required")
		flag.Usage()
		os.Exit(1)
	}

	a := &server{
		key: *apiKey,
		fs:  afero.OsFs{},
		sites: sitedb{
			sites: []site{
				kickAssTorrents,
				thePirateBay,
			},
		},
		torcacheURL: "http://torcache.net",
		downloadDir: *downloadDir,
	}

	log.Printf("Listening on %s", *httpAddr)
	log.Fatal(http.ListenAndServe(*httpAddr, router(a)))
}

func router(a *server) http.Handler {
	r := mux.NewRouter()

	r.HandleFunc("/sites", a.handleAllSites).Methods("GET")
	r.HandleFunc("/sites/{id}", a.handleSingleSite).Methods("GET")
	r.HandleFunc("/torrents", a.handleTorrents).Methods("GET")
	r.HandleFunc("/download/{hash}", a.handleDownload).Methods("POST")

	chain := alice.New(mwLogger, mwAuthenticationCheck(a.key)).Then(r)

	return chain
}

func mwLogger(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		log.Println("Serving", r.Method, r.URL.String(), "to", r.RemoteAddr)
		next.ServeHTTP(w, r)
	})
}

func mwAuthenticationCheck(key string) func(http.Handler) http.Handler {
	return func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			if !magopie.CheckMAC(r.Header.Get("X-Request-ID"), r.Header.Get("X-HMAC"), key) {
				log.Println("Request failed HMAC")
				w.WriteHeader(http.StatusUnauthorized)
				return
			}
			next.ServeHTTP(w, r)
		})
	}
}
<commit_msg>Add support for TLS<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"

	"github.com/devict/magopie"
	"github.com/gorilla/mux"
	"github.com/justinas/alice"
	"github.com/spf13/afero"
)

var (
	addr        = flag.String("addr", ":8080", "HTTP(S) service address")
	downloadDir = flag.String("dir", ".", "Directory where magopie should download .torrent files")
	apiKey      = flag.String("key", "", "Shared API key for clients (required)")
	tlsKey      = flag.String("tlsKey", "", "Path to TLS Key")
	tlsCert     = flag.String("tlsCert", "", "Path to TLS Cert")
)

func init() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
}

func main() {
	flag.Parse()

	if *apiKey == "" {
		fmt.Fprintln(os.Stderr, "-key flag is required")
		flag.Usage()
		os.Exit(1)
	}

	a := &server{
		key: *apiKey,
		fs:  afero.OsFs{},
		sites: sitedb{
			sites: []site{
				kickAssTorrents,
				thePirateBay,
			},
		},
		torcacheURL: "http://torcache.net",
		downloadDir: *downloadDir,
	}

	if *tlsKey != "" && *tlsCert != "" {
		log.Printf("Listening for HTTPS on %s with key %s and cert %s", *addr, *tlsKey, *tlsCert)
		log.Fatal(http.ListenAndServeTLS(*addr, *tlsCert, *tlsKey, router(a)))
	} else {
		log.Printf("Listening for HTTP on %s", *addr)
		log.Fatal(http.ListenAndServe(*addr, router(a)))
	}
}

func router(a *server) http.Handler {
	r := mux.NewRouter()

	r.HandleFunc("/sites", a.handleAllSites).Methods("GET")
	r.HandleFunc("/sites/{id}", a.handleSingleSite).Methods("GET")
	r.HandleFunc("/torrents", a.handleTorrents).Methods("GET")
	r.HandleFunc("/download/{hash}", a.handleDownload).Methods("POST")

	chain := alice.New(mwLogger, mwAuthenticationCheck(a.key)).Then(r)

	return chain
}

func mwLogger(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		log.Println("Serving", r.Method, r.URL.String(), "to", r.RemoteAddr)
		next.ServeHTTP(w, r)
	})
}

func mwAuthenticationCheck(key string) func(http.Handler) http.Handler {
	return func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			if !magopie.CheckMAC(r.Header.Get("X-Request-ID"), r.Header.Get("X-HMAC"), key) {
				log.Println("Request failed HMAC")
				w.WriteHeader(http.StatusUnauthorized)
				return
			}
			next.ServeHTTP(w, r)
		})
	}
}
<|endoftext|>


<commit_before>package cmd

import (
    "testing"

    "github.com/stretchr/testify/assert"
)

func TestValidateStatus(t *testing.T) {
    res := map[string]int{}
    res["200"] = 5
    assert.True(t, ValidateStatus(res, 5, "200:100"), "5x 200, should get us a 100% 200")
    assert.False(t, ValidateStatus(res, 10, "200:100"), "10x 200, should get us only 50% 200")
    res["500"] = 5
    assert.True(t, ValidateStatus(res, 10, "200:50,500:50"), "10x 200, should get us only 50% 200")
}
<commit_msg>fix monitor test<commit_after>package cmd

import (
	"testing"

	"github.com/stretchr/testify/assert"

	"github.com/cp2017/go-client/util"
)

func TestValidateStatus(t *testing.T) {
	res := map[string]int{}
	res["200"] = 5
	assert.True(t, util.ValidateStatus(res, 5, "200:100"), "5x 200, should get us a 100% 200")
	assert.False(t, util.ValidateStatus(res, 10, "200:100"), "10x 200, should get us only 50% 200")
	res["500"] = 5
	assert.True(t, util.ValidateStatus(res, 10, "200:50,500:50"), "10x 200, should get us only 50% 200")
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"encoding/json"
	"fmt"
	"net/url"
	"path/filepath"
	"strings"

	"cloud.google.com/go/storage"
	"google.golang.org/api/iterator"
)

type CloudStorage struct {
	Client storage.Client
	Bucket string
	ctx    context.Context
}

func NewCloudStorage(bucket string) (CloudStorage, error) {
	cs := CloudStorage{}

	ctx := context.Background()
	client, err := storage.NewClient(ctx)
	if err != nil {
		return cs, fmt.Errorf("failed to create client: %v", err)
	}
	cs.Client = *client
	cs.Bucket = bucket
	cs.ctx = ctx

	return cs, nil
}

func (cs *CloudStorage) Close() error {
	return cs.Client.Close()
}

func (cs CloudStorage) List() (CSFiles, error) {
	i := CSFiles{}
	bucket := cs.Client.Bucket(cs.Bucket)

	query := &storage.Query{}
	it := bucket.Objects(cs.ctx, query)
	for {
		obj, err := it.Next()
		if err == iterator.Done {
			break
		}
		if err != nil {
			return i, fmt.Errorf("error iterating over bucket query: %s", err)
		}

		u, err := url.Parse(obj.MediaLink)
		if err != nil {
			return i, fmt.Errorf("cannot create url from %s: %s", obj.MediaLink, err)
		}
		img := CSFile{obj.Name, u}
		i = append(i, img)

	}

	return i, nil
}

type CSFile struct {
	Name string
	URL  *url.URL
}

type CSFiles []CSFile

type Image struct {
	Name      string `json:"name"`
	Original  string `json:"original"`
	Thumbnail string `json:"thumbnail"`
}

type Images []Image

func NewImages(fs CSFiles) (Images, error) {
	is := Images{}
	err := is.Load(fs)
	return is, err
}

func (is *Images) Load(fs CSFiles) error {
	for _, v := range fs {
		if strings.Index(v.Name, "original.") > -1 {
			base := filepath.Dir(v.Name)
			name := strings.Replace(base, "processed/", "", 1)
			o := v.URL.String()
			t := strings.Replace(o, "original.", "thumbnail.", 1)
			i := Image{name, o, t}
			*is = append(*is, i)
		}
	}

	return nil
}

// JSON marshalls the content of Images to json.
func (is Images) JSON() (string, error) {
	bytes, err := json.Marshal(is)
	if err != nil {
		return "", fmt.Errorf("could not marshal json for response: %s", err)
	}

	return string(bytes), nil
}

// JSONBytes marshalls the content of Images to json.
func (is Images) JSONBytes() ([]byte, error) {
	bytes, err := json.Marshal(is)
	if err != nil {
		return []byte{}, fmt.Errorf("could not marshal json for response: %s", err)
	}

	return bytes, nil
}
<commit_msg>Getting list of processed file working.<commit_after>package main

import (
	"context"
	"encoding/json"
	"fmt"
	"net/url"
	"path/filepath"
	"strings"

	"cloud.google.com/go/storage"
	"google.golang.org/api/iterator"
)

type CloudStorage struct {
	Client storage.Client
	Bucket string
	ctx    context.Context
}

func NewCloudStorage(bucket string) (CloudStorage, error) {
	cs := CloudStorage{}

	ctx := context.Background()
	client, err := storage.NewClient(ctx)
	if err != nil {
		return cs, fmt.Errorf("failed to create client: %v", err)
	}
	cs.Client = *client
	cs.Bucket = bucket
	cs.ctx = ctx

	return cs, nil
}

func (cs *CloudStorage) Close() error {
	return cs.Client.Close()
}

func (cs CloudStorage) List() (CSFiles, error) {
	i := CSFiles{}
	bucket := cs.Client.Bucket(cs.Bucket)

	query := &storage.Query{}
	it := bucket.Objects(cs.ctx, query)
	for {
		obj, err := it.Next()
		if err == iterator.Done {
			break
		}
		if err != nil {
			return i, fmt.Errorf("error iterating over bucket query: %s", err)
		}

		u, err := url.Parse(obj.MediaLink)
		if err != nil {
			return i, fmt.Errorf("cannot create url from %s: %s", obj.MediaLink, err)
		}
		img := CSFile{obj.Name, cs.Bucket, u}
		i = append(i, img)

	}

	return i, nil
}

type CSFile struct {
	Name   string
	Bucket string
	URL    *url.URL
}

type CSFiles []CSFile

type Image struct {
	Name      string `json:"name"`
	Original  string `json:"original"`
	Thumbnail string `json:"thumbnail"`
}

type Images []Image

func NewImages(fs CSFiles) (Images, error) {
	is := Images{}
	err := is.Load(fs)
	return is, err
}

func (is *Images) Load(fs CSFiles) error {
	for _, v := range fs {
		if strings.Index(v.Name, "original.") > -1 {
			dir := filepath.Dir(v.Name)
			base := filepath.Base(v.Name)
			name := strings.Replace(dir, "processed/", "", 1)
			o := fmt.Sprintf("https://storage.googleapis.com/%s/%s/%s", v.Bucket, dir, base)
			t := fmt.Sprintf("https://storage.googleapis.com/%s/%s/%s", v.Bucket, dir, strings.Replace(base, "original.", "thumbnail.", 1))
			i := Image{name, o, t}
			*is = append(*is, i)
		}
	}

	return nil
}

// JSON marshalls the content of Images to json.
func (is Images) JSON() (string, error) {
	bytes, err := json.Marshal(is)
	if err != nil {
		return "", fmt.Errorf("could not marshal json for response: %s", err)
	}

	return string(bytes), nil
}

// JSONBytes marshalls the content of Images to json.
func (is Images) JSONBytes() ([]byte, error) {
	bytes, err := json.Marshal(is)
	if err != nil {
		return []byte{}, fmt.Errorf("could not marshal json for response: %s", err)
	}

	return bytes, nil
}
<|endoftext|>


<commit_before>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package router

import (
	"io"

	"github.com/TheThingsNetwork/ttn/api"
	"github.com/TheThingsNetwork/ttn/api/gateway"
	"github.com/TheThingsNetwork/ttn/utils/errors"
	"github.com/golang/protobuf/ptypes/empty"
	"google.golang.org/grpc/metadata"
)

// RouterStreamServer handles gRPC streams as channels
type RouterStreamServer struct {
	ctx                   api.Logger
	UplinkChanFunc        func(md metadata.MD) (ch chan *UplinkMessage, err error)
	GatewayStatusChanFunc func(md metadata.MD) (ch chan *gateway.Status, err error)
	DownlinkChanFunc      func(md metadata.MD) (ch <-chan *DownlinkMessage, cancel func(), err error)
}

// NewRouterStreamServer returns a new RouterStreamServer
func NewRouterStreamServer() *RouterStreamServer {
	return &RouterStreamServer{
		ctx: api.GetLogger(),
	}
}

// SetLogger sets the logger
func (s *RouterStreamServer) SetLogger(logger api.Logger) {
	s.ctx = logger
}

// Uplink handles uplink streams
func (s *RouterStreamServer) Uplink(stream Router_UplinkServer) (err error) {
	md, err := api.MetadataFromContext(stream.Context())
	if err != nil {
		return err
	}
	ch, err := s.UplinkChanFunc(md)
	if err != nil {
		return err
	}
	defer func() {
		ctx := s.ctx
		if err != nil {
			ctx = ctx.WithError(err)
		}
		close(ch)
		ctx.Debug("Closed Uplink stream")
	}()
	for {
		uplink, err := stream.Recv()
		if err == io.EOF {
			return stream.SendAndClose(&empty.Empty{})
		}
		if err != nil {
			return err
		}
		if err := uplink.Validate(); err != nil {
			return errors.BuildGRPCError(errors.Wrap(err, "Invalid Uplink"))
		}
		ch <- uplink
	}
}

// Subscribe handles downlink streams
func (s *RouterStreamServer) Subscribe(req *SubscribeRequest, stream Router_SubscribeServer) (err error) {
	md, err := api.MetadataFromContext(stream.Context())
	if err != nil {
		return err
	}
	ch, cancel, err := s.DownlinkChanFunc(md)
	if err != nil {
		return err
	}
	defer cancel()
	for downlink := range ch {
		if err := stream.Send(downlink); err != nil {
			return err
		}
	}
	return nil
}

// GatewayStatus handles gateway status streams
func (s *RouterStreamServer) GatewayStatus(stream Router_GatewayStatusServer) error {
	md, err := api.MetadataFromContext(stream.Context())
	if err != nil {
		return err
	}
	ch, err := s.GatewayStatusChanFunc(md)
	if err != nil {
		return err
	}
	defer func() {
		close(ch)
	}()
	for {
		status, err := stream.Recv()
		if err == io.EOF {
			return stream.SendAndClose(&empty.Empty{})
		}
		if err != nil {
			return err
		}
		if err := status.Validate(); err != nil {
			return errors.BuildGRPCError(errors.Wrap(err, "Invalid Gateway Status"))
		}
		ch <- status
	}
}
<commit_msg>Fix closing of Router Subscribe stream<commit_after>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package router

import (
	"io"

	"github.com/TheThingsNetwork/ttn/api"
	"github.com/TheThingsNetwork/ttn/api/gateway"
	"github.com/TheThingsNetwork/ttn/utils/errors"
	"github.com/golang/protobuf/ptypes/empty"
	"google.golang.org/grpc/metadata"
)

// RouterStreamServer handles gRPC streams as channels
type RouterStreamServer struct {
	ctx                   api.Logger
	UplinkChanFunc        func(md metadata.MD) (ch chan *UplinkMessage, err error)
	GatewayStatusChanFunc func(md metadata.MD) (ch chan *gateway.Status, err error)
	DownlinkChanFunc      func(md metadata.MD) (ch <-chan *DownlinkMessage, cancel func(), err error)
}

// NewRouterStreamServer returns a new RouterStreamServer
func NewRouterStreamServer() *RouterStreamServer {
	return &RouterStreamServer{
		ctx: api.GetLogger(),
	}
}

// SetLogger sets the logger
func (s *RouterStreamServer) SetLogger(logger api.Logger) {
	s.ctx = logger
}

// Uplink handles uplink streams
func (s *RouterStreamServer) Uplink(stream Router_UplinkServer) (err error) {
	md, err := api.MetadataFromContext(stream.Context())
	if err != nil {
		return err
	}
	ch, err := s.UplinkChanFunc(md)
	if err != nil {
		return err
	}
	defer func() {
		ctx := s.ctx
		if err != nil {
			ctx = ctx.WithError(err)
		}
		close(ch)
		ctx.Debug("Closed Uplink stream")
	}()
	for {
		uplink, err := stream.Recv()
		if err == io.EOF {
			return stream.SendAndClose(&empty.Empty{})
		}
		if err != nil {
			return err
		}
		if err := uplink.Validate(); err != nil {
			return errors.BuildGRPCError(errors.Wrap(err, "Invalid Uplink"))
		}
		ch <- uplink
	}
}

// Subscribe handles downlink streams
func (s *RouterStreamServer) Subscribe(req *SubscribeRequest, stream Router_SubscribeServer) (err error) {
	md, err := api.MetadataFromContext(stream.Context())
	if err != nil {
		return err
	}
	ch, cancel, err := s.DownlinkChanFunc(md)
	if err != nil {
		return err
	}
	go func() {
		<-stream.Context().Done()
		err = stream.Context().Err()
		cancel()
	}()
	for downlink := range ch {
		if err := stream.Send(downlink); err != nil {
			return err
		}
	}
	return
}

// GatewayStatus handles gateway status streams
func (s *RouterStreamServer) GatewayStatus(stream Router_GatewayStatusServer) error {
	md, err := api.MetadataFromContext(stream.Context())
	if err != nil {
		return err
	}
	ch, err := s.GatewayStatusChanFunc(md)
	if err != nil {
		return err
	}
	defer func() {
		close(ch)
	}()
	for {
		status, err := stream.Recv()
		if err == io.EOF {
			return stream.SendAndClose(&empty.Empty{})
		}
		if err != nil {
			return err
		}
		if err := status.Validate(); err != nil {
			return errors.BuildGRPCError(errors.Wrap(err, "Invalid Gateway Status"))
		}
		ch <- status
	}
}
<|endoftext|>


<commit_before>package github

import (
	"errors"
	"fmt"
	"github.com/jingweno/gh/git"
	"github.com/jingweno/gh/utils"
	"regexp"
)

type Project struct {
	Name  string
	Owner string
}

func (p *Project) WebURL(name, owner, path string) string {
	if owner == "" {
		owner = p.Owner
	}
	if name == "" {
		name = p.Name
	}

	url := fmt.Sprintf("https://%s", utils.ConcatPaths(GitHubHost, owner, name))
	if path != "" {
		url = utils.ConcatPaths(url, path)
	}

	return url
}

func (p *Project) LocalRepoWith(base, head string) *Repo {
	if base == "" {
		base = "master"
	}
	if head == "" {
		head, _ = git.Head()
	}

	return &Repo{base, head, p}
}

func (p *Project) LocalRepo() *Repo {
	return p.LocalRepoWith("", "")
}

func CurrentProject() *Project {
	remote, err := git.Remote()
	utils.Check(err)

	owner, name := parseOwnerAndName(remote)

	return &Project{name, owner}
}

func parseOwnerAndName(remote string) (owner string, name string) {
	url, err := mustMatchGitHubURL(remote)
	utils.Check(err)

	return url[1], url[2]
}

func mustMatchGitHubURL(url string) ([]string, error) {
	httpRegex := regexp.MustCompile("https://github.com/(.+)/(.+?)(.git|$)")
	if httpRegex.MatchString(url) {
		return httpRegex.FindStringSubmatch(url), nil
	}

	readOnlyRegex := regexp.MustCompile("git://github.com/(.+)/(.+?)(.git|$)")
	if readOnlyRegex.MatchString(url) {
		return readOnlyRegex.FindStringSubmatch(url), nil
	}

	sshRegex := regexp.MustCompile("git@github.com:(.+)/(.+?)(.git|$)")
	if sshRegex.MatchString(url) {
		return sshRegex.FindStringSubmatch(url), nil
	}

	return nil, errors.New("The origin remote doesn't point to a GitHub repository: " + url)
}
<commit_msg>Escaped regexp dots<commit_after>package github

import (
	"errors"
	"fmt"
	"github.com/jingweno/gh/git"
	"github.com/jingweno/gh/utils"
	"regexp"
)

type Project struct {
	Name  string
	Owner string
}

func (p *Project) WebURL(name, owner, path string) string {
	if owner == "" {
		owner = p.Owner
	}
	if name == "" {
		name = p.Name
	}

	url := fmt.Sprintf("https://%s", utils.ConcatPaths(GitHubHost, owner, name))
	if path != "" {
		url = utils.ConcatPaths(url, path)
	}

	return url
}

func (p *Project) LocalRepoWith(base, head string) *Repo {
	if base == "" {
		base = "master"
	}
	if head == "" {
		head, _ = git.Head()
	}

	return &Repo{base, head, p}
}

func (p *Project) LocalRepo() *Repo {
	return p.LocalRepoWith("", "")
}

func CurrentProject() *Project {
	remote, err := git.Remote()
	utils.Check(err)

	owner, name := parseOwnerAndName(remote)

	return &Project{name, owner}
}

func parseOwnerAndName(remote string) (owner string, name string) {
	url, err := mustMatchGitHubURL(remote)
	utils.Check(err)

	return url[1], url[2]
}

func mustMatchGitHubURL(url string) ([]string, error) {
	httpRegex := regexp.MustCompile("https://github\.com/(.+)/(.+)(\.git|$)")
	if httpRegex.MatchString(url) {
		return httpRegex.FindStringSubmatch(url), nil
	}

	readOnlyRegex := regexp.MustCompile("git://github\.com/(.+)/(.+)(\.git|$)")
	if readOnlyRegex.MatchString(url) {
		return readOnlyRegex.FindStringSubmatch(url), nil
	}

	sshRegex := regexp.MustCompile("git@github\.com:(.+)/(.+)(\.git|$)")
	if sshRegex.MatchString(url) {
		return sshRegex.FindStringSubmatch(url), nil
	}

	return nil, errors.New("The origin remote doesn't point to a GitHub repository: " + url)
}
<|endoftext|>


<commit_before>package goanna

import (
	"time"
)

// SessionFinder finds a session based on the request
type SessionFinder func(*Request) Session

type Session interface {
	GetId() string
	Get(string) string
	Set(string, string)
	SetMaxAge(time.Duration)
	HasExpired() bool
	Clear()
	WriteToResponse(Response)
}

func NopSessionFinder(_ *Request) Session { return &NopSession{} }

type NopSession struct{}

func (s NopSession) Get(_ string) string             { return "" }
func (s NopSession) Set(_string, _ string)           {}
func (s NopSession) Expiry() time.Time               { return time.Now() }
func (s NopSession) SetExpiry(_ time.Time) time.Time { return time.Now() }
func (s NopSession) WriteToResponse(_ Response)      {}
func (s NopSession) GetId() string                   { return "" }
func (s NopSession) Clear()                          {}
<commit_msg>Update handler test with new session interface<commit_after>package goanna

import (
	"time"
)

// SessionFinder finds a session based on the request
type SessionFinder func(*Request) Session

type Session interface {
	GetId() string
	Get(string) string
	Set(string, string)
	SetMaxAge(time.Duration)
	HasExpired() bool
	Clear()
	WriteToResponse(Response)
}

func NopSessionFinder(_ *Request) Session { return &NopSession{} }

type NopSession struct{}

func (s NopSession) GetId() string              { return "" }
func (s NopSession) Get(_ string) string        { return "" }
func (s NopSession) Set(_string, _ string)      {}
func (s NopSession) SetMaxAge(_ time.Duration)  {}
func (s NopSession) HasExpired() bool           { return false }
func (s NopSession) Clear()                     {}
func (s NopSession) WriteToResponse(_ Response) {}
<|endoftext|>


<commit_before>package pki

import (
	"encoding/base64"
	"fmt"
	"time"

	"github.com/hashicorp/vault/helper/certutil"
	"github.com/hashicorp/vault/logical"
	"github.com/hashicorp/vault/logical/framework"
)

func pathIssue(b *backend) *framework.Path {
	ret := &framework.Path{
		Pattern: "issue/" + framework.GenericNameRegex("role"),

		Callbacks: map[logical.Operation]framework.OperationFunc{
			logical.UpdateOperation: b.pathIssue,
		},

		HelpSynopsis:    pathIssueHelpSyn,
		HelpDescription: pathIssueHelpDesc,
	}

	ret.Fields = addNonCACommonFields(map[string]*framework.FieldSchema{})

	return ret
}

func pathSign(b *backend) *framework.Path {
	ret := &framework.Path{
		Pattern: "sign/" + framework.GenericNameRegex("role"),

		Callbacks: map[logical.Operation]framework.OperationFunc{
			logical.UpdateOperation: b.pathSign,
		},

		HelpSynopsis:    pathSignHelpSyn,
		HelpDescription: pathSignHelpDesc,
	}

	ret.Fields = addNonCACommonFields(map[string]*framework.FieldSchema{})

	ret.Fields["csr"] = &framework.FieldSchema{
		Type:        framework.TypeString,
		Default:     "",
		Description: `PEM-format CSR to be signed.`,
	}

	return ret
}

func pathSignVerbatim(b *backend) *framework.Path {
	ret := &framework.Path{
		Pattern: "sign-verbatim/" + framework.GenericNameRegex("role"),

		Callbacks: map[logical.Operation]framework.OperationFunc{
			logical.UpdateOperation: b.pathSignVerbatim,
		},

		HelpSynopsis:    pathSignHelpSyn,
		HelpDescription: pathSignHelpDesc,
	}

	ret.Fields = addNonCACommonFields(map[string]*framework.FieldSchema{})

	ret.Fields["csr"] = &framework.FieldSchema{
		Type:    framework.TypeString,
		Default: "",
		Description: `PEM-format CSR to be signed. Values will be
taken verbatim from the CSR, except for
basic constraints.`,
	}

	return ret
}

// pathIssue issues a certificate and private key from given parameters,
// subject to role restrictions
func (b *backend) pathIssue(
	req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
	roleName := data.Get("role").(string)

	// Get the role
	role, err := b.getRole(req.Storage, roleName)
	if err != nil {
		return nil, err
	}
	if role == nil {
		return logical.ErrorResponse(fmt.Sprintf("Unknown role: %s", roleName)), nil
	}

	return b.pathIssueSignCert(req, data, role, false, false)
}

// pathSign issues a certificate from a submitted CSR, subject to role
// restrictions
func (b *backend) pathSign(
	req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
	roleName := data.Get("role").(string)

	// Get the role
	role, err := b.getRole(req.Storage, roleName)
	if err != nil {
		return nil, err
	}
	if role == nil {
		return logical.ErrorResponse(fmt.Sprintf("Unknown role: %s", roleName)), nil
	}

	return b.pathIssueSignCert(req, data, role, true, false)
}

// pathSignVerbatim issues a certificate from a submitted CSR, *not* subject to
// role restrictions
func (b *backend) pathSignVerbatim(
	req *logical.Request, data *framework.FieldData) (*logical.Response, error) {

	ttl := b.System().DefaultLeaseTTL()
	role := &roleEntry{
		TTL:              ttl.String(),
		AllowLocalhost:   true,
		AllowAnyName:     true,
		AllowIPSANs:      true,
		EnforceHostnames: false,
		KeyType:          "any",
	}

	return b.pathIssueSignCert(req, data, role, true, true)
}

func (b *backend) pathIssueSignCert(
	req *logical.Request, data *framework.FieldData, role *roleEntry, useCSR, useCSRValues bool) (*logical.Response, error) {
	format := getFormat(data)
	if format == "" {
		return logical.ErrorResponse(
			`The "format" path parameter must be "pem", "der", or "pem_bundle"`), nil
	}

	var caErr error
	signingBundle, caErr := fetchCAInfo(req)
	switch caErr.(type) {
	case certutil.UserError:
		return nil, certutil.UserError{Err: fmt.Sprintf(
			"Could not fetch the CA certificate (was one set?): %s", caErr)}
	case certutil.InternalError:
		return nil, certutil.InternalError{Err: fmt.Sprintf(
			"Error fetching CA certificate: %s", caErr)}
	}

	var parsedBundle *certutil.ParsedCertBundle
	var err error
	if useCSR {
		parsedBundle, err = signCert(b, role, signingBundle, false, useCSRValues, req, data)
	} else {
		parsedBundle, err = generateCert(b, role, signingBundle, false, req, data)
	}
	if err != nil {
		switch err.(type) {
		case certutil.UserError:
			return logical.ErrorResponse(err.Error()), nil
		case certutil.InternalError:
			return nil, err
		}
	}

	cb, err := parsedBundle.ToCertBundle()
	if err != nil {
		return nil, fmt.Errorf("Error converting raw cert bundle to cert bundle: %s", err)
	}

	resp := b.Secret(SecretCertsType).Response(
		map[string]interface{}{
			"certificate": cb.Certificate,
			"issuing_ca":  cb.IssuingCA,
		},
		map[string]interface{}{
			"serial_number": cb.SerialNumber,
		})

	switch format {
	case "pem":
		resp.Data["issuing_ca"] = cb.IssuingCA
		resp.Data["certificate"] = cb.Certificate

		if !useCSR {
			resp.Data["private_key"] = cb.PrivateKey
			resp.Data["private_key_type"] = cb.PrivateKeyType
		}

	case "pem_bundle":
		resp.Data["issuing_ca"] = cb.IssuingCA
		resp.Data["certificate"] = fmt.Sprintf("%s\n%s", cb.Certificate, cb.IssuingCA)
		if !useCSR {
			resp.Data["private_key"] = cb.PrivateKey
			resp.Data["private_key_type"] = cb.PrivateKeyType
			resp.Data["certificate"] = fmt.Sprintf("%s\n%s\n%s", cb.PrivateKey, cb.Certificate, cb.IssuingCA)
		}

	case "der":
		resp.Data["certificate"] = base64.StdEncoding.EncodeToString(parsedBundle.CertificateBytes)
		resp.Data["issuing_ca"] = base64.StdEncoding.EncodeToString(parsedBundle.IssuingCABytes)
		if !useCSR {
			resp.Data["private_key"] = base64.StdEncoding.EncodeToString(parsedBundle.PrivateKeyBytes)
		}
	}

	resp.Secret.TTL = parsedBundle.Certificate.NotAfter.Sub(time.Now())

	err = req.Storage.Put(&logical.StorageEntry{
		Key:   "certs/" + cb.SerialNumber,
		Value: parsedBundle.CertificateBytes,
	})
	if err != nil {
		return nil, fmt.Errorf("Unable to store certificate locally")
	}

	return resp, nil
}

const pathIssueHelpSyn = `
Request a certificate using a certain role with the provided details.
`

const pathIssueHelpDesc = `
This path allows requesting a certificate to be issued according to the
policy of the given role. The certificate will only be issued if the
requested details are allowed by the role policy.

This path returns a certificate and a private key. If you want a workflow
that does not expose a private key, generate a CSR locally and use the
sign path instead.
`

const pathSignHelpSyn = `
Request certificates using a certain role with the provided details.
`

const pathSignHelpDesc = `
This path allows requesting certificates to be issued according to the
policy of the given role. The certificate will only be issued if the
requested common name is allowed by the role policy.

This path requires a CSR; if you want Vault to generate a private key
for you, use the issue path instead.
`
<commit_msg>Add serial_number back to path_issue_sign responses in PKI<commit_after>package pki

import (
	"encoding/base64"
	"fmt"
	"time"

	"github.com/hashicorp/vault/helper/certutil"
	"github.com/hashicorp/vault/logical"
	"github.com/hashicorp/vault/logical/framework"
)

func pathIssue(b *backend) *framework.Path {
	ret := &framework.Path{
		Pattern: "issue/" + framework.GenericNameRegex("role"),

		Callbacks: map[logical.Operation]framework.OperationFunc{
			logical.UpdateOperation: b.pathIssue,
		},

		HelpSynopsis:    pathIssueHelpSyn,
		HelpDescription: pathIssueHelpDesc,
	}

	ret.Fields = addNonCACommonFields(map[string]*framework.FieldSchema{})

	return ret
}

func pathSign(b *backend) *framework.Path {
	ret := &framework.Path{
		Pattern: "sign/" + framework.GenericNameRegex("role"),

		Callbacks: map[logical.Operation]framework.OperationFunc{
			logical.UpdateOperation: b.pathSign,
		},

		HelpSynopsis:    pathSignHelpSyn,
		HelpDescription: pathSignHelpDesc,
	}

	ret.Fields = addNonCACommonFields(map[string]*framework.FieldSchema{})

	ret.Fields["csr"] = &framework.FieldSchema{
		Type:        framework.TypeString,
		Default:     "",
		Description: `PEM-format CSR to be signed.`,
	}

	return ret
}

func pathSignVerbatim(b *backend) *framework.Path {
	ret := &framework.Path{
		Pattern: "sign-verbatim/" + framework.GenericNameRegex("role"),

		Callbacks: map[logical.Operation]framework.OperationFunc{
			logical.UpdateOperation: b.pathSignVerbatim,
		},

		HelpSynopsis:    pathSignHelpSyn,
		HelpDescription: pathSignHelpDesc,
	}

	ret.Fields = addNonCACommonFields(map[string]*framework.FieldSchema{})

	ret.Fields["csr"] = &framework.FieldSchema{
		Type:    framework.TypeString,
		Default: "",
		Description: `PEM-format CSR to be signed. Values will be
taken verbatim from the CSR, except for
basic constraints.`,
	}

	return ret
}

// pathIssue issues a certificate and private key from given parameters,
// subject to role restrictions
func (b *backend) pathIssue(
	req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
	roleName := data.Get("role").(string)

	// Get the role
	role, err := b.getRole(req.Storage, roleName)
	if err != nil {
		return nil, err
	}
	if role == nil {
		return logical.ErrorResponse(fmt.Sprintf("Unknown role: %s", roleName)), nil
	}

	return b.pathIssueSignCert(req, data, role, false, false)
}

// pathSign issues a certificate from a submitted CSR, subject to role
// restrictions
func (b *backend) pathSign(
	req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
	roleName := data.Get("role").(string)

	// Get the role
	role, err := b.getRole(req.Storage, roleName)
	if err != nil {
		return nil, err
	}
	if role == nil {
		return logical.ErrorResponse(fmt.Sprintf("Unknown role: %s", roleName)), nil
	}

	return b.pathIssueSignCert(req, data, role, true, false)
}

// pathSignVerbatim issues a certificate from a submitted CSR, *not* subject to
// role restrictions
func (b *backend) pathSignVerbatim(
	req *logical.Request, data *framework.FieldData) (*logical.Response, error) {

	ttl := b.System().DefaultLeaseTTL()
	role := &roleEntry{
		TTL:              ttl.String(),
		AllowLocalhost:   true,
		AllowAnyName:     true,
		AllowIPSANs:      true,
		EnforceHostnames: false,
		KeyType:          "any",
	}

	return b.pathIssueSignCert(req, data, role, true, true)
}

func (b *backend) pathIssueSignCert(
	req *logical.Request, data *framework.FieldData, role *roleEntry, useCSR, useCSRValues bool) (*logical.Response, error) {
	format := getFormat(data)
	if format == "" {
		return logical.ErrorResponse(
			`The "format" path parameter must be "pem", "der", or "pem_bundle"`), nil
	}

	var caErr error
	signingBundle, caErr := fetchCAInfo(req)
	switch caErr.(type) {
	case certutil.UserError:
		return nil, certutil.UserError{Err: fmt.Sprintf(
			"Could not fetch the CA certificate (was one set?): %s", caErr)}
	case certutil.InternalError:
		return nil, certutil.InternalError{Err: fmt.Sprintf(
			"Error fetching CA certificate: %s", caErr)}
	}

	var parsedBundle *certutil.ParsedCertBundle
	var err error
	if useCSR {
		parsedBundle, err = signCert(b, role, signingBundle, false, useCSRValues, req, data)
	} else {
		parsedBundle, err = generateCert(b, role, signingBundle, false, req, data)
	}
	if err != nil {
		switch err.(type) {
		case certutil.UserError:
			return logical.ErrorResponse(err.Error()), nil
		case certutil.InternalError:
			return nil, err
		}
	}

	cb, err := parsedBundle.ToCertBundle()
	if err != nil {
		return nil, fmt.Errorf("Error converting raw cert bundle to cert bundle: %s", err)
	}

	resp := b.Secret(SecretCertsType).Response(
		map[string]interface{}{
			"certificate":   cb.Certificate,
			"issuing_ca":    cb.IssuingCA,
			"serial_number": cb.SerialNumber,
		},
		map[string]interface{}{
			"serial_number": cb.SerialNumber,
		})

	switch format {
	case "pem":
		resp.Data["issuing_ca"] = cb.IssuingCA
		resp.Data["certificate"] = cb.Certificate

		if !useCSR {
			resp.Data["private_key"] = cb.PrivateKey
			resp.Data["private_key_type"] = cb.PrivateKeyType
		}

	case "pem_bundle":
		resp.Data["issuing_ca"] = cb.IssuingCA
		resp.Data["certificate"] = fmt.Sprintf("%s\n%s", cb.Certificate, cb.IssuingCA)
		if !useCSR {
			resp.Data["private_key"] = cb.PrivateKey
			resp.Data["private_key_type"] = cb.PrivateKeyType
			resp.Data["certificate"] = fmt.Sprintf("%s\n%s\n%s", cb.PrivateKey, cb.Certificate, cb.IssuingCA)
		}

	case "der":
		resp.Data["certificate"] = base64.StdEncoding.EncodeToString(parsedBundle.CertificateBytes)
		resp.Data["issuing_ca"] = base64.StdEncoding.EncodeToString(parsedBundle.IssuingCABytes)
		if !useCSR {
			resp.Data["private_key"] = base64.StdEncoding.EncodeToString(parsedBundle.PrivateKeyBytes)
		}
	}

	resp.Secret.TTL = parsedBundle.Certificate.NotAfter.Sub(time.Now())

	err = req.Storage.Put(&logical.StorageEntry{
		Key:   "certs/" + cb.SerialNumber,
		Value: parsedBundle.CertificateBytes,
	})
	if err != nil {
		return nil, fmt.Errorf("Unable to store certificate locally")
	}

	return resp, nil
}

const pathIssueHelpSyn = `
Request a certificate using a certain role with the provided details.
`

const pathIssueHelpDesc = `
This path allows requesting a certificate to be issued according to the
policy of the given role. The certificate will only be issued if the
requested details are allowed by the role policy.

This path returns a certificate and a private key. If you want a workflow
that does not expose a private key, generate a CSR locally and use the
sign path instead.
`

const pathSignHelpSyn = `
Request certificates using a certain role with the provided details.
`

const pathSignHelpDesc = `
This path allows requesting certificates to be issued according to the
policy of the given role. The certificate will only be issued if the
requested common name is allowed by the role policy.

This path requires a CSR; if you want Vault to generate a private key
for you, use the issue path instead.
`
<|endoftext|>


<commit_before>package config

import (
	"errors"
	"net/url"
	"github.com/stellar/go/keypair"
	"regexp"
)

// Config contains config params of the bridge server
type Config struct {
	Port              *int
	Horizon           string
	Compliance        string
	LogFormat         string `mapstructure:"log_format"`
	MACKey            string `mapstructure:"mac_key"`
	APIKey            string `mapstructure:"api_key"`
	NetworkPassphrase string `mapstructure:"network_passphrase"`
	Assets            []Asset
	Database          struct {
		Type string
		URL  string
	}
	Accounts
	Callbacks
}

// Asset represents credit asset
type Asset struct {
	Code   string
	Issuer string
}

// Accounts contains values of `accounts` config group
type Accounts struct {
	AuthorizingSeed    string `mapstructure:"authorizing_seed"`
	BaseSeed           string `mapstructure:"base_seed"`
	IssuingAccountID   string `mapstructure:"issuing_account_id"`
	ReceivingAccountID string `mapstructure:"receiving_account_id"`
}

// Callbacks contains values of `callbacks` config group
type Callbacks struct {
	Receive string
	Error   string
}

// Validate validates config and returns error if any of config values is incorrect
func (c *Config) Validate() (err error) {
	if c.Port == nil {
		err = errors.New("port param is required")
		return
	}

	if c.Horizon == "" {
		err = errors.New("horizon param is required")
		return
	}

	_, err = url.Parse(c.Horizon)
	if err != nil {
		err = errors.New("Cannot parse horizon param")
		return
	}

	if c.NetworkPassphrase == "" {
		err = errors.New("network_passphrase param is required")
		return
	}

	for _, asset := range c.Assets {
		if asset.Issuer == "" {
			if asset.Code != "XLM" {
				err = errors.New("Issuer param is required for "+asset.Code)
				return
			}
		}

		if asset.Issuer != "" {
			_, err = keypair.Parse(asset.Issuer)
			if err != nil {
				err = errors.New("Issuing account is invalid for "+asset.Code)
				return
			}
		}

		if asset.Code == "" {
			err = errors.New("Asset code is required")
			return
		}

		if len(asset.Code) > 12 {
			err = errors.New("Asset code can not be more than 12 characters long: "+ asset.Code)
			return
		}

		if asset.Code != "" {

			for _, s := range asset.Code {
				matched, err := regexp.MatchString("^[a-zA-Z0-9]*$", string(s))
				if err != nil {
					return err
				}

				if !matched {
					err = errors.New("Invalid character '"+string(s)+"' in asset code: "+asset.Code)
					return err
				}
    	}

		}
	}

	var dbURL *url.URL
	dbURL, err = url.Parse(c.Database.URL)
	if err != nil {
		err = errors.New("Cannot parse database.url param")
		return
	}

	switch c.Database.Type {
	case "mysql":
		// Add `parseTime=true` param to mysql url
		query := dbURL.Query()
		query.Set("parseTime", "true")
		dbURL.RawQuery = query.Encode()
		c.Database.URL = dbURL.String()
	case "postgres":
		break
	case "":
		// Allow to start gateway server with a single endpoint: /payment
		break
	default:
		err = errors.New("Invalid database.type param")
		return
	}

	if c.Accounts.AuthorizingSeed != "" {
		_, err = keypair.Parse(c.Accounts.AuthorizingSeed)
		if err != nil {
			err = errors.New("accounts.authorizing_seed is invalid")
			return
		}
	}

	if c.Accounts.BaseSeed != "" {
		_, err = keypair.Parse(c.Accounts.BaseSeed)
		if err != nil {
			err = errors.New("accounts.base_seed is invalid")
			return
		}
	}

	if c.Accounts.IssuingAccountID != "" {
		_, err = keypair.Parse(c.Accounts.IssuingAccountID)
		if err != nil {
			err = errors.New("accounts.issuing_account_id is invalid")
			return
		}
	}

	if c.Accounts.ReceivingAccountID != "" {
		_, err = keypair.Parse(c.Accounts.ReceivingAccountID)
		if err != nil {
			err = errors.New("accounts.receiving_account_id is invalid")
			return
		}
	}

	if c.Callbacks.Receive != "" {
		_, err = url.Parse(c.Callbacks.Receive)
		if err != nil {
			err = errors.New("Cannot parse callbacks.receive param")
			return
		}
	}

	if c.Callbacks.Error != "" {
		_, err = url.Parse(c.Callbacks.Error)
		if err != nil {
			err = errors.New("Cannot parse callbacks.error param")
			return
		}
	}

	return
}
<commit_msg>regexp for asset validation<commit_after>package config

import (
	"errors"
	"net/url"
	"github.com/stellar/go/keypair"
	"regexp"
)

// Config contains config params of the bridge server
type Config struct {
	Port              *int
	Horizon           string
	Compliance        string
	LogFormat         string `mapstructure:"log_format"`
	MACKey            string `mapstructure:"mac_key"`
	APIKey            string `mapstructure:"api_key"`
	NetworkPassphrase string `mapstructure:"network_passphrase"`
	Assets            []Asset
	Database          struct {
		Type string
		URL  string
	}
	Accounts
	Callbacks
}

// Asset represents credit asset
type Asset struct {
	Code   string
	Issuer string
}

// Accounts contains values of `accounts` config group
type Accounts struct {
	AuthorizingSeed    string `mapstructure:"authorizing_seed"`
	BaseSeed           string `mapstructure:"base_seed"`
	IssuingAccountID   string `mapstructure:"issuing_account_id"`
	ReceivingAccountID string `mapstructure:"receiving_account_id"`
}

// Callbacks contains values of `callbacks` config group
type Callbacks struct {
	Receive string
	Error   string
}

// Validate validates config and returns error if any of config values is incorrect
func (c *Config) Validate() (err error) {
	if c.Port == nil {
		err = errors.New("port param is required")
		return
	}

	if c.Horizon == "" {
		err = errors.New("horizon param is required")
		return
	}

	_, err = url.Parse(c.Horizon)
	if err != nil {
		err = errors.New("Cannot parse horizon param")
		return
	}

	if c.NetworkPassphrase == "" {
		err = errors.New("network_passphrase param is required")
		return
	}

	for _, asset := range c.Assets {
		if asset.Issuer == "" {
			if asset.Code != "XLM" {
				err = errors.New("Issuer param is required for "+asset.Code)
				return
			}
		}

		if asset.Issuer != "" {
			_, err = keypair.Parse(asset.Issuer)
			if err != nil {
				err = errors.New("Issuing account is invalid for "+asset.Code)
				return
			}
		}

		matched, err := regexp.MatchString("^[a-zA-Z0-9]{1,12}$", asset.Code)
		if err != nil {
			return err
		}

		if !matched {
			err = errors.New("Invalid asset code: "+asset.Code)
			return err
		}
	}

	var dbURL *url.URL
	dbURL, err = url.Parse(c.Database.URL)
	if err != nil {
		err = errors.New("Cannot parse database.url param")
		return
	}

	switch c.Database.Type {
	case "mysql":
		// Add `parseTime=true` param to mysql url
		query := dbURL.Query()
		query.Set("parseTime", "true")
		dbURL.RawQuery = query.Encode()
		c.Database.URL = dbURL.String()
	case "postgres":
		break
	case "":
		// Allow to start gateway server with a single endpoint: /payment
		break
	default:
		err = errors.New("Invalid database.type param")
		return
	}

	if c.Accounts.AuthorizingSeed != "" {
		_, err = keypair.Parse(c.Accounts.AuthorizingSeed)
		if err != nil {
			err = errors.New("accounts.authorizing_seed is invalid")
			return
		}
	}

	if c.Accounts.BaseSeed != "" {
		_, err = keypair.Parse(c.Accounts.BaseSeed)
		if err != nil {
			err = errors.New("accounts.base_seed is invalid")
			return
		}
	}

	if c.Accounts.IssuingAccountID != "" {
		_, err = keypair.Parse(c.Accounts.IssuingAccountID)
		if err != nil {
			err = errors.New("accounts.issuing_account_id is invalid")
			return
		}
	}

	if c.Accounts.ReceivingAccountID != "" {
		_, err = keypair.Parse(c.Accounts.ReceivingAccountID)
		if err != nil {
			err = errors.New("accounts.receiving_account_id is invalid")
			return
		}
	}

	if c.Callbacks.Receive != "" {
		_, err = url.Parse(c.Callbacks.Receive)
		if err != nil {
			err = errors.New("Cannot parse callbacks.receive param")
			return
		}
	}

	if c.Callbacks.Error != "" {
		_, err = url.Parse(c.Callbacks.Error)
		if err != nil {
			err = errors.New("Cannot parse callbacks.error param")
			return
		}
	}

	return
}
<|endoftext|>


<commit_before>// +build unit

// Copyright 2016 Mesosphere, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package framework

import (
	"testing"

	"github.com/dcos/dcos-metrics/producers"
	. "github.com/smartystreets/goconvey/convey"
)

func TestExtract(t *testing.T) {
	Convey("When extracting a datapoint from an Avro record", t, func() {
		testRecord := record{
			Name: "dcos.metrics.Datapoint",
			Fields: []field{
				{
					Name:  "test-name",
					Datum: "name-field-test",
				},
				{
					Name:  "test-value",
					Datum: "value-field-test",
				},
				{
					Name:  "test-unit",
					Datum: "unit-field-test",
				},
			},
		}

		avroDatapoint := avroRecord{testRecord}
		pmmTest := producers.MetricsMessage{}

		err := avroDatapoint.extract(&pmmTest)

		Convey("Should extract the datapoint without errors", func() {
			So(err, ShouldBeNil)
			So(len(pmmTest.Datapoints), ShouldEqual, 1)
		})

		Convey("Should return the expected name and values from the datapoint", func() {
			So(pmmTest.Datapoints[0].Name, ShouldEqual, "name-field-test")
			So(pmmTest.Datapoints[0].Value, ShouldEqual, "value-field-test")
			So(pmmTest.Datapoints[0].Unit, ShouldEqual, "unit-field-test")
		})
	})

	Convey("When extracting tags from an Avro record", t, func() {
		testRecord := record{
			Name: "dcos.metrics.Tag",
			Fields: []field{
				{
					Name:  "test-tag-name",
					Datum: "tag-name-field-test",
				},
				{
					Name:  "test-tag-value",
					Datum: "tag-value-field-test",
				},
			},
		}

		avroDatapoint := avroRecord{testRecord}
		pmmTest := producers.MetricsMessage{
			Dimensions: producers.Dimensions{
				Labels: make(map[string]string),
			},
		}

		Convey("Should extract the tag without errors", func() {
			err := avroDatapoint.extract(&pmmTest)
			value, ok := pmmTest.Dimensions.Labels["tag-name-field-test"]

			So(err, ShouldBeNil)
			So(ok, ShouldBeTrue)
			So(value, ShouldEqual, "tag-value-field-test")
		})
	})
}
<commit_msg>add tests for createObjectFromRecord()<commit_after>// +build unit

// Copyright 2016 Mesosphere, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package framework

import (
	"testing"

	"github.com/dcos/dcos-metrics/producers"
	"github.com/dcos/dcos-metrics/schema/metrics_schema"
	"github.com/linkedin/goavro"
	. "github.com/smartystreets/goconvey/convey"
)

var (
	testDatapoint = record{
		Name: "dcos.metrics.Datapoint",
		Fields: []field{
			{
				Name:  "test-name",
				Datum: "name-field-test",
			},
			{
				Name:  "test-value",
				Datum: "value-field-test",
			},
			{
				Name:  "test-unit",
				Datum: "unit-field-test",
			},
		},
	}
	testTag = record{
		Name: "dcos.metrics.Tag",
		Fields: []field{
			{
				Name:  "test-tag-name",
				Datum: "tag-name-field-test",
			},
			{
				Name:  "test-tag-value",
				Datum: "tag-value-field-test",
			},
		},
	}
)

func TestExtract(t *testing.T) {
	Convey("When extracting a datapoint from an Avro record", t, func() {
		avroDatapoint := avroRecord{testDatapoint}
		pmmTest := producers.MetricsMessage{}
		err := avroDatapoint.extract(&pmmTest)

		Convey("Should extract the datapoint without errors", func() {
			So(err, ShouldBeNil)
			So(len(pmmTest.Datapoints), ShouldEqual, 1)
		})

		Convey("Should return the expected name and values from the datapoint", func() {
			So(pmmTest.Datapoints[0].Name, ShouldEqual, "name-field-test")
			So(pmmTest.Datapoints[0].Value, ShouldEqual, "value-field-test")
			So(pmmTest.Datapoints[0].Unit, ShouldEqual, "unit-field-test")
		})
	})

	Convey("When extracting tags from an Avro record", t, func() {
		avroDatapoint := avroRecord{testTag}
		pmmTest := producers.MetricsMessage{
			Dimensions: producers.Dimensions{
				Labels: make(map[string]string),
			},
		}

		Convey("Should extract the tag without errors", func() {
			err := avroDatapoint.extract(&pmmTest)
			value, ok := pmmTest.Dimensions.Labels["tag-name-field-test"]

			So(err, ShouldBeNil)
			So(ok, ShouldBeTrue)
			So(value, ShouldEqual, "tag-value-field-test")
		})
	})
}

func TestCreateObjectFromRecord(t *testing.T) {
	// Create a test record
	var (
		metricListNamespace = goavro.RecordEnclosingNamespace(metrics_schema.MetricListNamespace)
		metricListSchema    = goavro.RecordSchema(metrics_schema.MetricListSchema)
		datapointNamespace  = goavro.RecordEnclosingNamespace(metrics_schema.DatapointNamespace)
		datapointSchema     = goavro.RecordSchema(metrics_schema.DatapointSchema)
		tagNamespace        = goavro.RecordEnclosingNamespace(metrics_schema.TagNamespace)
		tagSchema           = goavro.RecordSchema(metrics_schema.TagSchema)
	)

	recDps, err := goavro.NewRecord(datapointNamespace, datapointSchema)
	if err != nil {
		panic(err)
	}
	recDps.Set("name", "some-name")
	recDps.Set("time_ms", 1000)
	recDps.Set("value", 42.0)

	recTags, err := goavro.NewRecord(tagNamespace, tagSchema)
	if err != nil {
		panic(err)
	}
	recTags.Set("key", "some-key")
	recTags.Set("value", "some-val")

	rec, err := goavro.NewRecord(metricListNamespace, metricListSchema)
	if err != nil {
		panic(err)
	}
	rec.Set("topic", "some-topic")
	rec.Set("tags", []interface{}{recTags})
	rec.Set("datapoints", []interface{}{recDps})

	// Run the test
	Convey("When creating an avroRecord object from an actual Avro record", t, func() {
		Convey("Should return the provided data in the expected structure without errors", func() {
			ar := avroRecord{}
			tags, err := rec.Get("tags")
			if err != nil {
				panic(err)
			}

			err = ar.createObjectFromRecord(tags)
			So(err, ShouldBeNil)
			So(ar, ShouldResemble, avroRecord{
				record{
					Name: "dcos.metrics.Tag",
					Fields: []field{
						field{
							Name:  "dcos.metrics.key",
							Datum: "some-key",
						},
						field{
							Name:  "dcos.metrics.value",
							Datum: "some-val",
						},
					},
				},
			})
		})

		Convey("Should return an error if the transformation failed", func() {
			type badData struct {
				SomeKey string
				SomeVal string
			}
			bd := badData{
				SomeKey: "foo-key",
				SomeVal: "foo-val",
			}
			ar := avroRecord{}
			err := ar.createObjectFromRecord(bd)
			So(err, ShouldNotBeNil)
		})
	})
}
<|endoftext|>


<commit_before>package gstruct

import (
	"errors"
	"fmt"
	"reflect"
	"runtime/debug"
	"strings"

	"github.com/onsi/gomega/format"
	errorsutil "github.com/onsi/gomega/gstruct/errors"
	"github.com/onsi/gomega/types"
)

//MatchAllFields succeeds if every field of a struct matches the field matcher associated with
//it, and every element matcher is matched.
//    actual := struct{
//      A int
//      B []bool
//      C string
//    }{
//      A: 5,
//      B: []bool{true, false},
//      C: "foo",
//    }
//
//    Expect(actual).To(MatchAllFields(Fields{
//      "A": Equal(5),
//      "B": ConsistOf(true, false),
//      "C": Equal("foo"),
//    }))
func MatchAllFields(fields Fields) types.GomegaMatcher {
	return &FieldsMatcher{
		Fields: fields,
	}
}

//MatchFields succeeds if each element of a struct matches the field matcher associated with
//it. It can ignore extra fields and/or missing fields.
//    actual := struct{
//      A int
//      B []bool
//      C string
//    }{
//      A: 5,
//      B: []bool{true, false},
//      C: "foo",
//    }
//
//    Expect(actual).To(MatchFields(IgnoreExtras, Fields{
//      "A": Equal(5),
//      "B": ConsistOf(true, false),
//    }))
//    Expect(actual).To(MatchFields(IgnoreMissing, Fields{
//      "A": Equal(5),
//      "B": ConsistOf(true, false),
//      "C": Equal("foo"),
//      "D": Equal("extra"),
//    }))
func MatchFields(options Options, fields Fields) types.GomegaMatcher {
	return &FieldsMatcher{
		Fields:        fields,
		IgnoreExtras:  options&IgnoreExtras != 0,
		IgnoreMissing: options&IgnoreMissing != 0,
	}
}

type FieldsMatcher struct {
	// Matchers for each field.
	Fields Fields

	// Whether to ignore extra elements or consider it an error.
	IgnoreExtras bool
	// Whether to ignore missing elements or consider it an error.
	IgnoreMissing bool

	// State.
	failures []error
}

// Field name to matcher.
type Fields map[string]types.GomegaMatcher

func (m *FieldsMatcher) Match(actual interface{}) (success bool, err error) {
	if reflect.TypeOf(actual).Kind() != reflect.Struct {
		return false, fmt.Errorf("%v is type %T, expected struct", actual, actual)
	}

	m.failures = m.matchFields(actual)
	if len(m.failures) > 0 {
		return false, nil
	}
	return true, nil
}

func (m *FieldsMatcher) matchFields(actual interface{}) (errs []error) {
	val := reflect.ValueOf(actual)
	typ := val.Type()
	fields := map[string]bool{}
	for i := 0; i < val.NumField(); i++ {
		fieldName := typ.Field(i).Name
		fields[fieldName] = true

		err := func() (err error) {
			// This test relies heavily on reflect, which tends to panic.
			// Recover here to provide more useful error messages in that case.
			defer func() {
				if r := recover(); r != nil {
					err = fmt.Errorf("panic checking %+v: %v\n%s", actual, r, debug.Stack())
				}
			}()

			matcher, expected := m.Fields[fieldName]
			if !expected {
				if !m.IgnoreExtras {
					return fmt.Errorf("unexpected field %s: %+v", fieldName, actual)
				}
				return nil
			}

			var field interface{}
			if val.Field(i).IsValid() {
				field = val.Field(i).Interface()
			} else {
				field = reflect.Zero(typ.Field(i).Type)
			}

			match, err := matcher.Match(field)
			if err != nil {
				return err
			} else if !match {
				if nesting, ok := matcher.(errorsutil.NestingMatcher); ok {
					return errorsutil.AggregateError(nesting.Failures())
				}
				return errors.New(matcher.FailureMessage(field))
			}
			return nil
		}()
		if err != nil {
			errs = append(errs, errorsutil.Nest("."+fieldName, err))
		}
	}

	for field := range m.Fields {
		if !fields[field] && !m.IgnoreMissing {
			errs = append(errs, fmt.Errorf("missing expected field %s", field))
		}
	}

	return errs
}

func (m *FieldsMatcher) FailureMessage(actual interface{}) (message string) {
	failures := make([]string, len(m.failures))
	for i := range m.failures {
		failures[i] = m.failures[i].Error()
	}
	return format.Message(reflect.TypeOf(actual).Name(),
		fmt.Sprintf("to match fields: {\n%v\n}\n", strings.Join(failures, "\n")))
}

func (m *FieldsMatcher) NegatedFailureMessage(actual interface{}) (message string) {
	return format.Message(actual, "not to match fields")
}

func (m *FieldsMatcher) Failures() []error {
	return m.failures
}
<commit_msg>remove redundant validity check<commit_after>package gstruct

import (
	"errors"
	"fmt"
	"reflect"
	"runtime/debug"
	"strings"

	"github.com/onsi/gomega/format"
	errorsutil "github.com/onsi/gomega/gstruct/errors"
	"github.com/onsi/gomega/types"
)

//MatchAllFields succeeds if every field of a struct matches the field matcher associated with
//it, and every element matcher is matched.
//    actual := struct{
//      A int
//      B []bool
//      C string
//    }{
//      A: 5,
//      B: []bool{true, false},
//      C: "foo",
//    }
//
//    Expect(actual).To(MatchAllFields(Fields{
//      "A": Equal(5),
//      "B": ConsistOf(true, false),
//      "C": Equal("foo"),
//    }))
func MatchAllFields(fields Fields) types.GomegaMatcher {
	return &FieldsMatcher{
		Fields: fields,
	}
}

//MatchFields succeeds if each element of a struct matches the field matcher associated with
//it. It can ignore extra fields and/or missing fields.
//    actual := struct{
//      A int
//      B []bool
//      C string
//    }{
//      A: 5,
//      B: []bool{true, false},
//      C: "foo",
//    }
//
//    Expect(actual).To(MatchFields(IgnoreExtras, Fields{
//      "A": Equal(5),
//      "B": ConsistOf(true, false),
//    }))
//    Expect(actual).To(MatchFields(IgnoreMissing, Fields{
//      "A": Equal(5),
//      "B": ConsistOf(true, false),
//      "C": Equal("foo"),
//      "D": Equal("extra"),
//    }))
func MatchFields(options Options, fields Fields) types.GomegaMatcher {
	return &FieldsMatcher{
		Fields:        fields,
		IgnoreExtras:  options&IgnoreExtras != 0,
		IgnoreMissing: options&IgnoreMissing != 0,
	}
}

type FieldsMatcher struct {
	// Matchers for each field.
	Fields Fields

	// Whether to ignore extra elements or consider it an error.
	IgnoreExtras bool
	// Whether to ignore missing elements or consider it an error.
	IgnoreMissing bool

	// State.
	failures []error
}

// Field name to matcher.
type Fields map[string]types.GomegaMatcher

func (m *FieldsMatcher) Match(actual interface{}) (success bool, err error) {
	if reflect.TypeOf(actual).Kind() != reflect.Struct {
		return false, fmt.Errorf("%v is type %T, expected struct", actual, actual)
	}

	m.failures = m.matchFields(actual)
	if len(m.failures) > 0 {
		return false, nil
	}
	return true, nil
}

func (m *FieldsMatcher) matchFields(actual interface{}) (errs []error) {
	val := reflect.ValueOf(actual)
	typ := val.Type()
	fields := map[string]bool{}
	for i := 0; i < val.NumField(); i++ {
		fieldName := typ.Field(i).Name
		fields[fieldName] = true

		err := func() (err error) {
			// This test relies heavily on reflect, which tends to panic.
			// Recover here to provide more useful error messages in that case.
			defer func() {
				if r := recover(); r != nil {
					err = fmt.Errorf("panic checking %+v: %v\n%s", actual, r, debug.Stack())
				}
			}()

			matcher, expected := m.Fields[fieldName]
			if !expected {
				if !m.IgnoreExtras {
					return fmt.Errorf("unexpected field %s: %+v", fieldName, actual)
				}
				return nil
			}

			field := val.Field(i).Interface()

			match, err := matcher.Match(field)
			if err != nil {
				return err
			} else if !match {
				if nesting, ok := matcher.(errorsutil.NestingMatcher); ok {
					return errorsutil.AggregateError(nesting.Failures())
				}
				return errors.New(matcher.FailureMessage(field))
			}
			return nil
		}()
		if err != nil {
			errs = append(errs, errorsutil.Nest("."+fieldName, err))
		}
	}

	for field := range m.Fields {
		if !fields[field] && !m.IgnoreMissing {
			errs = append(errs, fmt.Errorf("missing expected field %s", field))
		}
	}

	return errs
}

func (m *FieldsMatcher) FailureMessage(actual interface{}) (message string) {
	failures := make([]string, len(m.failures))
	for i := range m.failures {
		failures[i] = m.failures[i].Error()
	}
	return format.Message(reflect.TypeOf(actual).Name(),
		fmt.Sprintf("to match fields: {\n%v\n}\n", strings.Join(failures, "\n")))
}

func (m *FieldsMatcher) NegatedFailureMessage(actual interface{}) (message string) {
	return format.Message(actual, "not to match fields")
}

func (m *FieldsMatcher) Failures() []error {
	return m.failures
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/APTrust/easy-store/bagit"
	"github.com/APTrust/easy-store/util/fileutil"
	"io"
	"os"
	"path/filepath"
	"strings"
	"time"
)

func main() {
	job := loadJob()
	bagPath, err := createBag(job)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	} else {
		fmt.Println("Created", bagPath)
	}
}

func createBag(job *bagit.Job) (string, error) {
	bagPath := filepath.Join(job.BaggingDirectory, job.BagName)
	if job.BagItProfile.MustBeTarred() && !strings.HasSuffix(bagPath, ".tar") {
		bagPath += ".tar"
	}

	bagger, err := bagit.NewBagger(bagPath, job.BagItProfile)
	if err != nil {
		return "", err
	}

	// Add files
	for _, fpath := range job.Files {
		if fileutil.IsFile(fpath) {
			addFile(bagger, job, fpath)
		} else if fileutil.IsDir(fpath) {
			err := filepath.Walk(fpath, func(filePath string, f os.FileInfo, err error) error {
				var e error
				if f != nil && f.Mode().IsRegular() {
					e = addFile(bagger, job, filePath)
				}
				return e
			})
			if err != nil {
				fmt.Fprintln(os.Stderr, err.Error())
				os.Exit(1)
			}
		}
	}

	// Add tags
	for _, tagDef := range job.BagItProfile.RequiredTags {
		kvp := bagit.NewKeyValuePair(tagDef.TagName, tagDef.UserValue)
		if tagDef.TagFile == "bag-info.txt" {
			if tagDef.TagName == "Bagging-Date" {
				kvp.Value = time.Now().Format("2006-01-02")
			} else if tagDef.TagName == "Payload-Oxum" {
				kvp.Value = bagger.GetPayloadOxum()
			}
		}
		bagger.AddTag(tagDef.TagFile, &kvp)
	}

	// Write bag
	fmt.Println("Writing bag to", bagPath)
	if job.BagItProfile.MustBeTarred() {
		bagger.WriteBagToTarFile(true, true)
	} else {
		bagger.WriteBag(true, true)
	}

	errors := bagger.Errors()
	for _, errMsg := range errors {
		fmt.Fprintln(os.Stderr, errMsg)
	}
	if len(errors) > 0 {
		os.Exit(1)
	}

	// Validate bag
	bag := bagit.NewBag(bagPath)
	validator := bagit.NewValidator(bag, job.BagItProfile)
	fmt.Println("Validating bag at", bagPath)
	if !validator.Validate() {
		fmt.Fprintln(os.Stderr, "Bag failed validation with the following errors:")
		for _, errMsg := range validator.Errors() {
			fmt.Fprintln(os.Stderr, errMsg)
		}
		os.Exit(1)
	} else {
		fmt.Println("Bag at", bagPath, "is valid")
	}

	return bagPath, nil
}

func addFile(bagger *bagit.Bagger, job *bagit.Job, sourcePath string) error {
	if job.ShouldIncludeFile(sourcePath) {
		relPath := "data" + sourcePath
		fmt.Println("Adding", sourcePath)
		if !bagger.AddFile(sourcePath, relPath) {
			errors := bagger.Errors()
			lastError := errors[len(errors)-1]
			return fmt.Errorf(lastError)
		}
	} else {
		fmt.Println("Skipping", sourcePath)
	}
	return nil
}

func loadJob() *bagit.Job {
	var stdin bool
	flag.BoolVar(&stdin, "stdin", false, "Load job from stdin instead of reading from file.")
	flag.Parse()
	if stdin {
		job, err := loadJobFromStdin()
		if err != nil {
			fmt.Fprintln(os.Stderr, err.Error())
			os.Exit(1)
		}
		return job
	}

	if len(os.Args) < 2 {
		fmt.Fprintln(os.Stderr, "You must specify a job file, or pass the job JSON in through STDIN.")
		os.Exit(1)
	}
	job, err := bagit.LoadJobFromFile(os.Args[1])
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	}
	errors := job.Validate()
	if len(errors) > 0 {
		for _, err := range errors {
			fmt.Fprintln(os.Stderr, err.Error())
		}
		os.Exit(1)
	}
	return job
}

func loadJobFromStdin() (*bagit.Job, error) {
	jsonBytes := make([]byte, 0)
	data := make([]byte, 4096)
	reader := bufio.NewReader(os.Stdin)
	for {
		bytesRead, err := reader.Read(data)
		jsonBytes = append(jsonBytes, data[0:bytesRead]...)
		if err != nil {
			if err == io.EOF {
				break
			}
			fmt.Fprintln(os.Stderr, err.Error())
			os.Exit(1)
		}
		if bytesRead < len(data) {
			break
		}
	}
	job := &bagit.Job{}
	err := json.Unmarshal(jsonBytes, job)
	return job, err
}
<commit_msg>Bagger now ignores tag if name and value are blank<commit_after>package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/APTrust/easy-store/bagit"
	"github.com/APTrust/easy-store/util/fileutil"
	"io"
	"os"
	"path/filepath"
	"strings"
	"time"
)

func main() {
	job := loadJob()
	bagPath, err := createBag(job)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	} else {
		fmt.Println("Created", bagPath)
	}
}

func createBag(job *bagit.Job) (string, error) {
	bagPath := filepath.Join(job.BaggingDirectory, job.BagName)
	if job.BagItProfile.MustBeTarred() && !strings.HasSuffix(bagPath, ".tar") {
		bagPath += ".tar"
	}

	bagger, err := bagit.NewBagger(bagPath, job.BagItProfile)
	if err != nil {
		return "", err
	}

	// Add files
	for _, fpath := range job.Files {
		if fileutil.IsFile(fpath) {
			addFile(bagger, job, fpath)
		} else if fileutil.IsDir(fpath) {
			err := filepath.Walk(fpath, func(filePath string, f os.FileInfo, err error) error {
				var e error
				if f != nil && f.Mode().IsRegular() {
					e = addFile(bagger, job, filePath)
				}
				return e
			})
			if err != nil {
				fmt.Fprintln(os.Stderr, err.Error())
				os.Exit(1)
			}
		}
	}

	// Add tags
	for _, tagDef := range job.BagItProfile.RequiredTags {
		// Empty tags can come from Easy Store UI for custom tag files.
		if tagDef.TagName == "" && tagDef.UserValue == "" {
			continue
		}
		kvp := bagit.NewKeyValuePair(tagDef.TagName, tagDef.UserValue)
		if tagDef.TagFile == "bag-info.txt" {
			if tagDef.TagName == "Bagging-Date" {
				kvp.Value = time.Now().Format("2006-01-02")
			} else if tagDef.TagName == "Payload-Oxum" {
				kvp.Value = bagger.GetPayloadOxum()
			}
		}
		bagger.AddTag(tagDef.TagFile, &kvp)
	}

	// Write bag
	fmt.Println("Writing bag to", bagPath)
	if job.BagItProfile.MustBeTarred() {
		bagger.WriteBagToTarFile(true, true)
	} else {
		bagger.WriteBag(true, true)
	}

	errors := bagger.Errors()
	for _, errMsg := range errors {
		fmt.Fprintln(os.Stderr, errMsg)
	}
	if len(errors) > 0 {
		os.Exit(1)
	}

	// Validate bag
	bag := bagit.NewBag(bagPath)
	validator := bagit.NewValidator(bag, job.BagItProfile)
	fmt.Println("Validating bag at", bagPath)
	if !validator.Validate() {
		fmt.Fprintln(os.Stderr, "Bag failed validation with the following errors:")
		for _, errMsg := range validator.Errors() {
			fmt.Fprintln(os.Stderr, errMsg)
		}
		os.Exit(1)
	} else {
		fmt.Println("Bag at", bagPath, "is valid")
	}

	return bagPath, nil
}

func addFile(bagger *bagit.Bagger, job *bagit.Job, sourcePath string) error {
	if job.ShouldIncludeFile(sourcePath) {
		relPath := "data" + sourcePath
		fmt.Println("Adding", sourcePath)
		if !bagger.AddFile(sourcePath, relPath) {
			errors := bagger.Errors()
			lastError := errors[len(errors)-1]
			return fmt.Errorf(lastError)
		}
	} else {
		fmt.Println("Skipping", sourcePath)
	}
	return nil
}

func loadJob() *bagit.Job {
	var stdin bool
	flag.BoolVar(&stdin, "stdin", false, "Load job from stdin instead of reading from file.")
	flag.Parse()
	if stdin {
		job, err := loadJobFromStdin()
		if err != nil {
			fmt.Fprintln(os.Stderr, err.Error())
			os.Exit(1)
		}
		return job
	}

	if len(os.Args) < 2 {
		fmt.Fprintln(os.Stderr, "You must specify a job file, or pass the job JSON in through STDIN.")
		os.Exit(1)
	}
	job, err := bagit.LoadJobFromFile(os.Args[1])
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	}
	errors := job.Validate()
	if len(errors) > 0 {
		for _, err := range errors {
			fmt.Fprintln(os.Stderr, err.Error())
		}
		os.Exit(1)
	}
	return job
}

func loadJobFromStdin() (*bagit.Job, error) {
	jsonBytes := make([]byte, 0)
	data := make([]byte, 4096)
	reader := bufio.NewReader(os.Stdin)
	for {
		bytesRead, err := reader.Read(data)
		jsonBytes = append(jsonBytes, data[0:bytesRead]...)
		if err != nil {
			if err == io.EOF {
				break
			}
			fmt.Fprintln(os.Stderr, err.Error())
			os.Exit(1)
		}
		if bytesRead < len(data) {
			break
		}
	}
	job := &bagit.Job{}
	err := json.Unmarshal(jsonBytes, job)
	return job, err
}
<|endoftext|>


<commit_before>package gxutil

import (
	"encoding/json"
	"io/ioutil"
)

type LockDep struct {
	// full ipfs path /ipfs/<hash>/foo
	Ref string

	// Mapping of dvcs import paths to LockData
	Deps map[string]LockDep
}

type LockFile struct {
	Language string
	Deps     map[string]LockDep
}

func LoadLockFile(lck *LockFile, fname string) error {
	data, err := ioutil.ReadFile(fname)
	if err != nil {
		return err
	}

	return json.Unmarshal(data, lck)
}
<commit_msg>add lockfile versioning<commit_after>package gxutil

import (
	"encoding/json"
	"fmt"
	"os"
)

const LockVersion = 1

type LockDep struct {
	// full ipfs path /ipfs/<hash>/foo
	Ref string `json:"ref"`

	// Mapping of dvcs import paths to LockData
	Deps map[string]LockDep `json:"deps"`
}

type LockFile struct {
	Language    string             `json:"language"`
	LockVersion int                `json:"lockVersion"`
	Deps        map[string]LockDep `json:"deps"`
}

func LoadLockFile(lck *LockFile, fname string) error {
	fi, err := os.Open(fname)
	if err != nil {
		return err
	}

	if err := json.NewDecoder(fi).Decode(lck); err != nil {
		return err
	}

	if lck.LockVersion != LockVersion {
		return fmt.Errorf("unsupported lockfile version: %d", lck.LockVersion)
	}

	return nil
}
<|endoftext|>


<commit_before>package spec

import (
	tp "athena/src/athena/proto"
	. "fmt"
	"fmt"
	xmlhelp "gokogiri/help"
	l4g "log4go"
	"os"
	. "path/filepath"
	"runtime/debug"
	. "tritium/src/tritium"
	"tritium/src/tritium/lamprey"
	"tritium/src/tritium/packager"
	"tritium/src/tritium/whale"
)

func All(command string, directory string, options ...string) {

	var mixerPath string
	if len(options) == 1 {
		mixerPath = options[0]
	}

	logger := make(l4g.Logger)
	logger.AddFilter("test", l4g.ERROR, l4g.NewConsoleLogWriter())
	l4g.Global = logger
	var eng Engine
	if command == "test" {
		eng = whale.NewEngine(logger)
	} else if command == "debug" {
		eng = lamprey.NewEngine(logger)
	}

	var pkg *tp.Package

	if len(mixerPath) > 0 {
		// Used when testing in ambrosia
		mixer := tp.OpenMixer(mixerPath)
		pkg = mixer.Package
	} else {
		bigPackage := packager.BuildDefaultPackage()
		pkg = bigPackage.Package
	}

	globalResult := NewResult()
	globalResult.all(directory, pkg, eng, logger)

	logger.AddFilter("stdout", l4g.ERROR, l4g.NewConsoleLogWriter())

	// TODO : Walk over the results here and print errors.

	var foundError = false

	for _, err := range globalResult.Errors {
		foundError = true
		println("\n=========================================", err.Location, "\n")
		if err.Panic {
			Printf(err.Message)
		} else {
			Printf("\n==========\n%v :: %v \n\n Got \n----------\n%v\n\n Expected \n----------\n%v\n", err.Name, err.Message, err.Got, err.Expected)
		}
	}
	println("\n\n")
	println("+++TEST COMPLETE+++\n\n")

	if foundError {
		os.Exit(1)
	}
	eng.Free()
	xmlhelp.LibxmlCleanUpParser()
	if xmlhelp.LibxmlGetMemoryAllocation() != 0 {
		fmt.Printf("Memeory leaks %d!!!", xmlhelp.LibxmlGetMemoryAllocation())
		xmlhelp.LibxmlReportMemoryLeak()
	}
}

func (result *Result) all(directory string, pkg *tp.Package, eng Engine, logger l4g.Logger) {
	_, err := Glob(Join(directory, "main.ts"))

	if err == nil {
		newResult := RunSpec(directory, pkg, eng, logger)
		result.Merge(newResult)
	}
	subdirs, _ := Glob(Join(directory, "*"))
	for _, subdir := range subdirs {
		result.all(subdir, pkg, eng, logger)
	}
}

func RunSpec(dir string, pkg *tp.Package, eng Engine, logger l4g.Logger) (result *Result) {
	result = NewResult()
	logWriter := NewTestLogWriter()
	logger["test"] = &l4g.Filter{l4g.WARNING, "test", logWriter}

	defer func() {
		//log.Println("done")  // Println executes normally even in there is a panic
		if x := recover(); x != nil {
			if err, ok := x.(error); ok {
				logger.Error(dir + " === " + err.Error() + "\n\n" + string(debug.Stack()))
			} else {
				logger.Error(dir + " === " + x.(string) + "\n\n" + string(debug.Stack()))
			}
		}
		for _, rec := range logWriter.Logs {
			//println("HAZ LOGS")
			err := l4g.FormatLogRecord("[%D %T] [%L] (%S) %M", rec)
			result.Error(dir, err)
		}
		print(result.CharStatus())
	}()
	spec, err := LoadSpec(dir, pkg)
	if err != nil {
		result.Error(dir, err.Error())
	} else {
		result.Merge(spec.Compare(eng.Run(spec.Script, spec.Input, spec.Vars)))
	}
	return
}
<commit_msg>Revert "Try this"<commit_after>package spec

import (
	tp "athena/src/athena/proto"
	. "fmt"
	"fmt"
	xmlhelp "gokogiri/help"
	l4g "log4go"
	"os"
	. "path/filepath"
	"runtime/debug"
	. "tritium/src/tritium"
	"tritium/src/tritium/lamprey"
	"tritium/src/tritium/packager"
	"tritium/src/tritium/whale"
)

func All(command string, directory string, options ...string) {

	var mixerPath string
	if len(options) == 1 {
		mixerPath = options[0]
	}

	logger := make(l4g.Logger)
	logger.AddFilter("test", l4g.ERROR, l4g.NewConsoleLogWriter())
	l4g.Global = logger
	var eng Engine
	if command == "test" {
		eng = whale.NewEngine(logger)
	} else if command == "debug" {
		eng = lamprey.NewEngine(logger)
	}

	var pkg *tp.Package

	if len(mixerPath) > 0 {
		// Used when testing in ambrosia
		mixer := tp.OpenMixer(mixerPath)
		pkg = mixer.Package
	} else {
		bigPackage := packager.BuildDefaultPackage()
		pkg = bigPackage.Package
	}

	globalResult := NewResult()
	globalResult.all(directory, pkg, eng, logger)

	logger.AddFilter("stdout", l4g.ERROR, l4g.NewConsoleLogWriter())

	// TODO : Walk over the results here and print errors.

	var foundError = false

	for _, err := range globalResult.Errors {
		foundError = true
		println("\n=========================================", err.Location, "\n")
		if err.Panic {
			Printf(err.Message)
		} else {
			Printf("\n==========\n%v :: %v \n\n Got \n----------\n%v\n\n Expected \n----------\n%v\n", err.Name, err.Message, err.Got, err.Expected)
		}
	}
	println("\n\n")
	println("+++TEST COMPLETE+++\n\n")

	if foundError {
		os.Exit(1)
	}
	eng.Free()
	xmlhelp.LibxmlCleanUpParser()
	if xmlhelp.LibxmlGetMemoryAllocation() != 0 {
		fmt.Printf("Memeory leaks %d!!!", xmlhelp.LibxmlGetMemoryAllocation())
		xmlhelp.LibxmlReportMemoryLeak()
	}
}

func (result *Result) all(directory string, pkg *tp.Package, eng Engine, logger l4g.Logger) {
	_, err := Glob(Join(directory, "main.ts"))

	if err == nil {
		newResult := RunSpec(directory, pkg, eng, logger)
		result.Merge(newResult)
	}
	subdirs, _ := Glob(Join(directory, "*"))
	for _, subdir := range subdirs {
		result.all(subdir, pkg, eng, logger)
	}
}

func RunSpec(dir string, pkg *tp.Package, eng Engine, logger l4g.Logger) (result *Result) {
	result = NewResult()
	logWriter := NewTestLogWriter()
	logger["test"] = &l4g.Filter{l4g.WARNING, "test", logWriter}

	defer func() {
		//log.Println("done")  // Println executes normally even in there is a panic
		if x := recover(); x != nil {
			err, ok := x.(error)
			if ok {
				logger.Error(dir + " === " + err.Error() + "\n\n" + string(debug.Stack()))
			} else {
				logger.Error(dir + " === " + x.(string) + "\n\n" + string(debug.Stack()))
			}
		}
		for _, rec := range logWriter.Logs {
			//println("HAZ LOGS")
			err := l4g.FormatLogRecord("[%D %T] [%L] (%S) %M", rec)
			result.Error(dir, err)
		}
		print(result.CharStatus())
	}()
	spec, err := LoadSpec(dir, pkg)
	if err != nil {
		result.Error(dir, err.Error())
	} else {
		result.Merge(spec.Compare(eng.Run(spec.Script, spec.Input, spec.Vars)))
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/koding/multiconfig"
)

type Config struct {
	Board string
}

var conf = loadConfig()

func loadConfig() *Config {
	m := multiconfig.New()
	conf := new(Config)
	m.MustLoad(conf)

	return conf
}

func main() {
	fmt.Println("Run ptt crawler with board - ", conf.Board)
}

func fetch(url string) *http.Response {
	resp, _ := http.Get(url)
	return resp
}<commit_msg>remove unused file<commit_after><|endoftext|>


<commit_before>package filer

import (
	"context"
	"fmt"
	"io"
	"math"
	"strings"
	"time"

	"github.com/golang/protobuf/proto"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/notification"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
)

func (f *Filer) NotifyUpdateEvent(ctx context.Context, oldEntry, newEntry *Entry, deleteChunks, isFromOtherCluster bool, signatures []int32) {
	var fullpath string
	if oldEntry != nil {
		fullpath = string(oldEntry.FullPath)
	} else if newEntry != nil {
		fullpath = string(newEntry.FullPath)
	} else {
		return
	}

	// println("fullpath:", fullpath)

	if strings.HasPrefix(fullpath, SystemLogDir) {
		return
	}
	foundSelf := false
	for _, sig := range signatures {
		if sig == f.Signature {
			foundSelf = true
		}
	}
	if !foundSelf {
		signatures = append(signatures, f.Signature)
	}

	newParentPath := ""
	if newEntry != nil {
		newParentPath, _ = newEntry.FullPath.DirAndName()
	}
	eventNotification := &filer_pb.EventNotification{
		OldEntry:           oldEntry.ToProtoEntry(),
		NewEntry:           newEntry.ToProtoEntry(),
		DeleteChunks:       deleteChunks,
		NewParentPath:      newParentPath,
		IsFromOtherCluster: isFromOtherCluster,
		Signatures:         signatures,
	}

	if notification.Queue != nil {
		glog.V(3).Infof("notifying entry update %v", fullpath)
		if err := notification.Queue.SendMessage(fullpath, eventNotification); err != nil {
			// throw message
			glog.Error(err)
		}
	}

	f.logMetaEvent(ctx, fullpath, eventNotification)

}

func (f *Filer) logMetaEvent(ctx context.Context, fullpath string, eventNotification *filer_pb.EventNotification) {

	dir, _ := util.FullPath(fullpath).DirAndName()

	event := &filer_pb.SubscribeMetadataResponse{
		Directory:         dir,
		EventNotification: eventNotification,
		TsNs:              time.Now().UnixNano(),
	}
	data, err := proto.Marshal(event)
	if err != nil {
		glog.Errorf("failed to marshal filer_pb.SubscribeMetadataResponse %+v: %v", event, err)
		return
	}

	f.LocalMetaLogBuffer.AddToBuffer([]byte(dir), data, event.TsNs)

}

func (f *Filer) logFlushFunc(startTime, stopTime time.Time, buf []byte) {

	if len(buf) == 0 {
		return
	}

	startTime, stopTime = startTime.UTC(), stopTime.UTC()

	targetFile := fmt.Sprintf("%s/%04d-%02d-%02d/%02d-%02d.%08x", SystemLogDir,
		startTime.Year(), startTime.Month(), startTime.Day(), startTime.Hour(), startTime.Minute(), f.UniqueFileId,
		// startTime.Second(), startTime.Nanosecond(),
	)

	for {
		if err := f.appendToFile(targetFile, buf); err != nil {
			glog.V(1).Infof("log write failed %s: %v", targetFile, err)
			time.Sleep(737 * time.Millisecond)
		} else {
			break
		}
	}
}

func (f *Filer) ReadPersistedLogBuffer(startTime time.Time, eachLogEntryFn func(logEntry *filer_pb.LogEntry) error) (lastTsNs int64, err error) {

	startTime = startTime.UTC()
	startDate := fmt.Sprintf("%04d-%02d-%02d", startTime.Year(), startTime.Month(), startTime.Day())
	startHourMinute := fmt.Sprintf("%02d-%02d", startTime.Hour(), startTime.Minute())

	sizeBuf := make([]byte, 4)
	startTsNs := startTime.UnixNano()

	dayEntries, _, listDayErr := f.ListDirectoryEntries(context.Background(), SystemLogDir, startDate, true, 366, "", "", "")
	if listDayErr != nil {
		return lastTsNs, fmt.Errorf("fail to list log by day: %v", listDayErr)
	}
	for _, dayEntry := range dayEntries {
		// println("checking day", dayEntry.FullPath)
		hourMinuteEntries, _, listHourMinuteErr := f.ListDirectoryEntries(context.Background(), util.NewFullPath(SystemLogDir, dayEntry.Name()), "", false, math.MaxInt32, "", "", "")
		if listHourMinuteErr != nil {
			return lastTsNs, fmt.Errorf("fail to list log %s by day: %v", dayEntry.Name(), listHourMinuteErr)
		}
		for _, hourMinuteEntry := range hourMinuteEntries {
			// println("checking hh-mm", hourMinuteEntry.FullPath)
			if dayEntry.Name() == startDate {
				hourMinute := util.FileNameBase(hourMinuteEntry.Name())
				if strings.Compare(hourMinute, startHourMinute) < 0 {
					continue
				}
			}
			// println("processing", hourMinuteEntry.FullPath)
			chunkedFileReader := NewChunkStreamReaderFromFiler(f.MasterClient, hourMinuteEntry.Chunks)
			if lastTsNs, err = ReadEachLogEntry(chunkedFileReader, sizeBuf, startTsNs, eachLogEntryFn); err != nil {
				chunkedFileReader.Close()
				if err == io.EOF {
					continue
				}
				return lastTsNs, fmt.Errorf("reading %s: %v", hourMinuteEntry.FullPath, err)
			}
			chunkedFileReader.Close()
		}
	}

	return lastTsNs, nil
}

func ReadEachLogEntry(r io.Reader, sizeBuf []byte, ns int64, eachLogEntryFn func(logEntry *filer_pb.LogEntry) error) (lastTsNs int64, err error) {
	for {
		n, err := r.Read(sizeBuf)
		if err != nil {
			return lastTsNs, err
		}
		if n != 4 {
			return lastTsNs, fmt.Errorf("size %d bytes, expected 4 bytes", n)
		}
		size := util.BytesToUint32(sizeBuf)
		// println("entry size", size)
		entryData := make([]byte, size)
		n, err = r.Read(entryData)
		if err != nil {
			return lastTsNs, err
		}
		if n != int(size) {
			return lastTsNs, fmt.Errorf("entry data %d bytes, expected %d bytes", n, size)
		}
		logEntry := &filer_pb.LogEntry{}
		if err = proto.Unmarshal(entryData, logEntry); err != nil {
			return lastTsNs, err
		}
		if logEntry.TsNs <= ns {
			continue
		}
		// println("each log: ", logEntry.TsNs)
		if err := eachLogEntryFn(logEntry); err != nil {
			return lastTsNs, err
		} else {
			lastTsNs = logEntry.TsNs
		}
	}
}
<commit_msg>adjust log level<commit_after>package filer

import (
	"context"
	"fmt"
	"io"
	"math"
	"strings"
	"time"

	"github.com/golang/protobuf/proto"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/notification"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
)

func (f *Filer) NotifyUpdateEvent(ctx context.Context, oldEntry, newEntry *Entry, deleteChunks, isFromOtherCluster bool, signatures []int32) {
	var fullpath string
	if oldEntry != nil {
		fullpath = string(oldEntry.FullPath)
	} else if newEntry != nil {
		fullpath = string(newEntry.FullPath)
	} else {
		return
	}

	// println("fullpath:", fullpath)

	if strings.HasPrefix(fullpath, SystemLogDir) {
		return
	}
	foundSelf := false
	for _, sig := range signatures {
		if sig == f.Signature {
			foundSelf = true
		}
	}
	if !foundSelf {
		signatures = append(signatures, f.Signature)
	}

	newParentPath := ""
	if newEntry != nil {
		newParentPath, _ = newEntry.FullPath.DirAndName()
	}
	eventNotification := &filer_pb.EventNotification{
		OldEntry:           oldEntry.ToProtoEntry(),
		NewEntry:           newEntry.ToProtoEntry(),
		DeleteChunks:       deleteChunks,
		NewParentPath:      newParentPath,
		IsFromOtherCluster: isFromOtherCluster,
		Signatures:         signatures,
	}

	if notification.Queue != nil {
		glog.V(3).Infof("notifying entry update %v", fullpath)
		if err := notification.Queue.SendMessage(fullpath, eventNotification); err != nil {
			// throw message
			glog.Error(err)
		}
	}

	f.logMetaEvent(ctx, fullpath, eventNotification)

}

func (f *Filer) logMetaEvent(ctx context.Context, fullpath string, eventNotification *filer_pb.EventNotification) {

	dir, _ := util.FullPath(fullpath).DirAndName()

	event := &filer_pb.SubscribeMetadataResponse{
		Directory:         dir,
		EventNotification: eventNotification,
		TsNs:              time.Now().UnixNano(),
	}
	data, err := proto.Marshal(event)
	if err != nil {
		glog.Errorf("failed to marshal filer_pb.SubscribeMetadataResponse %+v: %v", event, err)
		return
	}

	f.LocalMetaLogBuffer.AddToBuffer([]byte(dir), data, event.TsNs)

}

func (f *Filer) logFlushFunc(startTime, stopTime time.Time, buf []byte) {

	if len(buf) == 0 {
		return
	}

	startTime, stopTime = startTime.UTC(), stopTime.UTC()

	targetFile := fmt.Sprintf("%s/%04d-%02d-%02d/%02d-%02d.%08x", SystemLogDir,
		startTime.Year(), startTime.Month(), startTime.Day(), startTime.Hour(), startTime.Minute(), f.UniqueFileId,
		// startTime.Second(), startTime.Nanosecond(),
	)

	for {
		if err := f.appendToFile(targetFile, buf); err != nil {
			glog.V(0).Infof("metadata log write failed %s: %v", targetFile, err)
			time.Sleep(737 * time.Millisecond)
		} else {
			break
		}
	}
}

func (f *Filer) ReadPersistedLogBuffer(startTime time.Time, eachLogEntryFn func(logEntry *filer_pb.LogEntry) error) (lastTsNs int64, err error) {

	startTime = startTime.UTC()
	startDate := fmt.Sprintf("%04d-%02d-%02d", startTime.Year(), startTime.Month(), startTime.Day())
	startHourMinute := fmt.Sprintf("%02d-%02d", startTime.Hour(), startTime.Minute())

	sizeBuf := make([]byte, 4)
	startTsNs := startTime.UnixNano()

	dayEntries, _, listDayErr := f.ListDirectoryEntries(context.Background(), SystemLogDir, startDate, true, 366, "", "", "")
	if listDayErr != nil {
		return lastTsNs, fmt.Errorf("fail to list log by day: %v", listDayErr)
	}
	for _, dayEntry := range dayEntries {
		// println("checking day", dayEntry.FullPath)
		hourMinuteEntries, _, listHourMinuteErr := f.ListDirectoryEntries(context.Background(), util.NewFullPath(SystemLogDir, dayEntry.Name()), "", false, math.MaxInt32, "", "", "")
		if listHourMinuteErr != nil {
			return lastTsNs, fmt.Errorf("fail to list log %s by day: %v", dayEntry.Name(), listHourMinuteErr)
		}
		for _, hourMinuteEntry := range hourMinuteEntries {
			// println("checking hh-mm", hourMinuteEntry.FullPath)
			if dayEntry.Name() == startDate {
				hourMinute := util.FileNameBase(hourMinuteEntry.Name())
				if strings.Compare(hourMinute, startHourMinute) < 0 {
					continue
				}
			}
			// println("processing", hourMinuteEntry.FullPath)
			chunkedFileReader := NewChunkStreamReaderFromFiler(f.MasterClient, hourMinuteEntry.Chunks)
			if lastTsNs, err = ReadEachLogEntry(chunkedFileReader, sizeBuf, startTsNs, eachLogEntryFn); err != nil {
				chunkedFileReader.Close()
				if err == io.EOF {
					continue
				}
				return lastTsNs, fmt.Errorf("reading %s: %v", hourMinuteEntry.FullPath, err)
			}
			chunkedFileReader.Close()
		}
	}

	return lastTsNs, nil
}

func ReadEachLogEntry(r io.Reader, sizeBuf []byte, ns int64, eachLogEntryFn func(logEntry *filer_pb.LogEntry) error) (lastTsNs int64, err error) {
	for {
		n, err := r.Read(sizeBuf)
		if err != nil {
			return lastTsNs, err
		}
		if n != 4 {
			return lastTsNs, fmt.Errorf("size %d bytes, expected 4 bytes", n)
		}
		size := util.BytesToUint32(sizeBuf)
		// println("entry size", size)
		entryData := make([]byte, size)
		n, err = r.Read(entryData)
		if err != nil {
			return lastTsNs, err
		}
		if n != int(size) {
			return lastTsNs, fmt.Errorf("entry data %d bytes, expected %d bytes", n, size)
		}
		logEntry := &filer_pb.LogEntry{}
		if err = proto.Unmarshal(entryData, logEntry); err != nil {
			return lastTsNs, err
		}
		if logEntry.TsNs <= ns {
			continue
		}
		// println("each log: ", logEntry.TsNs)
		if err := eachLogEntryFn(logEntry); err != nil {
			return lastTsNs, err
		} else {
			lastTsNs = logEntry.TsNs
		}
	}
}
<|endoftext|>