<commit_before>/*
 * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2017 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package minio

import "io"
import "encoding/hex"

// Inherits Client and adds new methods to expose the low level S3 APIs.
type Core struct {
	*Client
}

// NewCoreClient - Returns new Core.
func NewCore(endpoint string, accessKeyID, secretAccessKey string, secure bool) (*Core, error) {
	var s3Client Core
	client, err := NewV4(endpoint, accessKeyID, secretAccessKey, secure)
	if err != nil {
		return nil, err
	}
	s3Client.Client = client
	return &s3Client, nil
}

// ListObjects - List the objects.
func (c Core) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListBucketResult, err error) {
	return c.listObjectsQuery(bucket, prefix, marker, delimiter, maxKeys)
}

// PutObject - Upload object. Uploads using single PUT call.
func (c Core) PutObject(bucket, object string, size int64, data io.Reader, md5Sum string, sha256Sum string, metadata map[string][]string) (ObjectInfo, error) {
	var md5SumBytes []byte
	var sha256SumBytes []byte
	var err error
	if md5Sum != "" {
		md5SumBytes, err = hex.DecodeString(md5Sum)
		if err != nil {
			return ObjectInfo{}, err
		}
	}
	if sha256Sum != "" {
		sha256SumBytes, err = hex.DecodeString(sha256Sum)
		if err != nil {
			return ObjectInfo{}, err
		}
	}
	return c.putObjectDo(bucket, object, data, md5SumBytes, sha256SumBytes, size, metadata)
}

// NewMultipartUpload - Initiates new multipart upload and returns the new uploaID.
func (c Core) NewMultipartUpload(bucket, object string, metadata map[string][]string) (uploadID string, err error) {
	result, err := c.initiateMultipartUpload(bucket, object, metadata)
	return result.UploadID, err
}

// ListMultipartUploads - List incomplete uploads.
func (c Core) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartUploadsResult, err error) {
	return c.listMultipartUploadsQuery(bucket, keyMarker, uploadIDMarker, prefix, delimiter, maxUploads)
}

// PutObjectPart - Upload an object part.
func (c Core) PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Sum, sha256Sum string) (ObjectPart, error) {
	var md5SumBytes []byte
	var sha256SumBytes []byte
	var err error
	if md5Sum != "" {
		md5SumBytes, err = hex.DecodeString(md5Sum)
		if err != nil {
			return ObjectPart{}, err
		}
	}
	if sha256Sum != "" {
		sha256SumBytes, err = hex.DecodeString(sha256Sum)
		if err != nil {
			return ObjectPart{}, err
		}
	}
	return c.uploadPart(bucket, object, uploadID, data, partID, md5SumBytes, sha256SumBytes, size)
}

// ListObjectParts - List uploaded parts of an incomplete upload.
func (c Core) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (result ListObjectPartsResult, err error) {
	return c.listObjectPartsQuery(bucket, object, uploadID, partNumberMarker, maxParts)
}

// CompleteMultipartUpload - Concatenate uploaded parts and commit to an object.
func (c Core) CompleteMultipartUpload(bucket, object, uploadID string, parts []CompletePart) error {
	_, err := c.completeMultipartUpload(bucket, object, uploadID, completeMultipartUpload{Parts: parts})
	return err
}

// AbortMultipartUpload - Abort an incomplete upload.
func (c Core) AbortMultipartUpload(bucket, object, uploadID string) error {
	return c.abortMultipartUpload(bucket, object, uploadID)
}
<commit_msg>Take md5 and sha256 arguments as []byte (#637)<commit_after>/*
 * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2017 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package minio

import "io"

// Inherits Client and adds new methods to expose the low level S3 APIs.
type Core struct {
	*Client
}

// NewCoreClient - Returns new Core.
func NewCore(endpoint string, accessKeyID, secretAccessKey string, secure bool) (*Core, error) {
	var s3Client Core
	client, err := NewV4(endpoint, accessKeyID, secretAccessKey, secure)
	if err != nil {
		return nil, err
	}
	s3Client.Client = client
	return &s3Client, nil
}

// ListObjects - List the objects.
func (c Core) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListBucketResult, err error) {
	return c.listObjectsQuery(bucket, prefix, marker, delimiter, maxKeys)
}

// PutObject - Upload object. Uploads using single PUT call.
func (c Core) PutObject(bucket, object string, size int64, data io.Reader, md5Sum, sha256Sum []byte, metadata map[string][]string) (ObjectInfo, error) {
	return c.putObjectDo(bucket, object, data, md5Sum, sha256Sum, size, metadata)
}

// NewMultipartUpload - Initiates new multipart upload and returns the new uploaID.
func (c Core) NewMultipartUpload(bucket, object string, metadata map[string][]string) (uploadID string, err error) {
	result, err := c.initiateMultipartUpload(bucket, object, metadata)
	return result.UploadID, err
}

// ListMultipartUploads - List incomplete uploads.
func (c Core) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartUploadsResult, err error) {
	return c.listMultipartUploadsQuery(bucket, keyMarker, uploadIDMarker, prefix, delimiter, maxUploads)
}

// PutObjectPart - Upload an object part.
func (c Core) PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Sum, sha256Sum []byte) (ObjectPart, error) {
	return c.uploadPart(bucket, object, uploadID, data, partID, md5Sum, sha256Sum, size)
}

// ListObjectParts - List uploaded parts of an incomplete upload.
func (c Core) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (result ListObjectPartsResult, err error) {
	return c.listObjectPartsQuery(bucket, object, uploadID, partNumberMarker, maxParts)
}

// CompleteMultipartUpload - Concatenate uploaded parts and commit to an object.
func (c Core) CompleteMultipartUpload(bucket, object, uploadID string, parts []CompletePart) error {
	_, err := c.completeMultipartUpload(bucket, object, uploadID, completeMultipartUpload{Parts: parts})
	return err
}

// AbortMultipartUpload - Abort an incomplete upload.
func (c Core) AbortMultipartUpload(bucket, object, uploadID string) error {
	return c.abortMultipartUpload(bucket, object, uploadID)
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"net"
	"net/http"
	_ "net/http/pprof" // nolint
	"os"
	"time"

	"github.com/getsentry/sentry-go"
	"github.com/projecteru2/core/auth"
	"github.com/projecteru2/core/cluster/calcium"
	"github.com/projecteru2/core/log"
	"github.com/projecteru2/core/metrics"
	"github.com/projecteru2/core/rpc"
	pb "github.com/projecteru2/core/rpc/gen"
	"github.com/projecteru2/core/utils"
	"github.com/projecteru2/core/version"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/sethvargo/go-signalcontext"
	cli "github.com/urfave/cli/v2"
	"google.golang.org/grpc"

	_ "go.uber.org/automaxprocs"
)

var (
	configPath      string
	embeddedStorage bool
)

func setupSentry(dsn string) (func(), error) {
	if dsn == "" {
		return nil, nil
	}

	sentryDefer := func() {
		defer sentry.Flush(2 * time.Second)
		err := recover()
		if err != nil {
			sentry.CaptureMessage(fmt.Sprintf("%+v", err))
			panic(err)
		}
	}
	return sentryDefer, sentry.Init(sentry.ClientOptions{
		Dsn: dsn,
	})
}

func serve(c *cli.Context) error {
	config, err := utils.LoadConfig(configPath)
	if err != nil {
		log.Fatalf("[main] %v", err)
	}

	if err := log.SetupLog(config.LogLevel); err != nil {
		log.Fatalf("[main] %v", err)
	}

	if sentryDefer, err := setupSentry(config.SentryDSN); err != nil {
		log.Warnf(context.TODO(), "[main] sentry %v", err)
	} else if sentryDefer != nil {
		defer sentryDefer()
	}

	if err := metrics.InitMetrics(config); err != nil {
		log.Errorf(context.TODO(), "[main] %v", err)
		return err
	}

	cluster, err := calcium.New(config, nil)
	if err != nil {
		log.Errorf(context.TODO(), "[main] %v", err)
		return err
	}
	defer cluster.Finalizer()
	cluster.DisasterRecover()

	stop := make(chan struct{}, 1)
	vibranium := rpc.New(cluster, config, stop)
	s, err := net.Listen("tcp", config.Bind)
	if err != nil {
		log.Errorf(context.TODO(), "[main] %v", err)
		return err
	}

	opts := []grpc.ServerOption{
		grpc.MaxConcurrentStreams(uint32(config.GRPCConfig.MaxConcurrentStreams)),
		grpc.MaxRecvMsgSize(config.GRPCConfig.MaxRecvMsgSize),
	}

	if config.Auth.Username != "" {
		log.Info("[main] Cluster auth enable.")
		auth := auth.NewAuth(config.Auth)
		opts = append(opts, grpc.StreamInterceptor(auth.StreamInterceptor))
		opts = append(opts, grpc.UnaryInterceptor(auth.UnaryInterceptor))
		log.Infof(context.TODO(), "[main] Username %s Password %s", config.Auth.Username, config.Auth.Password)
	}

	grpcServer := grpc.NewServer(opts...)
	pb.RegisterCoreRPCServer(grpcServer, vibranium)
	utils.SentryGo(func() {
		if err := grpcServer.Serve(s); err != nil {
			log.Errorf(context.TODO(), "[main] start grpc failed %v", err)
		}
	})

	if config.Profile != "" {
		http.Handle("/metrics", metrics.Client.ResourceMiddleware(cluster)(promhttp.Handler()))
		utils.SentryGo(func() {
			if err := http.ListenAndServe(config.Profile, nil); err != nil {
				log.Errorf(context.TODO(), "[main] start http failed %v", err)
			}
		})
	}

	unregisterService, err := cluster.RegisterService(c.Context)
	if err != nil {
		log.Errorf(context.TODO(), "[main] failed to register service: %v", err)
		return err
	}
	log.Info("[main] Cluster started successfully.")

	// wait for unix signals and try to GracefulStop
	ctx, cancel := signalcontext.OnInterrupt()
	defer cancel()
	<-ctx.Done()

	log.Info("[main] Interrupt by signal")
	close(stop)
	unregisterService()
	grpcServer.GracefulStop()
	log.Info("[main] gRPC server gracefully stopped.")

	log.Info("[main] Check if cluster still have running tasks.")
	vibranium.Wait()
	log.Info("[main] cluster gracefully stopped.")
	return nil
}

func main() {
	cli.VersionPrinter = func(c *cli.Context) {
		fmt.Print(version.String())
	}

	app := cli.NewApp()
	app.Name = version.NAME
	app.Usage = "Run eru core"
	app.Version = version.VERSION
	app.Flags = []cli.Flag{
		&cli.StringFlag{
			Name:        "config",
			Value:       "/etc/eru/core.yaml",
			Usage:       "config file path for core, in yaml",
			Destination: &configPath,
			EnvVars:     []string{"ERU_CONFIG_PATH"},
		},
		&cli.BoolFlag{
			Name:        "embedded-storage",
			Usage:       "active embedded storage",
			Destination: &embeddedStorage,
		},
	}
	app.Action = serve
	_ = app.Run(os.Args)
}
<commit_msg>can use testing storage when someone like it (#435)<commit_after>package main

import (
	"context"
	"fmt"
	"net"
	"net/http"
	_ "net/http/pprof" // nolint
	"os"
	"testing"
	"time"

	"github.com/getsentry/sentry-go"
	"github.com/projecteru2/core/auth"
	"github.com/projecteru2/core/cluster/calcium"
	"github.com/projecteru2/core/log"
	"github.com/projecteru2/core/metrics"
	"github.com/projecteru2/core/rpc"
	pb "github.com/projecteru2/core/rpc/gen"
	"github.com/projecteru2/core/utils"
	"github.com/projecteru2/core/version"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/sethvargo/go-signalcontext"
	cli "github.com/urfave/cli/v2"
	"google.golang.org/grpc"

	_ "go.uber.org/automaxprocs"
)

var (
	configPath      string
	embeddedStorage bool
)

func setupSentry(dsn string) (func(), error) {
	if dsn == "" {
		return nil, nil
	}

	sentryDefer := func() {
		defer sentry.Flush(2 * time.Second)
		err := recover()
		if err != nil {
			sentry.CaptureMessage(fmt.Sprintf("%+v", err))
			panic(err)
		}
	}
	return sentryDefer, sentry.Init(sentry.ClientOptions{
		Dsn: dsn,
	})
}

func serve(c *cli.Context) error {
	config, err := utils.LoadConfig(configPath)
	if err != nil {
		log.Fatalf("[main] %v", err)
	}

	if err := log.SetupLog(config.LogLevel); err != nil {
		log.Fatalf("[main] %v", err)
	}

	if sentryDefer, err := setupSentry(config.SentryDSN); err != nil {
		log.Warnf(context.TODO(), "[main] sentry %v", err)
	} else if sentryDefer != nil {
		defer sentryDefer()
	}

	if err := metrics.InitMetrics(config); err != nil {
		log.Errorf(context.TODO(), "[main] %v", err)
		return err
	}

	var t *testing.T
	if embeddedStorage {
		t = &testing.T{}
	}
	cluster, err := calcium.New(config, t)
	if err != nil {
		log.Errorf(context.TODO(), "[main] %v", err)
		return err
	}
	defer cluster.Finalizer()
	cluster.DisasterRecover()

	stop := make(chan struct{}, 1)
	vibranium := rpc.New(cluster, config, stop)
	s, err := net.Listen("tcp", config.Bind)
	if err != nil {
		log.Errorf(context.TODO(), "[main] %v", err)
		return err
	}

	opts := []grpc.ServerOption{
		grpc.MaxConcurrentStreams(uint32(config.GRPCConfig.MaxConcurrentStreams)),
		grpc.MaxRecvMsgSize(config.GRPCConfig.MaxRecvMsgSize),
	}

	if config.Auth.Username != "" {
		log.Info("[main] Cluster auth enable.")
		auth := auth.NewAuth(config.Auth)
		opts = append(opts, grpc.StreamInterceptor(auth.StreamInterceptor))
		opts = append(opts, grpc.UnaryInterceptor(auth.UnaryInterceptor))
		log.Infof(context.TODO(), "[main] Username %s Password %s", config.Auth.Username, config.Auth.Password)
	}

	grpcServer := grpc.NewServer(opts...)
	pb.RegisterCoreRPCServer(grpcServer, vibranium)
	utils.SentryGo(func() {
		if err := grpcServer.Serve(s); err != nil {
			log.Errorf(context.TODO(), "[main] start grpc failed %v", err)
		}
	})

	if config.Profile != "" {
		http.Handle("/metrics", metrics.Client.ResourceMiddleware(cluster)(promhttp.Handler()))
		utils.SentryGo(func() {
			if err := http.ListenAndServe(config.Profile, nil); err != nil {
				log.Errorf(context.TODO(), "[main] start http failed %v", err)
			}
		})
	}

	unregisterService, err := cluster.RegisterService(c.Context)
	if err != nil {
		log.Errorf(context.TODO(), "[main] failed to register service: %v", err)
		return err
	}
	log.Info("[main] Cluster started successfully.")

	// wait for unix signals and try to GracefulStop
	ctx, cancel := signalcontext.OnInterrupt()
	defer cancel()
	<-ctx.Done()

	log.Info("[main] Interrupt by signal")
	close(stop)
	unregisterService()
	grpcServer.GracefulStop()
	log.Info("[main] gRPC server gracefully stopped.")

	log.Info("[main] Check if cluster still have running tasks.")
	vibranium.Wait()
	log.Info("[main] cluster gracefully stopped.")
	return nil
}

func main() {
	cli.VersionPrinter = func(c *cli.Context) {
		fmt.Print(version.String())
	}

	app := cli.NewApp()
	app.Name = version.NAME
	app.Usage = "Run eru core"
	app.Version = version.VERSION
	app.Flags = []cli.Flag{
		&cli.StringFlag{
			Name:        "config",
			Value:       "/etc/eru/core.yaml",
			Usage:       "config file path for core, in yaml",
			Destination: &configPath,
			EnvVars:     []string{"ERU_CONFIG_PATH"},
		},
		&cli.BoolFlag{
			Name:        "embedded-storage",
			Usage:       "active embedded storage",
			Destination: &embeddedStorage,
		},
	}
	app.Action = serve
	_ = app.Run(os.Args)
}
<|endoftext|>


<commit_before>package handler

import (
	"github.com/oursky/ourd/handler/handlertest"
	. "github.com/oursky/ourd/ourtest"
	. "github.com/smartystreets/goconvey/convey"
	"testing"

	"github.com/oursky/ourd/oddb"
	"github.com/oursky/ourd/push"
	"github.com/oursky/ourd/router"
)

func TestPushToDevice(t *testing.T) {
	Convey("push to device", t, func() {
		testdevice := oddb.Device{
			ID:         "device",
			Type:       "ios",
			Token:      "token",
			UserInfoID: "userid",
		}
		conn := simpleDeviceConn{
			devices: []oddb.Device{testdevice},
		}

		r := handlertest.NewSingleRouteRouter(PushToDeviceHandler, func(p *router.Payload) {
			p.DBConn = &conn
		})

		originalSendFunc := sendPushNotification
		defer func() {
			sendPushNotification = originalSendFunc
		}()

		Convey("push to single device", func() {
			called := false
			sendPushNotification = func(sender *push.APNSPusher, device *oddb.Device, m push.Mapper) {
				So(device, ShouldResemble, &testdevice)
				So(m.Map(), ShouldResemble, map[string]interface{}{
					"aps": map[string]interface{}{
						"alert": "This is a message.",
						"sound": "sosumi.mp3",
					},
					"acme": "interesting",
				})
				called = true
			}
			resp := r.POST(`{
					"device_ids": ["device"],
					"notification": {
						"aps": {
							"alert": "This is a message.",
							"sound": "sosumi.mp3"
						},
						"acme": "interesting"
					}
				}`)
			So(resp.Code, ShouldEqual, 200)
			So(resp.Body.Bytes(), ShouldEqualJSON, `{
	"result": [{
		"_id": "device"
	}]
}`)
			So(called, ShouldBeTrue)
		})

		Convey("push to non-existent device", func() {
			called := false
			sendPushNotification = func(sender *push.APNSPusher, device *oddb.Device, m push.Mapper) {
				called = true
			}
			resp := r.POST(`{
						"device_ids": ["nonexistent"],
						"notification": {
							"aps": {
								"alert": "This is a message.",
								"sound": "sosumi.mp3"
							},
							"acme": "interesting"
						}
					}`)
			So(resp.Code, ShouldEqual, 200)
			So(resp.Body.Bytes(), ShouldEqualJSON, `{
	"result": [{
		"_id": "nonexistent",
		"_type": "error",
		"message": "cannot find device \"nonexistent\"",
		"type": "ResourceNotFound",
		"code": 101,
		"info": {"id": "nonexistent"}
	}]
}`)
			So(called, ShouldBeFalse)
		})
	})

}

func TestPushToUser(t *testing.T) {
	Convey("push to user", t, func() {
		testdevice1 := oddb.Device{
			ID:         "device1",
			Type:       "ios",
			Token:      "token",
			UserInfoID: "johndoe",
		}
		testdevice2 := oddb.Device{
			ID:         "device2",
			Type:       "ios",
			Token:      "token",
			UserInfoID: "johndoe",
		}
		testdevice3 := oddb.Device{
			ID:         "device2",
			Type:       "ios",
			Token:      "token",
			UserInfoID: "janedoe",
		}
		conn := simpleDeviceConn{
			devices: []oddb.Device{testdevice1, testdevice2, testdevice3},
		}

		r := handlertest.NewSingleRouteRouter(PushToUserHandler, func(p *router.Payload) {
			p.DBConn = &conn
		})

		originalSendFunc := sendPushNotification
		defer func() {
			sendPushNotification = originalSendFunc
		}()

		Convey("push to single user", func() {
			sentDevices := make([]oddb.Device, 0)
			sendPushNotification = func(sender *push.APNSPusher, device *oddb.Device, m push.Mapper) {
				So(m.Map(), ShouldResemble, map[string]interface{}{
					"aps": map[string]interface{}{
						"alert": "This is a message.",
						"sound": "sosumi.mp3",
					},
					"acme": "interesting",
				})
				sentDevices = append(sentDevices, *device)
			}
			resp := r.POST(`{
					"user_ids": ["johndoe"],
					"notification": {
						"aps": {
							"alert": "This is a message.",
							"sound": "sosumi.mp3"
						},
						"acme": "interesting"
					}
				}`)
			So(resp.Code, ShouldEqual, 200)
			So(resp.Body.Bytes(), ShouldEqualJSON, `{
	"result": [{"_id":"johndoe"}]
}`)

			So(len(sentDevices), ShouldEqual, 2)
			So(sentDevices[0], ShouldResemble, testdevice1)
			So(sentDevices[1], ShouldResemble, testdevice2)
		})

		Convey("push to non-existent user", func() {
			called := false
			sendPushNotification = func(sender *push.APNSPusher, device *oddb.Device, m push.Mapper) {
				called = true
			}
			resp := r.POST(`{
					"user_ids": ["nonexistent"],
					"notification": {
						"aps": {
							"alert": "This is a message.",
							"sound": "sosumi.mp3"
						},
						"acme": "interesting"
					}
				}`)
			So(resp.Code, ShouldEqual, 200)
			So(resp.Body.Bytes(), ShouldEqualJSON, `{
	"result": [{
		"_id": "nonexistent",
		"_type": "error",
		"message": "cannot find user \"nonexistent\"",
		"type": "ResourceNotFound",
		"code": 101,
		"info": {"id": "nonexistent"}
	}]
}`)
			So(called, ShouldBeFalse)
		})
	})

}

type simpleDeviceConn struct {
	devices []oddb.Device
	oddb.Conn
}

func (conn *simpleDeviceConn) GetDevice(id string, device *oddb.Device) error {
	for _, prospectiveDevice := range conn.devices {
		if prospectiveDevice.ID == id {
			*device = prospectiveDevice
			return nil
		}
	}
	return oddb.ErrDeviceNotFound
}

func (conn *simpleDeviceConn) QueryDevicesByUser(user string) ([]oddb.Device, error) {
	result := make([]oddb.Device, 0)
	for _, prospectiveDevice := range conn.devices {
		if prospectiveDevice.UserInfoID == user {
			result = append(result, prospectiveDevice)
		}
	}
	if len(result) == 0 {
		return nil, oddb.ErrUserNotFound
	} else {
		return result, nil
	}
}
<commit_msg>subscription: fix push_test<commit_after>package handler

import (
	"testing"

	"github.com/oursky/ourd/handler/handlertest"
	. "github.com/oursky/ourd/ourtest"
	. "github.com/smartystreets/goconvey/convey"

	"github.com/oursky/ourd/oddb"
	"github.com/oursky/ourd/push"
	"github.com/oursky/ourd/router"
)

func TestPushToDevice(t *testing.T) {
	Convey("push to device", t, func() {
		testdevice := oddb.Device{
			ID:         "device",
			Type:       "ios",
			Token:      "token",
			UserInfoID: "userid",
		}
		conn := simpleDeviceConn{
			devices: []oddb.Device{testdevice},
		}

		r := handlertest.NewSingleRouteRouter(PushToDeviceHandler, func(p *router.Payload) {
			p.DBConn = &conn
		})

		originalSendFunc := sendPushNotification
		defer func() {
			sendPushNotification = originalSendFunc
		}()

		Convey("push to single device", func() {
			called := false
			sendPushNotification = func(sender push.Sender, device *oddb.Device, m push.Mapper) {
				So(device, ShouldResemble, &testdevice)
				So(m.Map(), ShouldResemble, map[string]interface{}{
					"aps": map[string]interface{}{
						"alert": "This is a message.",
						"sound": "sosumi.mp3",
					},
					"acme": "interesting",
				})
				called = true
			}
			resp := r.POST(`{
					"device_ids": ["device"],
					"notification": {
						"aps": {
							"alert": "This is a message.",
							"sound": "sosumi.mp3"
						},
						"acme": "interesting"
					}
				}`)
			So(resp.Code, ShouldEqual, 200)
			So(resp.Body.Bytes(), ShouldEqualJSON, `{
	"result": [{
		"_id": "device"
	}]
}`)
			So(called, ShouldBeTrue)
		})

		Convey("push to non-existent device", func() {
			called := false
			sendPushNotification = func(sender push.Sender, device *oddb.Device, m push.Mapper) {
				called = true
			}
			resp := r.POST(`{
						"device_ids": ["nonexistent"],
						"notification": {
							"aps": {
								"alert": "This is a message.",
								"sound": "sosumi.mp3"
							},
							"acme": "interesting"
						}
					}`)
			So(resp.Code, ShouldEqual, 200)
			So(resp.Body.Bytes(), ShouldEqualJSON, `{
	"result": [{
		"_id": "nonexistent",
		"_type": "error",
		"message": "cannot find device \"nonexistent\"",
		"type": "ResourceNotFound",
		"code": 101,
		"info": {"id": "nonexistent"}
	}]
}`)
			So(called, ShouldBeFalse)
		})
	})

}

func TestPushToUser(t *testing.T) {
	Convey("push to user", t, func() {
		testdevice1 := oddb.Device{
			ID:         "device1",
			Type:       "ios",
			Token:      "token",
			UserInfoID: "johndoe",
		}
		testdevice2 := oddb.Device{
			ID:         "device2",
			Type:       "ios",
			Token:      "token",
			UserInfoID: "johndoe",
		}
		testdevice3 := oddb.Device{
			ID:         "device2",
			Type:       "ios",
			Token:      "token",
			UserInfoID: "janedoe",
		}
		conn := simpleDeviceConn{
			devices: []oddb.Device{testdevice1, testdevice2, testdevice3},
		}

		r := handlertest.NewSingleRouteRouter(PushToUserHandler, func(p *router.Payload) {
			p.DBConn = &conn
		})

		originalSendFunc := sendPushNotification
		defer func() {
			sendPushNotification = originalSendFunc
		}()

		Convey("push to single user", func() {
			sentDevices := make([]oddb.Device, 0)
			sendPushNotification = func(sender push.Sender, device *oddb.Device, m push.Mapper) {
				So(m.Map(), ShouldResemble, map[string]interface{}{
					"aps": map[string]interface{}{
						"alert": "This is a message.",
						"sound": "sosumi.mp3",
					},
					"acme": "interesting",
				})
				sentDevices = append(sentDevices, *device)
			}
			resp := r.POST(`{
					"user_ids": ["johndoe"],
					"notification": {
						"aps": {
							"alert": "This is a message.",
							"sound": "sosumi.mp3"
						},
						"acme": "interesting"
					}
				}`)
			So(resp.Code, ShouldEqual, 200)
			So(resp.Body.Bytes(), ShouldEqualJSON, `{
	"result": [{"_id":"johndoe"}]
}`)

			So(len(sentDevices), ShouldEqual, 2)
			So(sentDevices[0], ShouldResemble, testdevice1)
			So(sentDevices[1], ShouldResemble, testdevice2)
		})

		Convey("push to non-existent user", func() {
			called := false
			sendPushNotification = func(sender push.Sender, device *oddb.Device, m push.Mapper) {
				called = true
			}
			resp := r.POST(`{
					"user_ids": ["nonexistent"],
					"notification": {
						"aps": {
							"alert": "This is a message.",
							"sound": "sosumi.mp3"
						},
						"acme": "interesting"
					}
				}`)
			So(resp.Code, ShouldEqual, 200)
			So(resp.Body.Bytes(), ShouldEqualJSON, `{
	"result": [{
		"_id": "nonexistent",
		"_type": "error",
		"message": "cannot find user \"nonexistent\"",
		"type": "ResourceNotFound",
		"code": 101,
		"info": {"id": "nonexistent"}
	}]
}`)
			So(called, ShouldBeFalse)
		})
	})

}

type simpleDeviceConn struct {
	devices []oddb.Device
	oddb.Conn
}

func (conn *simpleDeviceConn) GetDevice(id string, device *oddb.Device) error {
	for _, prospectiveDevice := range conn.devices {
		if prospectiveDevice.ID == id {
			*device = prospectiveDevice
			return nil
		}
	}
	return oddb.ErrDeviceNotFound
}

func (conn *simpleDeviceConn) QueryDevicesByUser(user string) ([]oddb.Device, error) {
	result := make([]oddb.Device, 0)
	for _, prospectiveDevice := range conn.devices {
		if prospectiveDevice.UserInfoID == user {
			result = append(result, prospectiveDevice)
		}
	}
	if len(result) == 0 {
		return nil, oddb.ErrUserNotFound
	} else {
		return result, nil
	}
}
<|endoftext|>


<commit_before>/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at
  http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.  See the License for the
specific language governing permissions and limitations
under the License.
*/

//sqlParser.go
//interacts with database
package sqlParser

import (
	_ "github.com/go-sql-driver/mysql"
	"github.com/jmoiron/sqlx"
)

func check(e error) {
	if e != nil {
		panic(e)
	}
}

//connects to and returns a pointer to the database
func ConnectToDatabase(username string, password string, environment string) sqlx.DB {
	db, err := sqlx.Open("mysql", username+":"+password+"@tcp(localhost:3306)/"+environment)
	check(err)

	return *db
}

//returns array of table name strings from queried database
func GetTableNames(db sqlx.DB) []string {
	var tableNames []string

	tableRawBytes := make([]byte, 1)
	tableInterface := make([]interface{}, 1)

	tableInterface[0] = &tableRawBytes

	rows, err := db.Query("SELECT TABLE_NAME FROM information_schema.tables WHERE TABLE_TYPE='BASE TABLE' and TABLE_SCHEMA='to_development'")
	check(err)

	for rows.Next() {
		err := rows.Scan(tableInterface...)
		check(err)

		tableNames = append(tableNames, string(tableRawBytes))
	}

	return tableNames
}

//returns *Rows from given table (name) from queried database
func GetRows(db sqlx.DB, tableName string) *sqlx.Rows {
	rows, err := db.Queryx("SELECT * from " + tableName)
	check(err)

	return rows
}

//returns array of column names from table in database
func GetColumnNames(db sqlx.DB, tableName string) []string {
	var colNames []string

	colRawBytes := make([]byte, 1)
	colInterface := make([]interface{}, 1)

	colInterface[0] = &colRawBytes

	rows, err := db.Query("SELECT COLUMN_NAME FROM information_schema.columns WHERE TABLE_NAME='" + tableName + "' and TABLE_SCHEMA='to_development'")
	check(err)

	for rows.Next() {
		err := rows.Scan(colInterface...)
		check(err)

		colNames = append(colNames, string(colRawBytes))
	}

	return colNames
}

//returns array of column names from table in database
func GetColumnTypes(db sqlx.DB, tableName string) []string {
	var colTypes []string

	colRawBytes := make([]byte, 1)
	colInterface := make([]interface{}, 1)

	colInterface[0] = &colRawBytes

	rows, err := db.Query("SELECT COLUMN_TYPE FROM information_schema.columns WHERE TABLE_NAME='" + tableName + "' and TABLE_SCHEMA='to_development'")
	check(err)

	for rows.Next() {
		err := rows.Scan(colInterface...)
		check(err)

		colTypes = append(colTypes, string(colRawBytes))
	}

	return colTypes
}
<commit_msg>Removed hard coded table schema<commit_after>/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at
  http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.  See the License for the
specific language governing permissions and limitations
under the License.
*/

//sqlParser.go
//interacts with database
package sqlParser

import (
	_ "github.com/go-sql-driver/mysql"
	"github.com/jmoiron/sqlx"
)

var (
	globalEnvironment = ""
)

func check(e error) {
	if e != nil {
		panic(e)
	}
}

//connects to and returns a pointer to the database
func ConnectToDatabase(username string, password string, environment string) sqlx.DB {
	db, err := sqlx.Open("mysql", username+":"+password+"@tcp(localhost:3306)/"+environment)
	check(err)

	//set globalEnvironment
	globalEnvironment = environment

	return *db
}

//returns array of table name strings from queried database
func GetTableNames(db sqlx.DB) []string {
	var tableNames []string

	tableRawBytes := make([]byte, 1)
	tableInterface := make([]interface{}, 1)

	tableInterface[0] = &tableRawBytes

	rows, err := db.Query("SELECT TABLE_NAME FROM information_schema.tables WHERE TABLE_TYPE='BASE TABLE' and TABLE_SCHEMA='" + globalEnvironment + "'")
	check(err)

	for rows.Next() {
		err := rows.Scan(tableInterface...)
		check(err)

		tableNames = append(tableNames, string(tableRawBytes))
	}

	return tableNames
}

//returns *Rows from given table (name) from queried database
func GetRows(db sqlx.DB, tableName string) *sqlx.Rows {
	rows, err := db.Queryx("SELECT * from " + tableName)
	check(err)

	return rows
}

//returns array of column names from table in database
func GetColumnNames(db sqlx.DB, tableName string) []string {
	var colNames []string

	colRawBytes := make([]byte, 1)
	colInterface := make([]interface{}, 1)

	colInterface[0] = &colRawBytes

	rows, err := db.Query("SELECT COLUMN_NAME FROM information_schema.columns WHERE TABLE_NAME='" + tableName + "' and TABLE_SCHEMA='" + globalEnvironment + "'")
	check(err)

	for rows.Next() {
		err := rows.Scan(colInterface...)
		check(err)

		colNames = append(colNames, string(colRawBytes))
	}

	return colNames
}

//returns array of column names from table in database
func GetColumnTypes(db sqlx.DB, tableName string) []string {
	var colTypes []string

	colRawBytes := make([]byte, 1)
	colInterface := make([]interface{}, 1)

	colInterface[0] = &colRawBytes

	rows, err := db.Query("SELECT COLUMN_TYPE FROM information_schema.columns WHERE TABLE_NAME='" + tableName + "' and TABLE_SCHEMA='" + globalEnvironment + "'")
	check(err)

	for rows.Next() {
		err := rows.Scan(colInterface...)
		check(err)

		colTypes = append(colTypes, string(colRawBytes))
	}

	return colTypes
}
<|endoftext|>


<commit_before>// This library implements a cron spec parser and runner.  See the README for
// more details.
package cron

import (
	"log"
	"runtime"
	"sort"
	"time"
)

// Cron keeps track of any number of entries, invoking the associated func as
// specified by the schedule. It may be started, stopped, and the entries may
// be inspected while running.
type Cron struct {
	entries  []*Entry
	stop     chan struct{}
	add      chan *Entry
	snapshot chan []*Entry
	running  bool
	ErrorLog *log.Logger
	location *time.Location
}

// Job is an interface for submitted cron jobs.
type Job interface {
	Run()
}

// The Schedule describes a job's duty cycle.
type Schedule interface {
	// Return the next activation time, later than the given time.
	// Next is invoked initially, and then each time the job is run.
	Next(time.Time) time.Time
}

// Entry consists of a schedule and the func to execute on that schedule.
type Entry struct {
	// The schedule on which this job should be run.
	Schedule Schedule

	// The next time the job will run. This is the zero time if Cron has not been
	// started or this entry's schedule is unsatisfiable
	Next time.Time

	// The last time this job was run. This is the zero time if the job has never
	// been run.
	Prev time.Time

	// The Job to run.
	Job Job
}

// byTime is a wrapper for sorting the entry array by time
// (with zero time at the end).
type byTime []*Entry

func (s byTime) Len() int      { return len(s) }
func (s byTime) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s byTime) Less(i, j int) bool {
	// Two zero times should return false.
	// Otherwise, zero is "greater" than any other time.
	// (To sort it at the end of the list.)
	if s[i].Next.IsZero() {
		return false
	}
	if s[j].Next.IsZero() {
		return true
	}
	return s[i].Next.Before(s[j].Next)
}

// New returns a new Cron job runner.
func New() *Cron {
	return NewWithLocation(time.Now().Location())
}

// NewWithLocation returns a new Cron job runner.
func NewWithLocation(location *time.Location) *Cron {
	return &Cron{
		entries:  nil,
		add:      make(chan *Entry),
		stop:     make(chan struct{}),
		snapshot: make(chan []*Entry),
		running:  false,
		ErrorLog: nil,
		location: location,
	}
}

// A wrapper that turns a func() into a cron.Job
type FuncJob func()

func (f FuncJob) Run() { f() }

// AddFunc adds a func to the Cron to be run on the given schedule.
func (c *Cron) AddFunc(spec string, cmd func()) error {
	return c.AddJob(spec, FuncJob(cmd))
}

// AddJob adds a Job to the Cron to be run on the given schedule.
func (c *Cron) AddJob(spec string, cmd Job) error {
	schedule, err := Parse(spec)
	if err != nil {
		return err
	}
	c.Schedule(schedule, cmd)
	return nil
}

// Schedule adds a Job to the Cron to be run on the given schedule.
func (c *Cron) Schedule(schedule Schedule, cmd Job) {
	entry := &Entry{
		Schedule: schedule,
		Job:      cmd,
	}
	if !c.running {
		c.entries = append(c.entries, entry)
		return
	}

	c.add <- entry
}

// Entries returns a snapshot of the cron entries.
func (c *Cron) Entries() []*Entry {
	if c.running {
		c.snapshot <- nil
		x := <-c.snapshot
		return x
	}
	return c.entrySnapshot()
}

// Location gets the time zone location
func (c *Cron) Location() *time.Location {
	return c.location
}

// Start the cron scheduler in its own go-routine.
func (c *Cron) Start() {
	c.running = true
	go c.run()
}

func (c *Cron) runWithRecovery(j Job) {
	defer func() {
		if r := recover(); r != nil {
			const size = 64 << 10
			buf := make([]byte, size)
			buf = buf[:runtime.Stack(buf, false)]
			c.logf("cron: panic running job: %v\n%s", r, buf)
		}
	}()
	j.Run()
}

// Run the scheduler.. this is private just due to the need to synchronize
// access to the 'running' state variable.
func (c *Cron) run() {
	// Figure out the next activation times for each entry.
	now := time.Now().In(c.location)
	for _, entry := range c.entries {
		entry.Next = entry.Schedule.Next(now)
	}

	for {
		// Determine the next entry to run.
		sort.Sort(byTime(c.entries))

		var effective time.Time
		if len(c.entries) == 0 || c.entries[0].Next.IsZero() {
			// If there are no entries yet, just sleep - it still handles new entries
			// and stop requests.
			effective = now.AddDate(10, 0, 0)
		} else {
			effective = c.entries[0].Next
		}

		select {
		case now = <-time.After(effective.Sub(now)):
			// Run every entry whose next time was this effective time.
			for _, e := range c.entries {
				if e.Next != effective {
					break
				}
				go c.runWithRecovery(e.Job)
				e.Prev = e.Next
				e.Next = e.Schedule.Next(now)
			}
			continue

		case newEntry := <-c.add:
			c.entries = append(c.entries, newEntry)
			newEntry.Next = newEntry.Schedule.Next(time.Now().Local())

		case <-c.snapshot:
			c.snapshot <- c.entrySnapshot()

		case <-c.stop:
			return
		}

		// 'now' should be updated after newEntry and snapshot cases.
		now = time.Now().In(c.location)
	}
}

// Logs an error to stderr or to the configured error log
func (c *Cron) logf(format string, args ...interface{}) {
	if c.ErrorLog != nil {
		c.ErrorLog.Printf(format, args...)
	} else {
		log.Printf(format, args...)
	}
}

// Stop stops the cron scheduler if it is running; otherwise it does nothing.
func (c *Cron) Stop() {
	if !c.running {
		return
	}
	c.stop <- struct{}{}
	c.running = false
}

// entrySnapshot returns a copy of the current cron entry list.
func (c *Cron) entrySnapshot() []*Entry {
	entries := []*Entry{}
	for _, e := range c.entries {
		entries = append(entries, &Entry{
			Schedule: e.Schedule,
			Next:     e.Next,
			Prev:     e.Prev,
			Job:      e.Job,
		})
	}
	return entries
}
<commit_msg>Comment update to clarify local time zone usage<commit_after>// This library implements a cron spec parser and runner.  See the README for
// more details.
package cron

import (
	"log"
	"runtime"
	"sort"
	"time"
)

// Cron keeps track of any number of entries, invoking the associated func as
// specified by the schedule. It may be started, stopped, and the entries may
// be inspected while running.
type Cron struct {
	entries  []*Entry
	stop     chan struct{}
	add      chan *Entry
	snapshot chan []*Entry
	running  bool
	ErrorLog *log.Logger
	location *time.Location
}

// Job is an interface for submitted cron jobs.
type Job interface {
	Run()
}

// The Schedule describes a job's duty cycle.
type Schedule interface {
	// Return the next activation time, later than the given time.
	// Next is invoked initially, and then each time the job is run.
	Next(time.Time) time.Time
}

// Entry consists of a schedule and the func to execute on that schedule.
type Entry struct {
	// The schedule on which this job should be run.
	Schedule Schedule

	// The next time the job will run. This is the zero time if Cron has not been
	// started or this entry's schedule is unsatisfiable
	Next time.Time

	// The last time this job was run. This is the zero time if the job has never
	// been run.
	Prev time.Time

	// The Job to run.
	Job Job
}

// byTime is a wrapper for sorting the entry array by time
// (with zero time at the end).
type byTime []*Entry

func (s byTime) Len() int      { return len(s) }
func (s byTime) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s byTime) Less(i, j int) bool {
	// Two zero times should return false.
	// Otherwise, zero is "greater" than any other time.
	// (To sort it at the end of the list.)
	if s[i].Next.IsZero() {
		return false
	}
	if s[j].Next.IsZero() {
		return true
	}
	return s[i].Next.Before(s[j].Next)
}

// New returns a new Cron job runner, in the Local time zone.
func New() *Cron {
	return NewWithLocation(time.Now().Location())
}

// NewWithLocation returns a new Cron job runner.
func NewWithLocation(location *time.Location) *Cron {
	return &Cron{
		entries:  nil,
		add:      make(chan *Entry),
		stop:     make(chan struct{}),
		snapshot: make(chan []*Entry),
		running:  false,
		ErrorLog: nil,
		location: location,
	}
}

// A wrapper that turns a func() into a cron.Job
type FuncJob func()

func (f FuncJob) Run() { f() }

// AddFunc adds a func to the Cron to be run on the given schedule.
func (c *Cron) AddFunc(spec string, cmd func()) error {
	return c.AddJob(spec, FuncJob(cmd))
}

// AddJob adds a Job to the Cron to be run on the given schedule.
func (c *Cron) AddJob(spec string, cmd Job) error {
	schedule, err := Parse(spec)
	if err != nil {
		return err
	}
	c.Schedule(schedule, cmd)
	return nil
}

// Schedule adds a Job to the Cron to be run on the given schedule.
func (c *Cron) Schedule(schedule Schedule, cmd Job) {
	entry := &Entry{
		Schedule: schedule,
		Job:      cmd,
	}
	if !c.running {
		c.entries = append(c.entries, entry)
		return
	}

	c.add <- entry
}

// Entries returns a snapshot of the cron entries.
func (c *Cron) Entries() []*Entry {
	if c.running {
		c.snapshot <- nil
		x := <-c.snapshot
		return x
	}
	return c.entrySnapshot()
}

// Location gets the time zone location
func (c *Cron) Location() *time.Location {
	return c.location
}

// Start the cron scheduler in its own go-routine.
func (c *Cron) Start() {
	c.running = true
	go c.run()
}

func (c *Cron) runWithRecovery(j Job) {
	defer func() {
		if r := recover(); r != nil {
			const size = 64 << 10
			buf := make([]byte, size)
			buf = buf[:runtime.Stack(buf, false)]
			c.logf("cron: panic running job: %v\n%s", r, buf)
		}
	}()
	j.Run()
}

// Run the scheduler.. this is private just due to the need to synchronize
// access to the 'running' state variable.
func (c *Cron) run() {
	// Figure out the next activation times for each entry.
	now := time.Now().In(c.location)
	for _, entry := range c.entries {
		entry.Next = entry.Schedule.Next(now)
	}

	for {
		// Determine the next entry to run.
		sort.Sort(byTime(c.entries))

		var effective time.Time
		if len(c.entries) == 0 || c.entries[0].Next.IsZero() {
			// If there are no entries yet, just sleep - it still handles new entries
			// and stop requests.
			effective = now.AddDate(10, 0, 0)
		} else {
			effective = c.entries[0].Next
		}

		select {
		case now = <-time.After(effective.Sub(now)):
			// Run every entry whose next time was this effective time.
			for _, e := range c.entries {
				if e.Next != effective {
					break
				}
				go c.runWithRecovery(e.Job)
				e.Prev = e.Next
				e.Next = e.Schedule.Next(now)
			}
			continue

		case newEntry := <-c.add:
			c.entries = append(c.entries, newEntry)
			newEntry.Next = newEntry.Schedule.Next(time.Now().Local())

		case <-c.snapshot:
			c.snapshot <- c.entrySnapshot()

		case <-c.stop:
			return
		}

		// 'now' should be updated after newEntry and snapshot cases.
		now = time.Now().In(c.location)
	}
}

// Logs an error to stderr or to the configured error log
func (c *Cron) logf(format string, args ...interface{}) {
	if c.ErrorLog != nil {
		c.ErrorLog.Printf(format, args...)
	} else {
		log.Printf(format, args...)
	}
}

// Stop stops the cron scheduler if it is running; otherwise it does nothing.
func (c *Cron) Stop() {
	if !c.running {
		return
	}
	c.stop <- struct{}{}
	c.running = false
}

// entrySnapshot returns a copy of the current cron entry list.
func (c *Cron) entrySnapshot() []*Entry {
	entries := []*Entry{}
	for _, e := range c.entries {
		entries = append(entries, &Entry{
			Schedule: e.Schedule,
			Next:     e.Next,
			Prev:     e.Prev,
			Job:      e.Job,
		})
	}
	return entries
}
<|endoftext|>


<commit_before>package weed_server

import (
	"context"
	"net/http"
	"time"

	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
	"github.com/chrislusf/seaweedfs/weed/storage/types"
)

func (vs *VolumeServer) BatchDelete(ctx context.Context, req *volume_server_pb.BatchDeleteRequest) (*volume_server_pb.BatchDeleteResponse, error) {

	resp := &volume_server_pb.BatchDeleteResponse{}

	now := uint64(time.Now().Unix())

	for _, fid := range req.FileIds {
		vid, id_cookie, err := operation.ParseFileId(fid)
		if err != nil {
			resp.Results = append(resp.Results, &volume_server_pb.DeleteResult{
				FileId: fid,
				Status: http.StatusBadRequest,
				Error:  err.Error()})
			continue
		}

		n := new(needle.Needle)
		volumeId, _ := needle.NewVolumeId(vid)
		if req.SkipCookieCheck {
			n.Id, err = types.ParseNeedleId(id_cookie)
			if err != nil {
				resp.Results = append(resp.Results, &volume_server_pb.DeleteResult{
					FileId: fid,
					Status: http.StatusBadRequest,
					Error:  err.Error()})
				continue
			}
		} else {
			n.ParsePath(id_cookie)
			cookie := n.Cookie
			if _, err := vs.store.ReadVolumeNeedle(volumeId, n, nil); err != nil {
				resp.Results = append(resp.Results, &volume_server_pb.DeleteResult{
					FileId: fid,
					Status: http.StatusNotFound,
					Error:  err.Error(),
				})
				continue
			}
			if n.Cookie != cookie {
				resp.Results = append(resp.Results, &volume_server_pb.DeleteResult{
					FileId: fid,
					Status: http.StatusBadRequest,
					Error:  "File Random Cookie does not match.",
				})
				break
			}
		}

		if n.IsChunkedManifest() {
			resp.Results = append(resp.Results, &volume_server_pb.DeleteResult{
				FileId: fid,
				Status: http.StatusNotAcceptable,
				Error:  "ChunkManifest: not allowed in batch delete mode.",
			})
			continue
		}

		n.LastModified = now
		if size, err := vs.store.DeleteVolumeNeedle(volumeId, n); err != nil {
			resp.Results = append(resp.Results, &volume_server_pb.DeleteResult{
				FileId: fid,
				Status: http.StatusInternalServerError,
				Error:  err.Error()},
			)
		} else {
			resp.Results = append(resp.Results, &volume_server_pb.DeleteResult{
				FileId: fid,
				Status: http.StatusAccepted,
				Size:   uint32(size)},
			)
		}
	}

	return resp, nil

}
<commit_msg>mount: fix for deletion stopped working since 2.53<commit_after>package weed_server

import (
	"context"
	"net/http"
	"time"

	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
)

func (vs *VolumeServer) BatchDelete(ctx context.Context, req *volume_server_pb.BatchDeleteRequest) (*volume_server_pb.BatchDeleteResponse, error) {

	resp := &volume_server_pb.BatchDeleteResponse{}

	now := uint64(time.Now().Unix())

	for _, fid := range req.FileIds {
		vid, id_cookie, err := operation.ParseFileId(fid)
		if err != nil {
			resp.Results = append(resp.Results, &volume_server_pb.DeleteResult{
				FileId: fid,
				Status: http.StatusBadRequest,
				Error:  err.Error()})
			continue
		}

		n := new(needle.Needle)
		volumeId, _ := needle.NewVolumeId(vid)
		if req.SkipCookieCheck {
			n.Id, _, err = needle.ParseNeedleIdCookie(id_cookie)
			if err != nil {
				resp.Results = append(resp.Results, &volume_server_pb.DeleteResult{
					FileId: fid,
					Status: http.StatusBadRequest,
					Error:  err.Error()})
				continue
			}
		} else {
			n.ParsePath(id_cookie)
			cookie := n.Cookie
			if _, err := vs.store.ReadVolumeNeedle(volumeId, n, nil); err != nil {
				resp.Results = append(resp.Results, &volume_server_pb.DeleteResult{
					FileId: fid,
					Status: http.StatusNotFound,
					Error:  err.Error(),
				})
				continue
			}
			if n.Cookie != cookie {
				resp.Results = append(resp.Results, &volume_server_pb.DeleteResult{
					FileId: fid,
					Status: http.StatusBadRequest,
					Error:  "File Random Cookie does not match.",
				})
				break
			}
		}

		if n.IsChunkedManifest() {
			resp.Results = append(resp.Results, &volume_server_pb.DeleteResult{
				FileId: fid,
				Status: http.StatusNotAcceptable,
				Error:  "ChunkManifest: not allowed in batch delete mode.",
			})
			continue
		}

		n.LastModified = now
		if size, err := vs.store.DeleteVolumeNeedle(volumeId, n); err != nil {
			resp.Results = append(resp.Results, &volume_server_pb.DeleteResult{
				FileId: fid,
				Status: http.StatusInternalServerError,
				Error:  err.Error()},
			)
		} else {
			resp.Results = append(resp.Results, &volume_server_pb.DeleteResult{
				FileId: fid,
				Status: http.StatusAccepted,
				Size:   uint32(size)},
			)
		}
	}

	return resp, nil

}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package discoverspaces

import (
	"fmt"
	"regexp"
	"strings"

	"github.com/juju/errors"
	"github.com/juju/loggo"
	"github.com/juju/names"
	"github.com/juju/utils/set"
	"launchpad.net/tomb"

	"github.com/juju/juju/api/discoverspaces"
	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/environs"
	"github.com/juju/juju/network"
	"github.com/juju/juju/worker"
)

var logger = loggo.GetLogger("juju.discoverspaces")

type discoverspacesWorker struct {
	api               *discoverspaces.API
	tomb              tomb.Tomb
	discoveringSpaces chan struct{}
}

var dashPrefix = regexp.MustCompile("^-*")
var dashSuffix = regexp.MustCompile("-*$")
var multipleDashes = regexp.MustCompile("--+")

func convertSpaceName(name string, existing set.Strings) string {
	// First lower case and replace spaces with dashes.
	name = strings.Replace(name, " ", "-", -1)
	name = strings.ToLower(name)
	// Replace any character that isn't in the set "-", "a-z", "0-9".
	name = network.SpaceInvalidChars.ReplaceAllString(name, "")
	// Get rid of any dashes at the start as that isn't valid.
	name = dashPrefix.ReplaceAllString(name, "")
	// And any at the end.
	name = dashSuffix.ReplaceAllString(name, "")
	// Repleace multiple dashes with a single dash.
	name = multipleDashes.ReplaceAllString(name, "-")
	// Special case of when the space name was only dashes or invalid
	// characters!
	if name == "" {
		name = "empty"
	}
	// If this name is in use add a numerical suffix.
	if existing.Contains(name) {
		counter := 2
		for existing.Contains(name + fmt.Sprintf("-%d", counter)) {
			counter += 1
		}
		name = name + fmt.Sprintf("-%d", counter)
	}
	return name
}

// NewWorker returns a worker
func NewWorker(api *discoverspaces.API) (worker.Worker, chan struct{}) {
	dw := &discoverspacesWorker{
		api:               api,
		discoveringSpaces: make(chan struct{}),
	}
	go func() {
		defer dw.tomb.Done()
		dw.tomb.Kill(dw.loop())
	}()
	return dw, dw.discoveringSpaces
}

func (dw *discoverspacesWorker) Kill() {
	dw.tomb.Kill(nil)
}

func (dw *discoverspacesWorker) Wait() error {
	return dw.tomb.Wait()
}

func (dw *discoverspacesWorker) loop() (err error) {
	modelCfg, err := dw.api.ModelConfig()
	if err != nil {
		return err
	}
	model, err := environs.New(modelCfg)
	if err != nil {
		return err
	}
	networkingModel, ok := environs.SupportsNetworking(model)

	if ok {
		err = dw.handleSubnets(networkingModel)
		if err != nil {
			close(dw.discoveringSpaces)
			return errors.Trace(err)
		}
	}
	close(dw.discoveringSpaces)

	// TODO(mfoord): we'll have a watcher here checking if we need to
	// update the spaces/subnets definition.
	dying := dw.tomb.Dying()
	for {
		select {
		case <-dying:
			return nil
		}
	}
	return nil
}

func (dw *discoverspacesWorker) handleSubnets(env environs.NetworkingEnviron) error {
	ok, err := env.SupportsSpaceDiscovery()
	if err != nil {
		return errors.Trace(err)
	}
	if !ok {
		// Nothing to do.
		return nil
	}
	providerSpaces, err := env.Spaces()
	if err != nil {
		return errors.Trace(err)
	}
	listSpacesResult, err := dw.api.ListSpaces()
	if err != nil {
		return errors.Trace(err)
	}

	stateSubnets, err := dw.api.ListSubnets(params.SubnetsFilters{})
	if err != nil {
		return errors.Trace(err)
	}
	stateSubnetIds := make(set.Strings)
	for _, subnet := range stateSubnets.Results {
		stateSubnetIds.Add(subnet.ProviderId)
	}
	stateSpaceMap := make(map[string]params.ProviderSpace)
	spaceNames := make(set.Strings)
	for _, space := range listSpacesResult.Results {
		stateSpaceMap[space.ProviderId] = space
		spaceNames.Add(space.Name)
	}

	// TODO(mfoord): we need to delete spaces and subnets that no longer
	// exist, so long as they're not in use.
	for _, space := range providerSpaces {
		// Check if the space is already in state, in which case we know
		// its name.
		stateSpace, ok := stateSpaceMap[string(space.ProviderId)]
		var spaceTag names.SpaceTag
		if ok {
			spaceName := stateSpace.Name
			if !names.IsValidSpace(spaceName) {
				// Can only happen if an invalid name is stored
				// in state.
				logger.Errorf("space %q has an invalid name, ignoring", spaceName)
				continue

			}
			spaceTag = names.NewSpaceTag(spaceName)

		} else {
			// The space is new, we need to create a valid name for it
			// in state.
			spaceName := string(space.ProviderId)
			// Convert the name into a valid name that isn't already in
			// use.
			spaceName = convertSpaceName(spaceName, spaceNames)
			spaceNames.Add(spaceName)
			spaceTag = names.NewSpaceTag(spaceName)
			// We need to create the space.
			args := params.CreateSpacesParams{
				Spaces: []params.CreateSpaceParams{{
					Public:     false,
					SpaceTag:   spaceTag.String(),
					ProviderId: string(space.ProviderId),
				}}}
			result, err := dw.api.CreateSpaces(args)
			if err != nil {
				logger.Errorf("error creating space %v", err)
				return errors.Trace(err)
			}
			if len(result.Results) != 1 {
				return errors.Errorf("unexpected number of results from CreateSpaces, should be 1: %v", result)
			}
			if result.Results[0].Error != nil {
				return errors.Errorf("error from CreateSpaces: %v", result.Results[0].Error)
			}
		}
		// TODO(mfoord): currently no way of removing subnets, or
		// changing the space they're in, so we can only add ones we
		// don't already know about.
		logger.Debugf("Created space %v with %v subnets", spaceTag.String(), len(space.Subnets))
		for _, subnet := range space.Subnets {
			if stateSubnetIds.Contains(string(subnet.ProviderId)) {
				continue
			}
			zones := subnet.AvailabilityZones
			if len(zones) == 0 {
				zones = []string{"default"}
			}
			args := params.AddSubnetsParams{
				Subnets: []params.AddSubnetParams{{
					SubnetProviderId: string(subnet.ProviderId),
					SpaceTag:         spaceTag.String(),
					Zones:            zones,
				}}}
			logger.Tracef("Adding subnet %v", subnet.CIDR)
			result, err := dw.api.AddSubnets(args)
			if err != nil {
				logger.Errorf("invalid creating subnet %v", err)
				return errors.Trace(err)
			}
			if len(result.Results) != 1 {
				return errors.Errorf("unexpected number of results from AddSubnets, should be 1: %v", result)
			}
			if result.Results[0].Error != nil {
				logger.Errorf("error creating subnet %v", result.Results[0].Error)
				return errors.Errorf("error creating subnet %v", result.Results[0].Error)
			}
		}
	}
	return nil
}
<commit_msg>Close discovery channel in event of errors<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package discoverspaces

import (
	"fmt"
	"regexp"
	"strings"

	"github.com/juju/errors"
	"github.com/juju/loggo"
	"github.com/juju/names"
	"github.com/juju/utils/set"
	"launchpad.net/tomb"

	"github.com/juju/juju/api/discoverspaces"
	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/environs"
	"github.com/juju/juju/network"
	"github.com/juju/juju/worker"
)

var logger = loggo.GetLogger("juju.discoverspaces")

type discoverspacesWorker struct {
	api               *discoverspaces.API
	tomb              tomb.Tomb
	discoveringSpaces chan struct{}
}

var dashPrefix = regexp.MustCompile("^-*")
var dashSuffix = regexp.MustCompile("-*$")
var multipleDashes = regexp.MustCompile("--+")

func convertSpaceName(name string, existing set.Strings) string {
	// First lower case and replace spaces with dashes.
	name = strings.Replace(name, " ", "-", -1)
	name = strings.ToLower(name)
	// Replace any character that isn't in the set "-", "a-z", "0-9".
	name = network.SpaceInvalidChars.ReplaceAllString(name, "")
	// Get rid of any dashes at the start as that isn't valid.
	name = dashPrefix.ReplaceAllString(name, "")
	// And any at the end.
	name = dashSuffix.ReplaceAllString(name, "")
	// Repleace multiple dashes with a single dash.
	name = multipleDashes.ReplaceAllString(name, "-")
	// Special case of when the space name was only dashes or invalid
	// characters!
	if name == "" {
		name = "empty"
	}
	// If this name is in use add a numerical suffix.
	if existing.Contains(name) {
		counter := 2
		for existing.Contains(name + fmt.Sprintf("-%d", counter)) {
			counter += 1
		}
		name = name + fmt.Sprintf("-%d", counter)
	}
	return name
}

// NewWorker returns a worker
func NewWorker(api *discoverspaces.API) (worker.Worker, chan struct{}) {
	dw := &discoverspacesWorker{
		api:               api,
		discoveringSpaces: make(chan struct{}),
	}
	go func() {
		defer dw.tomb.Done()
		dw.tomb.Kill(dw.loop())
	}()
	return dw, dw.discoveringSpaces
}

func (dw *discoverspacesWorker) Kill() {
	dw.tomb.Kill(nil)
}

func (dw *discoverspacesWorker) Wait() error {
	return dw.tomb.Wait()
}

func (dw *discoverspacesWorker) loop() (err error) {
	modelCfg, err := dw.api.ModelConfig()
	if err != nil {
		close(dw.discoveringSpaces)
		return err
	}
	model, err := environs.New(modelCfg)
	if err != nil {
		close(dw.discoveringSpaces)
		return err
	}
	networkingModel, ok := environs.SupportsNetworking(model)

	if ok {
		err = dw.handleSubnets(networkingModel)
		if err != nil {
			close(dw.discoveringSpaces)
			return errors.Trace(err)
		}
	}
	close(dw.discoveringSpaces)

	// TODO(mfoord): we'll have a watcher here checking if we need to
	// update the spaces/subnets definition.
	dying := dw.tomb.Dying()
	for {
		select {
		case <-dying:
			return nil
		}
	}
	return nil
}

func (dw *discoverspacesWorker) handleSubnets(env environs.NetworkingEnviron) error {
	ok, err := env.SupportsSpaceDiscovery()
	if err != nil {
		return errors.Trace(err)
	}
	if !ok {
		// Nothing to do.
		return nil
	}
	providerSpaces, err := env.Spaces()
	if err != nil {
		return errors.Trace(err)
	}
	listSpacesResult, err := dw.api.ListSpaces()
	if err != nil {
		return errors.Trace(err)
	}

	stateSubnets, err := dw.api.ListSubnets(params.SubnetsFilters{})
	if err != nil {
		return errors.Trace(err)
	}
	stateSubnetIds := make(set.Strings)
	for _, subnet := range stateSubnets.Results {
		stateSubnetIds.Add(subnet.ProviderId)
	}
	stateSpaceMap := make(map[string]params.ProviderSpace)
	spaceNames := make(set.Strings)
	for _, space := range listSpacesResult.Results {
		stateSpaceMap[space.ProviderId] = space
		spaceNames.Add(space.Name)
	}

	// TODO(mfoord): we need to delete spaces and subnets that no longer
	// exist, so long as they're not in use.
	for _, space := range providerSpaces {
		// Check if the space is already in state, in which case we know
		// its name.
		stateSpace, ok := stateSpaceMap[string(space.ProviderId)]
		var spaceTag names.SpaceTag
		if ok {
			spaceName := stateSpace.Name
			if !names.IsValidSpace(spaceName) {
				// Can only happen if an invalid name is stored
				// in state.
				logger.Errorf("space %q has an invalid name, ignoring", spaceName)
				continue

			}
			spaceTag = names.NewSpaceTag(spaceName)

		} else {
			// The space is new, we need to create a valid name for it
			// in state.
			spaceName := string(space.ProviderId)
			// Convert the name into a valid name that isn't already in
			// use.
			spaceName = convertSpaceName(spaceName, spaceNames)
			spaceNames.Add(spaceName)
			spaceTag = names.NewSpaceTag(spaceName)
			// We need to create the space.
			args := params.CreateSpacesParams{
				Spaces: []params.CreateSpaceParams{{
					Public:     false,
					SpaceTag:   spaceTag.String(),
					ProviderId: string(space.ProviderId),
				}}}
			result, err := dw.api.CreateSpaces(args)
			if err != nil {
				logger.Errorf("error creating space %v", err)
				return errors.Trace(err)
			}
			if len(result.Results) != 1 {
				return errors.Errorf("unexpected number of results from CreateSpaces, should be 1: %v", result)
			}
			if result.Results[0].Error != nil {
				return errors.Errorf("error from CreateSpaces: %v", result.Results[0].Error)
			}
		}
		// TODO(mfoord): currently no way of removing subnets, or
		// changing the space they're in, so we can only add ones we
		// don't already know about.
		logger.Debugf("Created space %v with %v subnets", spaceTag.String(), len(space.Subnets))
		for _, subnet := range space.Subnets {
			if stateSubnetIds.Contains(string(subnet.ProviderId)) {
				continue
			}
			zones := subnet.AvailabilityZones
			if len(zones) == 0 {
				zones = []string{"default"}
			}
			args := params.AddSubnetsParams{
				Subnets: []params.AddSubnetParams{{
					SubnetProviderId: string(subnet.ProviderId),
					SpaceTag:         spaceTag.String(),
					Zones:            zones,
				}}}
			logger.Tracef("Adding subnet %v", subnet.CIDR)
			result, err := dw.api.AddSubnets(args)
			if err != nil {
				logger.Errorf("invalid creating subnet %v", err)
				return errors.Trace(err)
			}
			if len(result.Results) != 1 {
				return errors.Errorf("unexpected number of results from AddSubnets, should be 1: %v", result)
			}
			if result.Results[0].Error != nil {
				logger.Errorf("error creating subnet %v", result.Results[0].Error)
				return errors.Errorf("error creating subnet %v", result.Results[0].Error)
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package ohetcd

import (
	"github.com/coreos/etcd/client"
	"golang.org/x/net/context"
	"gopkg.in/yaml.v2"
	"log"
	"strings"
)

type Linkable interface {
	Set()
	Update()
	Save()
	Link()
	Unlink()
	Watch()
	Unwatch()
}

type Data struct {
	Directory string
	Object    interface{}
	Deep      bool
}

func NewData() *Data {
	return &Data{}
}

// set etcd path for data
func (d *Data) Set(dir string, object interface{}, deep bool) {
	d.Directory = dir
	d.Object = object
	d.Deep = deep
}

// get up-to-date value from etcd
func (d *Data) Update() {
	if d.Deep {
		deepRetrieve(d.Directory, d.Object)
		return
	}
	resp, err := kapi.Get(context.Background(), d.Directory, &client.GetOptions{
		Recursive: true,
	})
	if err != nil {
		if strings.Contains(err.Error(), string(client.ErrorCodeKeyNotFound)) {
			// if key not found, init etcd with current variable
			d.Save()
			return
		}
		log.Println(err)
	}
	data := resp.Node.Value
	err = yaml.Unmarshal([]byte(data), d.Object)
	if err != nil {
		log.Println(err)
	}
}

// changes made, save to etcd
func (d *Data) Save() error {
	if d.Deep {
		deepSave(d.Directory, d.Object)
		return
	}
	val, err := yaml.Marshal(d.Object)
	if err != nil {
		return err
	}
	_, err = kapi.Set(context.Background(), d.Directory, string(val), &client.SetOptions{})
	if err != nil {
		return err
	}
	return nil
}

func (d *Data) Link() {
	ch := make(chan int)
	// save ch into global chMap
	heartBeatChMap[d] = ch
	go func(d *Data, ch chan int) {
		var i int
		for {
			i = <-ch
			if i == CH_CLOSE_SIG {
				break
			} else {
				d.Update()
			}
		}
	}(d, ch)
}

func (d *Data) Unlink() {
	heartBeatChMap[d] <- CH_CLOSE_SIG
}

func (d *Data) Watch() {
	watcher := kapi.Watcher(d.Directory, &client.WatcherOptions{
		Recursive: true,
	})
	ch := make(chan int)
	watchChMap[d] = ch
	go func(watcher client.Watcher, d *Data, ch chan int) {
		for {
			select {
			case <-ch:
				return
			default:
				resp, err := watcher.Next(context.Background())
				if err != nil {
					log.Println(err)
				} else {
					log.Println(resp)
					d.Update()
				}
			}
		}
	}(watcher, d, ch)
}

func (d *Data) Unwatch() {
	watchChMap[d] <- CH_CLOSE_SIG
}
<commit_msg>fix return err<commit_after>package ohetcd

import (
	"github.com/coreos/etcd/client"
	"golang.org/x/net/context"
	"gopkg.in/yaml.v2"
	"log"
	"strings"
)

type Linkable interface {
	Set()
	Update()
	Save()
	Link()
	Unlink()
	Watch()
	Unwatch()
}

type Data struct {
	Directory string
	Object    interface{}
	Deep      bool
}

func NewData() *Data {
	return &Data{}
}

// set etcd path for data
func (d *Data) Set(dir string, object interface{}, deep bool) {
	d.Directory = dir
	d.Object = object
	d.Deep = deep
}

// get up-to-date value from etcd
func (d *Data) Update() {
	if d.Deep {
		deepRetrieve(d.Directory, d.Object)
		return
	}
	resp, err := kapi.Get(context.Background(), d.Directory, &client.GetOptions{
		Recursive: true,
	})
	if err != nil {
		if strings.Contains(err.Error(), string(client.ErrorCodeKeyNotFound)) {
			// if key not found, init etcd with current variable
			d.Save()
			return
		}
		log.Println(err)
	}
	data := resp.Node.Value
	err = yaml.Unmarshal([]byte(data), d.Object)
	if err != nil {
		log.Println(err)
	}
}

// changes made, save to etcd
func (d *Data) Save() error {
	if d.Deep {
		return deepSave(d.Directory, d.Object)
	}
	val, err := yaml.Marshal(d.Object)
	if err != nil {
		return err
	}
	_, err = kapi.Set(context.Background(), d.Directory, string(val), &client.SetOptions{})
	if err != nil {
		return err
	}
	return nil
}

func (d *Data) Link() {
	ch := make(chan int)
	// save ch into global chMap
	heartBeatChMap[d] = ch
	go func(d *Data, ch chan int) {
		var i int
		for {
			i = <-ch
			if i == CH_CLOSE_SIG {
				break
			} else {
				d.Update()
			}
		}
	}(d, ch)
}

func (d *Data) Unlink() {
	heartBeatChMap[d] <- CH_CLOSE_SIG
}

func (d *Data) Watch() {
	watcher := kapi.Watcher(d.Directory, &client.WatcherOptions{
		Recursive: true,
	})
	ch := make(chan int)
	watchChMap[d] = ch
	go func(watcher client.Watcher, d *Data, ch chan int) {
		for {
			select {
			case <-ch:
				return
			default:
				resp, err := watcher.Next(context.Background())
				if err != nil {
					log.Println(err)
				} else {
					log.Println(resp)
					d.Update()
				}
			}
		}
	}(watcher, d, ch)
}

func (d *Data) Unwatch() {
	watchChMap[d] <- CH_CLOSE_SIG
}
<|endoftext|>


<commit_before>// +build linux

package sh

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
	"testing"
)

var (
	enableUserNS bool
)

func init() {
	const usernsOk = "1"
	const kernelcfg = "CONFIG_USER_NS"

	logUsernsDetection := func(err error) {
		if enableUserNS {
			fmt.Printf("Linux user namespaces enabled!")
			return
		}

		fmt.Printf("Warning: Impossible to know if kernel support USER namespace.\n")
		fmt.Printf("Warning: USER namespace tests will not run.\n")
		if err != nil {
			fmt.Printf("ERROR: %s\n", err)
		}
	}

	usernsCfg := "/proc/sys/kernel/unprivileged_userns_clone"
	val, permerr := ioutil.ReadFile(usernsCfg)

	// Travis build doesn't support /proc/config.gz but kernel has userns
	if os.Getenv("TRAVIS_BUILD") == usernsOk {
		enableUserNS = permerr == nil && string(val) == usernsOk
		logUsernsDetection(permerr)
		return
	}

	if permerr == nil {
		enableUserNS = string(val) == usernsOk
		logUsernsDetection(permerr)
		return
	}

	// old kernels dont has sysctl configurations
	// than just checking the /proc/config suffices
	usernsCmd := exec.Command("zgrep", kernelcfg, "/proc/config.gz")

	content, err := usernsCmd.CombinedOutput()
	if err != nil {
		enableUserNS = false
		logUsernsDetection(fmt.Errorf("Failed to get kernel config: %s", err))
		return
	}

	cfgVal := strings.Trim(string(content), "\n\t ")
	enableUserNS = cfgVal == kernelcfg+"=y"
	logUsernsDetection(fmt.Errorf("%s not enabled in kernel config", kernelcfg))
}

func TestExecuteRforkUserNS(t *testing.T) {
	if !enableUserNS {
		t.Skip("User namespace not enabled")
		return
	}

	var out bytes.Buffer
	f, teardown := setup(t)
	defer teardown()

	sh, err := NewShell()

	if err != nil {
		t.Error(err)
		return
	}

	sh.SetNashdPath(f.nashdPath)
	sh.SetStdout(&out)

	err = sh.Exec("rfork test", `
        rfork u {
            id -u
        }
        `)

	if err != nil {
		t.Error(err)
		return
	}

	if string(out.Bytes()) != "0\n" {
		t.Errorf("User namespace not supported in your kernel: %s", string(out.Bytes()))
		return
	}
}

func TestExecuteRforkEnvVars(t *testing.T) {
	if !enableUserNS {
		t.Skip("User namespace not enabled")
		return
	}

	f, teardown := setup(t)
	defer teardown()

	sh, err := NewShell()

	if err != nil {
		t.Error(err)
		return
	}

	sh.SetNashdPath(f.nashdPath)

	err = sh.Exec("test env", `abra = "cadabra"
setenv abra
rfork up {
	echo $abra
}`)

	if err != nil {
		t.Error(err)
		return
	}
}

func TestExecuteRforkUserNSNested(t *testing.T) {
	if !enableUserNS {
		t.Skip("User namespace not enabled")
		return
	}

	var out bytes.Buffer
	f, teardown := setup(t)
	defer teardown()

	sh, err := NewShell()

	if err != nil {
		t.Error(err)
		return
	}

	sh.SetNashdPath(f.nashdPath)
	sh.SetStdout(&out)

	err = sh.Exec("rfork userns nested", `
        rfork u {
            id -u
            rfork u {
                id -u
            }
        }
        `)

	if err != nil {
		t.Error(err)
		return
	}

	if string(out.Bytes()) != "0\n0\n" {
		t.Errorf("User namespace not supported in your kernel")
		return
	}
}
<commit_msg>fix travis_build detection<commit_after>// +build linux

package sh

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
	"testing"
)

var (
	enableUserNS bool
)

func init() {
	const usernsOk = "1"
	const kernelcfg = "CONFIG_USER_NS"

	logUsernsDetection := func(err error) {
		if enableUserNS {
			fmt.Printf("Linux user namespaces enabled!")
			return
		}

		fmt.Printf("Warning: Impossible to know if kernel support USER namespace.\n")
		fmt.Printf("Warning: USER namespace tests will not run.\n")
		if err != nil {
			fmt.Printf("ERROR: %s\n", err)
		}
	}

	usernsCfg := "/proc/sys/kernel/unprivileged_userns_clone"
	val, permerr := ioutil.ReadFile(usernsCfg)

	// Travis build doesn't support /proc/config.gz but kernel has userns
	if os.Getenv("TRAVIS_BUILD") == "1" {
		enableUserNS = permerr == nil && string(val) == usernsOk
		logUsernsDetection(permerr)
		return
	}

	if permerr == nil {
		enableUserNS = string(val) == usernsOk
		logUsernsDetection(permerr)
		return
	}

	// old kernels dont has sysctl configurations
	// than just checking the /proc/config suffices
	usernsCmd := exec.Command("zgrep", kernelcfg, "/proc/config.gz")

	content, err := usernsCmd.CombinedOutput()
	if err != nil {
		enableUserNS = false
		logUsernsDetection(fmt.Errorf("Failed to get kernel config: %s", err))
		return
	}

	cfgVal := strings.Trim(string(content), "\n\t ")
	enableUserNS = cfgVal == kernelcfg+"=y"
	logUsernsDetection(fmt.Errorf("%s not enabled in kernel config", kernelcfg))
}

func TestExecuteRforkUserNS(t *testing.T) {
	if !enableUserNS {
		t.Skip("User namespace not enabled")
		return
	}

	var out bytes.Buffer
	f, teardown := setup(t)
	defer teardown()

	sh, err := NewShell()

	if err != nil {
		t.Error(err)
		return
	}

	sh.SetNashdPath(f.nashdPath)
	sh.SetStdout(&out)

	err = sh.Exec("rfork test", `
        rfork u {
            id -u
        }
        `)

	if err != nil {
		t.Error(err)
		return
	}

	if string(out.Bytes()) != "0\n" {
		t.Errorf("User namespace not supported in your kernel: %s", string(out.Bytes()))
		return
	}
}

func TestExecuteRforkEnvVars(t *testing.T) {
	if !enableUserNS {
		t.Skip("User namespace not enabled")
		return
	}

	f, teardown := setup(t)
	defer teardown()

	sh, err := NewShell()

	if err != nil {
		t.Error(err)
		return
	}

	sh.SetNashdPath(f.nashdPath)

	err = sh.Exec("test env", `abra = "cadabra"
setenv abra
rfork up {
	echo $abra
}`)

	if err != nil {
		t.Error(err)
		return
	}
}

func TestExecuteRforkUserNSNested(t *testing.T) {
	if !enableUserNS {
		t.Skip("User namespace not enabled")
		return
	}

	var out bytes.Buffer
	f, teardown := setup(t)
	defer teardown()

	sh, err := NewShell()

	if err != nil {
		t.Error(err)
		return
	}

	sh.SetNashdPath(f.nashdPath)
	sh.SetStdout(&out)

	err = sh.Exec("rfork userns nested", `
        rfork u {
            id -u
            rfork u {
                id -u
            }
        }
        `)

	if err != nil {
		t.Error(err)
		return
	}

	if string(out.Bytes()) != "0\n0\n" {
		t.Errorf("User namespace not supported in your kernel")
		return
	}
}
<|endoftext|>


<commit_before>// Package stmtcache is a cache that can be used to implement lazy prepared statements.
package stmtcache

import (
	"strconv"
	"sync/atomic"

	"github.com/jackc/pgx/v5/pgconn"
)

var stmtCounter int64

// NextStatementName returns a statement name that will be unique for the lifetime of the program.
func NextStatementName() string {
	n := atomic.AddInt64(&stmtCounter, 1)
	return "stmtcache_" + strconv.FormatInt(n, 10)
}

// Cache caches statement descriptions.
type Cache interface {
	// Get returns the statement description for sql. Returns nil if not found.
	Get(sql string) *pgconn.StatementDescription

	// Put stores sd in the cache. Put panics if sd.SQL is "". Put does nothing if sd.SQL already exists in the cache.
	Put(sd *pgconn.StatementDescription)

	// Invalidate invalidates statement description identified by sql. Does nothing if not found.
	Invalidate(sql string)

	// InvalidateAll invalidates all statement descriptions.
	InvalidateAll()

	// HandleInvalidated returns a slice of all statement descriptions invalidated since the last call to HandleInvalidated.
	HandleInvalidated() []*pgconn.StatementDescription

	// Len returns the number of cached prepared statement descriptions.
	Len() int

	// Cap returns the maximum number of cached prepared statement descriptions.
	Cap() int
}

func IsStatementInvalid(err error) bool {
	pgErr, ok := err.(*pgconn.PgError)
	if !ok {
		return false
	}

	// https://github.com/jackc/pgx/issues/1162
	//
	// We used to look for the message "cached plan must not change result type". However, that message can be localized.
	// Unfortunately, error code "0A000" - "FEATURE NOT SUPPORTED" is used for many different errors and the only way to
	// tell the difference is by the message. But all that happens is we clear a statement that we otherwise wouldn't
	// have so it should be safe.
	possibleInvalidCachedPlanError := pgErr.Code == "0A000"
	return possibleInvalidCachedPlanError
}
<commit_msg>Update docs<commit_after>// Package stmtcache is a cache for statement descriptions.
package stmtcache

import (
	"strconv"
	"sync/atomic"

	"github.com/jackc/pgx/v5/pgconn"
)

var stmtCounter int64

// NextStatementName returns a statement name that will be unique for the lifetime of the program.
func NextStatementName() string {
	n := atomic.AddInt64(&stmtCounter, 1)
	return "stmtcache_" + strconv.FormatInt(n, 10)
}

// Cache caches statement descriptions.
type Cache interface {
	// Get returns the statement description for sql. Returns nil if not found.
	Get(sql string) *pgconn.StatementDescription

	// Put stores sd in the cache. Put panics if sd.SQL is "". Put does nothing if sd.SQL already exists in the cache.
	Put(sd *pgconn.StatementDescription)

	// Invalidate invalidates statement description identified by sql. Does nothing if not found.
	Invalidate(sql string)

	// InvalidateAll invalidates all statement descriptions.
	InvalidateAll()

	// HandleInvalidated returns a slice of all statement descriptions invalidated since the last call to HandleInvalidated.
	HandleInvalidated() []*pgconn.StatementDescription

	// Len returns the number of cached prepared statement descriptions.
	Len() int

	// Cap returns the maximum number of cached prepared statement descriptions.
	Cap() int
}

func IsStatementInvalid(err error) bool {
	pgErr, ok := err.(*pgconn.PgError)
	if !ok {
		return false
	}

	// https://github.com/jackc/pgx/issues/1162
	//
	// We used to look for the message "cached plan must not change result type". However, that message can be localized.
	// Unfortunately, error code "0A000" - "FEATURE NOT SUPPORTED" is used for many different errors and the only way to
	// tell the difference is by the message. But all that happens is we clear a statement that we otherwise wouldn't
	// have so it should be safe.
	possibleInvalidCachedPlanError := pgErr.Code == "0A000"
	return possibleInvalidCachedPlanError
}
<|endoftext|>