<commit_before>package device

import (
	"errors"
	"fmt"
	"github.com/Comcast/webpa-common/logging"
	"github.com/stretchr/testify/assert"
	"net/http"
	"net/http/httptest"
	"net/url"
	"sync"
	"testing"
)

var (
	testDeviceIDs = map[ID]int{
		IntToMAC(0xDEADBEEF):     1,
		IntToMAC(0x112233445566): 1,
		IntToMAC(0xFE881212CDCD): 2,
		IntToMAC(0x7F551928ABCD): 3,
	}

	testConnectionCount = func() (total int) {
		for _, connectionCount := range testDeviceIDs {
			total += connectionCount
		}

		return
	}()
)

// startWebsocketServer sets up a server-side environment for testing device-related websocket code
func startWebsocketServer(o *Options) (Manager, *httptest.Server, string) {
	manager := NewManager(o, nil)
	server := httptest.NewServer(
		NewConnectHandler(
			manager,
			nil,
			o.logger(),
		),
	)

	websocketURL, err := url.Parse(server.URL)
	if err != nil {
		server.Close()
		panic(fmt.Errorf("Unable to parse test server URL: %s", err))
	}

	websocketURL.Scheme = "ws"
	return manager, server, websocketURL.String()
}

func connectTestDevices(t *testing.T, assert *assert.Assertions, dialer Dialer, connectURL string) map[ID][]Connection {
	devices := make(map[ID][]Connection, len(testDeviceIDs))

	for id, connectionCount := range testDeviceIDs {
		connections := make([]Connection, 0, connectionCount)
		for repeat := 0; repeat < connectionCount; repeat++ {
			deviceConnection, response, err := dialer.Dial(connectURL, id, nil, nil)
			if assert.NotNil(deviceConnection) && assert.NotNil(response) && assert.Nil(err) {
				connections = append(connections, deviceConnection)
			} else {
				t.FailNow()
			}
		}

		devices[id] = connections
	}

	return devices
}

func closeTestDevices(assert *assert.Assertions, devices map[ID][]Connection) {
	for _, connections := range devices {
		for _, connection := range connections {
			assert.Nil(connection.Close())
		}
	}
}

func TestManagerConnectMissingDeviceNameHeader(t *testing.T) {
	assert := assert.New(t)
	options := &Options{
		Logger: logging.TestLogger(t),
	}

	manager := NewManager(options, nil)
	response := httptest.NewRecorder()
	request := httptest.NewRequest("POST", "http://localhost.com", nil)

	device, err := manager.Connect(response, request, nil)
	assert.Nil(device)
	assert.NotNil(err)
	assert.Equal(response.Code, http.StatusBadRequest)
}

func TestManagerBadDeviceNameHeader(t *testing.T) {
	assert := assert.New(t)
	options := &Options{
		Logger: logging.TestLogger(t),
	}

	manager := NewManager(options, nil)
	response := httptest.NewRecorder()
	request := httptest.NewRequest("POST", "http://localhost.com", nil)
	request.Header.Set(DefaultDeviceNameHeader, "this is not valid")

	device, err := manager.Connect(response, request, nil)
	assert.Nil(device)
	assert.NotNil(err)
	assert.Equal(response.Code, http.StatusBadRequest)
}

func TestManagerBadConveyHeader(t *testing.T) {
	assert := assert.New(t)
	options := &Options{
		Logger: logging.TestLogger(t),
	}

	manager := NewManager(options, nil)
	response := httptest.NewRecorder()
	request := httptest.NewRequest("POST", "http://localhost.com", nil)
	request.Header.Set(DefaultDeviceNameHeader, "mac:112233445566")
	request.Header.Set(DefaultConveyHeader, "this is not valid")

	device, err := manager.Connect(response, request, nil)
	assert.Nil(device)
	assert.NotNil(err)
	assert.Equal(response.Code, http.StatusBadRequest)
}

func TestManagerKeyError(t *testing.T) {
	assert := assert.New(t)
	badKeyFunc := func(ID, Convey, *http.Request) (Key, error) {
		return invalidKey, errors.New("expected")
	}

	options := &Options{
		Logger:  logging.TestLogger(t),
		KeyFunc: badKeyFunc,
	}

	manager := NewManager(options, nil)
	response := httptest.NewRecorder()
	request := httptest.NewRequest("POST", "http://localhost.com", nil)
	request.Header.Set(DefaultDeviceNameHeader, "mac:112233445566")

	device, err := manager.Connect(response, request, nil)
	assert.Nil(device)
	assert.NotNil(err)
	assert.Equal(response.Code, http.StatusBadRequest)
}

func TestManagerPongCallbackFor(t *testing.T) {
	assert := assert.New(t)
	expectedDevice := newDevice(ID("ponged device"), Key("expected"), nil, 1)
	expectedData := "expected pong data"
	listenerCalled := false

	manager := &manager{
		logger: logging.TestLogger(t),
		pongListener: func(actualDevice Interface, actualData string) {
			listenerCalled = true
			assert.True(expectedDevice == actualDevice)
			assert.Equal(expectedData, actualData)
		},
	}

	pongCallback := manager.pongCallbackFor(expectedDevice)
	pongCallback(expectedData)
	assert.True(listenerCalled)
}

func TestManagerConnectListener(t *testing.T) {
	assert := assert.New(t)
	connections := make(chan Interface, testConnectionCount)

	options := &Options{
		Logger: logging.TestLogger(t),
		ConnectListener: func(candidate Interface) {
			select {
			case connections <- candidate:
			default:
				assert.Fail("Too many connect listener invocations")
			}
		},
	}

	_, server, connectURL := startWebsocketServer(options)
	defer server.Close()

	dialer := NewDialer(options, nil)
	testDevices := connectTestDevices(t, assert, dialer, connectURL)
	defer closeTestDevices(assert, testDevices)

	close(connections)
	assert.Equal(testConnectionCount, len(connections))
	visitedDeviceIDs := make(map[ID]int)
	for connected := range connections {
		visitedDeviceIDs[connected.ID()] += 1
	}

	assert.Equal(testDeviceIDs, visitedDeviceIDs)
}

func TestManagerDisconnect(t *testing.T) {
	assert := assert.New(t)
	disconnectWait := new(sync.WaitGroup)
	disconnectWait.Add(testConnectionCount)
	disconnected := make(map[Key]Interface)

	options := &Options{
		Logger: logging.TestLogger(t),
		DisconnectListener: func(candidate Interface) {
			disconnectWait.Done()
			disconnected[candidate.Key()] = candidate
			assert.True(candidate.Closed())
		},
	}

	manager, server, connectURL := startWebsocketServer(options)
	defer server.Close()

	dialer := NewDialer(options, nil)
	testDevices := connectTestDevices(t, assert, dialer, connectURL)
	defer closeTestDevices(assert, testDevices)

	for id, connectionCount := range testDeviceIDs {
		assert.Equal(connectionCount, manager.Disconnect(id))
	}

	disconnectWait.Wait()
	assert.Equal(testConnectionCount, len(disconnected))
}
<commit_msg>Fixed two race conditions in test code<commit_after>package device

import (
	"errors"
	"fmt"
	"github.com/Comcast/webpa-common/logging"
	"github.com/stretchr/testify/assert"
	"net/http"
	"net/http/httptest"
	"net/url"
	"sync"
	"testing"
)

var (
	testDeviceIDs = map[ID]int{
		IntToMAC(0xDEADBEEF):     1,
		IntToMAC(0x112233445566): 1,
		IntToMAC(0xFE881212CDCD): 2,
		IntToMAC(0x7F551928ABCD): 3,
	}

	testConnectionCount = func() (total int) {
		for _, connectionCount := range testDeviceIDs {
			total += connectionCount
		}

		return
	}()
)

// startWebsocketServer sets up a server-side environment for testing device-related websocket code
func startWebsocketServer(o *Options) (Manager, *httptest.Server, string) {
	manager := NewManager(o, nil)
	server := httptest.NewServer(
		NewConnectHandler(
			manager,
			nil,
			o.logger(),
		),
	)

	websocketURL, err := url.Parse(server.URL)
	if err != nil {
		server.Close()
		panic(fmt.Errorf("Unable to parse test server URL: %s", err))
	}

	websocketURL.Scheme = "ws"
	return manager, server, websocketURL.String()
}

func connectTestDevices(t *testing.T, assert *assert.Assertions, dialer Dialer, connectURL string) map[ID][]Connection {
	devices := make(map[ID][]Connection, len(testDeviceIDs))

	for id, connectionCount := range testDeviceIDs {
		connections := make([]Connection, 0, connectionCount)
		for repeat := 0; repeat < connectionCount; repeat++ {
			deviceConnection, response, err := dialer.Dial(connectURL, id, nil, nil)
			if assert.NotNil(deviceConnection) && assert.NotNil(response) && assert.Nil(err) {
				connections = append(connections, deviceConnection)
			} else {
				t.FailNow()
			}
		}

		devices[id] = connections
	}

	return devices
}

func closeTestDevices(assert *assert.Assertions, devices map[ID][]Connection) {
	for _, connections := range devices {
		for _, connection := range connections {
			assert.Nil(connection.Close())
		}
	}
}

func TestManagerConnectMissingDeviceNameHeader(t *testing.T) {
	assert := assert.New(t)
	options := &Options{
		Logger: logging.TestLogger(t),
	}

	manager := NewManager(options, nil)
	response := httptest.NewRecorder()
	request := httptest.NewRequest("POST", "http://localhost.com", nil)

	device, err := manager.Connect(response, request, nil)
	assert.Nil(device)
	assert.NotNil(err)
	assert.Equal(response.Code, http.StatusBadRequest)
}

func TestManagerBadDeviceNameHeader(t *testing.T) {
	assert := assert.New(t)
	options := &Options{
		Logger: logging.TestLogger(t),
	}

	manager := NewManager(options, nil)
	response := httptest.NewRecorder()
	request := httptest.NewRequest("POST", "http://localhost.com", nil)
	request.Header.Set(DefaultDeviceNameHeader, "this is not valid")

	device, err := manager.Connect(response, request, nil)
	assert.Nil(device)
	assert.NotNil(err)
	assert.Equal(response.Code, http.StatusBadRequest)
}

func TestManagerBadConveyHeader(t *testing.T) {
	assert := assert.New(t)
	options := &Options{
		Logger: logging.TestLogger(t),
	}

	manager := NewManager(options, nil)
	response := httptest.NewRecorder()
	request := httptest.NewRequest("POST", "http://localhost.com", nil)
	request.Header.Set(DefaultDeviceNameHeader, "mac:112233445566")
	request.Header.Set(DefaultConveyHeader, "this is not valid")

	device, err := manager.Connect(response, request, nil)
	assert.Nil(device)
	assert.NotNil(err)
	assert.Equal(response.Code, http.StatusBadRequest)
}

func TestManagerKeyError(t *testing.T) {
	assert := assert.New(t)
	badKeyFunc := func(ID, Convey, *http.Request) (Key, error) {
		return invalidKey, errors.New("expected")
	}

	options := &Options{
		Logger:  logging.TestLogger(t),
		KeyFunc: badKeyFunc,
	}

	manager := NewManager(options, nil)
	response := httptest.NewRecorder()
	request := httptest.NewRequest("POST", "http://localhost.com", nil)
	request.Header.Set(DefaultDeviceNameHeader, "mac:112233445566")

	device, err := manager.Connect(response, request, nil)
	assert.Nil(device)
	assert.NotNil(err)
	assert.Equal(response.Code, http.StatusBadRequest)
}

func TestManagerPongCallbackFor(t *testing.T) {
	assert := assert.New(t)
	expectedDevice := newDevice(ID("ponged device"), Key("expected"), nil, 1)
	expectedData := "expected pong data"
	listenerCalled := false

	manager := &manager{
		logger: logging.TestLogger(t),
		pongListener: func(actualDevice Interface, actualData string) {
			listenerCalled = true
			assert.True(expectedDevice == actualDevice)
			assert.Equal(expectedData, actualData)
		},
	}

	pongCallback := manager.pongCallbackFor(expectedDevice)
	pongCallback(expectedData)
	assert.True(listenerCalled)
}

func TestManagerConnectListener(t *testing.T) {
	assert := assert.New(t)
	connectWait := new(sync.WaitGroup)
	connectWait.Add(testConnectionCount)
	connections := make(chan Interface, testConnectionCount)

	options := &Options{
		Logger: logging.TestLogger(t),
		ConnectListener: func(candidate Interface) {
			defer connectWait.Done()
			select {
			case connections <- candidate:
			default:
				assert.Fail("The connect listener should not block")
			}
		},
	}

	_, server, connectURL := startWebsocketServer(options)
	defer server.Close()

	dialer := NewDialer(options, nil)
	testDevices := connectTestDevices(t, assert, dialer, connectURL)
	defer closeTestDevices(assert, testDevices)

	connectWait.Wait()
	close(connections)
	assert.Equal(testConnectionCount, len(connections))

	visited := make(map[Interface]bool)
	for candidate := range connections {
		visited[candidate] = true
	}

	assert.Equal(testConnectionCount, len(visited))
}

func TestManagerDisconnect(t *testing.T) {
	assert := assert.New(t)
	disconnectWait := new(sync.WaitGroup)
	disconnectWait.Add(testConnectionCount)
	disconnections := make(chan Interface, testConnectionCount)

	options := &Options{
		Logger: logging.TestLogger(t),
		DisconnectListener: func(candidate Interface) {
			defer disconnectWait.Done()
			assert.True(candidate.Closed())
			disconnections <- candidate
		},
	}

	manager, server, connectURL := startWebsocketServer(options)
	defer server.Close()

	dialer := NewDialer(options, nil)
	testDevices := connectTestDevices(t, assert, dialer, connectURL)
	defer closeTestDevices(assert, testDevices)

	assert.Zero(manager.Disconnect(ID("nosuch")))
	for id, connectionCount := range testDeviceIDs {
		assert.Equal(connectionCount, manager.Disconnect(id))
	}

	disconnectWait.Wait()
	close(disconnections)
	assert.Equal(testConnectionCount, len(disconnections))

	visited := make(map[Interface]bool)
	for candidate := range disconnections {
		visited[candidate] = true
	}

	assert.Equal(testConnectionCount, len(visited))
}
<|endoftext|>


<commit_before>package margelet_test

import (
	. "github.com/smartystreets/goconvey/convey"
	"testing"
)

func TestChatRepository(t *testing.T) {
	Convey("Given chat repository", t, func() {
		m := getMargelet()

		Convey("When new chat added", func() {
			m.ChatRepository.Add(100500)
			Convey("New chat should be found in repo", func() {
				So(m.ChatRepository.All(), ShouldResemble, []int{100500})
			})
		})

		Convey("Given existing chats", func() {
			m.ChatRepository.Add(100500)
			m.ChatRepository.Add(100501)
			m.ChatRepository.Add(100502)
			m.ChatRepository.Add(100503)

			Convey("When removing chat", func() {
				m.ChatRepository.Remove(100500)

				Convey("Removed chat should not be found in repo", func() {
					So(m.ChatRepository.All(), ShouldResemble, []int{100501, 100502, 100503})
				})
			})

			Convey("Repo should return all chats", func() {
				So(m.ChatRepository.All(), ShouldResemble, []int{100500, 100501, 100502, 100503})
			})
		})
	})
}
<commit_msg>Fix tests for int64 Chat IDs.<commit_after>package margelet_test

import (
	. "github.com/smartystreets/goconvey/convey"
	"testing"
)

func TestChatRepository(t *testing.T) {
	Convey("Given chat repository", t, func() {
		m := getMargelet()

		Convey("When new chat added", func() {
			m.ChatRepository.Add(100500)
			Convey("New chat should be found in repo", func() {
				So(m.ChatRepository.All(), ShouldResemble, []int64{100500})
			})
		})

		Convey("Given existing chats", func() {
			m.ChatRepository.Add(100500)
			m.ChatRepository.Add(100501)
			m.ChatRepository.Add(100502)
			m.ChatRepository.Add(100503)

			Convey("When removing chat", func() {
				m.ChatRepository.Remove(100500)

				Convey("Removed chat should not be found in repo", func() {
					So(m.ChatRepository.All(), ShouldResemble, []int64{100501, 100502, 100503})
				})
			})

			Convey("Repo should return all chats", func() {
				So(m.ChatRepository.All(), ShouldResemble, []int64{100500, 100501, 100502, 100503})
			})
		})
	})
}
<|endoftext|>


<commit_before>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package walking

import (
	"context"
	"encoding/base64"
	"fmt"
	"io"
	"math/rand"
	"time"

	"github.com/containerd/containerd/archive"
	"github.com/containerd/containerd/archive/compression"
	"github.com/containerd/containerd/content"
	"github.com/containerd/containerd/diff"
	"github.com/containerd/containerd/errdefs"
	"github.com/containerd/containerd/log"
	"github.com/containerd/containerd/mount"
	digest "github.com/opencontainers/go-digest"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
)

type walkingDiff struct {
	store content.Store
}

var emptyDesc = ocispec.Descriptor{}
var uncompressed = "containerd.io/uncompressed"

// NewWalkingDiff is a generic implementation of diff.Comparer.  The diff is
// calculated by mounting both the upper and lower mount sets and walking the
// mounted directories concurrently. Changes are calculated by comparing files
// against each other or by comparing file existence between directories.
// NewWalkingDiff uses no special characteristics of the mount sets and is
// expected to work with any filesystem.
func NewWalkingDiff(store content.Store) diff.Comparer {
	return &walkingDiff{
		store: store,
	}
}

// Compare creates a diff between the given mounts and uploads the result
// to the content store.
func (s *walkingDiff) Compare(ctx context.Context, lower, upper []mount.Mount, opts ...diff.Opt) (d ocispec.Descriptor, err error) {
	var config diff.Config
	for _, opt := range opts {
		if err := opt(&config); err != nil {
			return emptyDesc, err
		}
	}

	if config.MediaType == "" {
		config.MediaType = ocispec.MediaTypeImageLayerGzip
	}

	var isCompressed bool
	switch config.MediaType {
	case ocispec.MediaTypeImageLayer:
	case ocispec.MediaTypeImageLayerGzip:
		isCompressed = true
	default:
		return emptyDesc, errors.Wrapf(errdefs.ErrNotImplemented, "unsupported diff media type: %v", config.MediaType)
	}

	var ocidesc ocispec.Descriptor
	if err := mount.WithTempMount(ctx, lower, func(lowerRoot string) error {
		return mount.WithTempMount(ctx, upper, func(upperRoot string) (retErr error) {
			var newReference bool
			if config.Reference == "" {
				newReference = true
				config.Reference = uniqueRef()
			}

			cw, err := s.store.Writer(ctx,
				content.WithRef(config.Reference),
				content.WithDescriptor(ocispec.Descriptor{
					MediaType: config.MediaType, // most contentstore implementations just ignore this
				}))
			if err != nil {
				return errors.Wrap(err, "failed to open writer")
			}
			defer func() {
				if retErr != nil {
					cw.Close()
					if newReference {
						if abortErr := s.store.Abort(ctx, config.Reference); abortErr != nil {
							log.G(ctx).WithError(abortErr).WithField("ref", config.Reference).Warnf("failed to delete diff upload")
						}
					}
				}
			}()
			if !newReference {
				if err = cw.Truncate(0); err != nil {
					return err
				}
			}

			if isCompressed {
				dgstr := digest.SHA256.Digester()
				var compressed io.WriteCloser
				compressed, err = compression.CompressStream(cw, compression.Gzip)
				if err != nil {
					return errors.Wrap(err, "failed to get compressed stream")
				}
				err = archive.WriteDiff(ctx, io.MultiWriter(compressed, dgstr.Hash()), lowerRoot, upperRoot)
				compressed.Close()
				if err != nil {
					return errors.Wrap(err, "failed to write compressed diff")
				}

				if config.Labels == nil {
					config.Labels = map[string]string{}
				}
				config.Labels[uncompressed] = dgstr.Digest().String()
			} else {
				if err = archive.WriteDiff(ctx, cw, lowerRoot, upperRoot); err != nil {
					return errors.Wrap(err, "failed to write diff")
				}
			}

			var commitopts []content.Opt
			if config.Labels != nil {
				commitopts = append(commitopts, content.WithLabels(config.Labels))
			}

			dgst := cw.Digest()
			if err := cw.Commit(ctx, 0, dgst, commitopts...); err != nil {
				if !errdefs.IsAlreadyExists(err) {
					return errors.Wrap(err, "failed to commit")
				}
			}

			info, err := s.store.Info(ctx, dgst)
			if err != nil {
				return errors.Wrap(err, "failed to get info from content store")
			}
			if info.Labels == nil {
				info.Labels = make(map[string]string)
			}
			// Set uncompressed label if digest already existed without label
			if _, ok := info.Labels[uncompressed]; !ok {
				info.Labels[uncompressed] = config.Labels[uncompressed]
				if _, err := s.store.Update(ctx, info, "labels."+uncompressed); err != nil {
					return errors.Wrap(err, "error setting uncompressed label")
				}
			}

			ocidesc = ocispec.Descriptor{
				MediaType: config.MediaType,
				Size:      info.Size,
				Digest:    info.Digest,
			}
			return nil
		})
	}); err != nil {
		return emptyDesc, err
	}

	return ocidesc, nil
}

func uniqueRef() string {
	t := time.Now()
	var b [3]byte
	// Ignore read failures, just decreases uniqueness
	rand.Read(b[:])
	return fmt.Sprintf("%d-%s", t.UnixNano(), base64.URLEncoding.EncodeToString(b[:]))
}
<commit_msg>Revert diff/walking error change<commit_after>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package walking

import (
	"context"
	"encoding/base64"
	"fmt"
	"io"
	"math/rand"
	"time"

	"github.com/containerd/containerd/archive"
	"github.com/containerd/containerd/archive/compression"
	"github.com/containerd/containerd/content"
	"github.com/containerd/containerd/diff"
	"github.com/containerd/containerd/errdefs"
	"github.com/containerd/containerd/log"
	"github.com/containerd/containerd/mount"
	digest "github.com/opencontainers/go-digest"
	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
)

type walkingDiff struct {
	store content.Store
}

var emptyDesc = ocispec.Descriptor{}
var uncompressed = "containerd.io/uncompressed"

// NewWalkingDiff is a generic implementation of diff.Comparer.  The diff is
// calculated by mounting both the upper and lower mount sets and walking the
// mounted directories concurrently. Changes are calculated by comparing files
// against each other or by comparing file existence between directories.
// NewWalkingDiff uses no special characteristics of the mount sets and is
// expected to work with any filesystem.
func NewWalkingDiff(store content.Store) diff.Comparer {
	return &walkingDiff{
		store: store,
	}
}

// Compare creates a diff between the given mounts and uploads the result
// to the content store.
func (s *walkingDiff) Compare(ctx context.Context, lower, upper []mount.Mount, opts ...diff.Opt) (d ocispec.Descriptor, err error) {
	var config diff.Config
	for _, opt := range opts {
		if err := opt(&config); err != nil {
			return emptyDesc, err
		}
	}

	if config.MediaType == "" {
		config.MediaType = ocispec.MediaTypeImageLayerGzip
	}

	var isCompressed bool
	switch config.MediaType {
	case ocispec.MediaTypeImageLayer:
	case ocispec.MediaTypeImageLayerGzip:
		isCompressed = true
	default:
		return emptyDesc, errors.Wrapf(errdefs.ErrNotImplemented, "unsupported diff media type: %v", config.MediaType)
	}

	var ocidesc ocispec.Descriptor
	if err := mount.WithTempMount(ctx, lower, func(lowerRoot string) error {
		return mount.WithTempMount(ctx, upper, func(upperRoot string) error {
			var newReference bool
			if config.Reference == "" {
				newReference = true
				config.Reference = uniqueRef()
			}

			cw, err := s.store.Writer(ctx,
				content.WithRef(config.Reference),
				content.WithDescriptor(ocispec.Descriptor{
					MediaType: config.MediaType, // most contentstore implementations just ignore this
				}))
			if err != nil {
				return errors.Wrap(err, "failed to open writer")
			}

			// errOpen is set when an error occurs while the content writer has not been
			// committed or closed yet to force a cleanup
			var errOpen error
			defer func() {
				if errOpen != nil {
					cw.Close()
					if newReference {
						if abortErr := s.store.Abort(ctx, config.Reference); abortErr != nil {
							log.G(ctx).WithError(abortErr).WithField("ref", config.Reference).Warnf("failed to delete diff upload")
						}
					}
				}
			}()
			if !newReference {
				if errOpen = cw.Truncate(0); errOpen != nil {
					return errOpen
				}
			}

			if isCompressed {
				dgstr := digest.SHA256.Digester()
				var compressed io.WriteCloser
				compressed, errOpen = compression.CompressStream(cw, compression.Gzip)
				if errOpen != nil {
					return errors.Wrap(errOpen, "failed to get compressed stream")
				}
				errOpen = archive.WriteDiff(ctx, io.MultiWriter(compressed, dgstr.Hash()), lowerRoot, upperRoot)
				compressed.Close()
				if errOpen != nil {
					return errors.Wrap(errOpen, "failed to write compressed diff")
				}

				if config.Labels == nil {
					config.Labels = map[string]string{}
				}
				config.Labels[uncompressed] = dgstr.Digest().String()
			} else {
				if errOpen = archive.WriteDiff(ctx, cw, lowerRoot, upperRoot); errOpen != nil {
					return errors.Wrap(errOpen, "failed to write diff")
				}
			}

			var commitopts []content.Opt
			if config.Labels != nil {
				commitopts = append(commitopts, content.WithLabels(config.Labels))
			}

			dgst := cw.Digest()
			if errOpen = cw.Commit(ctx, 0, dgst, commitopts...); errOpen != nil {
				if !errdefs.IsAlreadyExists(errOpen) {
					return errors.Wrap(errOpen, "failed to commit")
				}
				errOpen = nil
			}

			info, err := s.store.Info(ctx, dgst)
			if err != nil {
				return errors.Wrap(err, "failed to get info from content store")
			}
			if info.Labels == nil {
				info.Labels = make(map[string]string)
			}
			// Set uncompressed label if digest already existed without label
			if _, ok := info.Labels[uncompressed]; !ok {
				info.Labels[uncompressed] = config.Labels[uncompressed]
				if _, err := s.store.Update(ctx, info, "labels."+uncompressed); err != nil {
					return errors.Wrap(err, "error setting uncompressed label")
				}
			}

			ocidesc = ocispec.Descriptor{
				MediaType: config.MediaType,
				Size:      info.Size,
				Digest:    info.Digest,
			}
			return nil
		})
	}); err != nil {
		return emptyDesc, err
	}

	return ocidesc, nil
}

func uniqueRef() string {
	t := time.Now()
	var b [3]byte
	// Ignore read failures, just decreases uniqueness
	rand.Read(b[:])
	return fmt.Sprintf("%d-%s", t.UnixNano(), base64.URLEncoding.EncodeToString(b[:]))
}
<|endoftext|>


<commit_before>package models

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestQueryTagsList(t *testing.T) {
	if !testReady {
		return
	}

	cases := []struct {
		reponame     string
		proto        string
		protoVersion string
		output       []string
		expected     bool
	}{
		{"notexist", "oci", "v1", nil, true},
		{"second/second", "oci", "v1", []string{"v0.1", "v0.2"}, true},
		{"second/second", "docker", "v1", nil, true},
		{"second/second", "oci", "v2", nil, true},
	}

	for _, c := range cases {
		tags, err := QueryTagsList(c.reponame, c.proto, c.protoVersion)
		assert.Equal(t, c.output, tags)
		assert.Equal(t, c.expected, err == nil)
	}

	dbLock.Lock()
	defer dbLock.Unlock()
	AlterTable()
	_, err := QueryTagsList("second/second", "oci", "v1")
	RecoverTable()
	assert.NotNil(t, err)
}

func TestQueryReposList(t *testing.T) {
	if !testReady {
		return
	}
	// init the testdata
	FreeTestDBData()
	InitTestDBData()

	testdata := []string{"first", "second/second", "third/third", "fourth/fourth/fourth"}

	repos, err := QueryReposList()
	assert.Equal(t, testdata, repos)
	assert.Nil(t, err)

	dbLock.Lock()
	defer dbLock.Unlock()
	AlterTable()
	_, err = QueryReposList()
	RecoverTable()
	assert.NotNil(t, err)
}

func TestAddRepo(t *testing.T) {
	if !testReady {
		return
	}

	// init the testdata
	FreeTestDBData()
	InitTestDBData()

	cases := []struct {
		reponame string
		expected bool
	}{
		{"TestAddRepo-A", true},
		// exist
		{"TestAddRepo-A", true},
	}
	for _, c := range cases {
		_, err := AddRepo(c.reponame)
		assert.Equal(t, c.expected, err == nil)
	}

	dbLock.Lock()
	defer dbLock.Unlock()
	AlterTable()
	_, err := AddRepo("TestAddRepo-B")
	RecoverTable()
	assert.NotNil(t, err)
}

func TestQueryImage(t *testing.T) {
	if !testReady {
		return
	}

	// init the testdata
	FreeTestDBData()
	InitTestDBData()

	cases := []struct {
		repo_id      int
		tag          string
		proto        string
		protoVersion string
		expected     bool
	}{
		{2, "v0.1", "oci", "v1", true},
		{2, "v10000", "oci", "v1", false},
	}

	for _, c := range cases {
		img, err := QueryImage(c.repo_id, c.tag, c.proto, c.protoVersion)
		assert.Nil(t, err)
		assert.Equal(t, c.expected, img != nil)
	}

	dbLock.Lock()
	defer dbLock.Unlock()
	AlterTable()
	_, err := QueryImage(2, "v0.1", "oci", "v1")
	RecoverTable()
	assert.NotNil(t, err)
}

func TestAddImage(t *testing.T) {
	if !testReady {
		return
	}

	// init the testdata
	FreeTestDBData()
	InitTestDBData()

	cases := []struct {
		reponame     string
		tag          string
		proto        string
		protoVersion string
		expected     bool
	}{
		{"TestAddImage-A", "0.1", "test", "vtest", true},
		{"TestAddImage-A", "0.1", "test", "vtest", true},
	}
	for _, c := range cases {
		_, err := AddImage(c.reponame, c.tag, c.proto, c.protoVersion)
		assert.Equal(t, c.expected, err == nil)
	}

	dbLock.Lock()
	defer dbLock.Unlock()
	AlterTable()
	_, err := AddImage("TestAddImage-B", "0.1", "test", "vtest")
	RecoverTable()
	assert.NotNil(t, err)
}
<commit_msg>fix golint, ..<commit_after>package models

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestQueryTagsList(t *testing.T) {
	if !testReady {
		return
	}

	cases := []struct {
		reponame     string
		proto        string
		protoVersion string
		output       []string
		expected     bool
	}{
		{"notexist", "oci", "v1", nil, true},
		{"second/second", "oci", "v1", []string{"v0.1", "v0.2"}, true},
		{"second/second", "docker", "v1", nil, true},
		{"second/second", "oci", "v2", nil, true},
	}

	for _, c := range cases {
		tags, err := QueryTagsList(c.reponame, c.proto, c.protoVersion)
		assert.Equal(t, c.output, tags)
		assert.Equal(t, c.expected, err == nil)
	}

	dbLock.Lock()
	defer dbLock.Unlock()
	AlterTable()
	_, err := QueryTagsList("second/second", "oci", "v1")
	RecoverTable()
	assert.NotNil(t, err)
}

func TestQueryReposList(t *testing.T) {
	if !testReady {
		return
	}
	// init the testdata
	FreeTestDBData()
	InitTestDBData()

	testdata := []string{"first", "second/second", "third/third", "fourth/fourth/fourth"}

	repos, err := QueryReposList()
	assert.Equal(t, testdata, repos)
	assert.Nil(t, err)

	dbLock.Lock()
	defer dbLock.Unlock()
	AlterTable()
	_, err = QueryReposList()
	RecoverTable()
	assert.NotNil(t, err)
}

func TestAddRepo(t *testing.T) {
	if !testReady {
		return
	}

	// init the testdata
	FreeTestDBData()
	InitTestDBData()

	cases := []struct {
		reponame string
		expected bool
	}{
		{"TestAddRepo-A", true},
		// exist
		{"TestAddRepo-A", true},
	}
	for _, c := range cases {
		_, err := AddRepo(c.reponame)
		assert.Equal(t, c.expected, err == nil)
	}

	dbLock.Lock()
	defer dbLock.Unlock()
	AlterTable()
	_, err := AddRepo("TestAddRepo-B")
	RecoverTable()
	assert.NotNil(t, err)
}

func TestQueryImage(t *testing.T) {
	if !testReady {
		return
	}

	// init the testdata
	FreeTestDBData()
	InitTestDBData()

	cases := []struct {
		repoID       int
		tag          string
		proto        string
		protoVersion string
		expected     bool
	}{
		{2, "v0.1", "oci", "v1", true},
		{2, "v10000", "oci", "v1", false},
	}

	for _, c := range cases {
		img, err := QueryImage(c.repoID, c.tag, c.proto, c.protoVersion)
		assert.Nil(t, err)
		assert.Equal(t, c.expected, img != nil)
	}

	dbLock.Lock()
	defer dbLock.Unlock()
	AlterTable()
	_, err := QueryImage(2, "v0.1", "oci", "v1")
	RecoverTable()
	assert.NotNil(t, err)
}

func TestAddImage(t *testing.T) {
	if !testReady {
		return
	}

	// init the testdata
	FreeTestDBData()
	InitTestDBData()

	cases := []struct {
		reponame     string
		tag          string
		proto        string
		protoVersion string
		expected     bool
	}{
		{"TestAddImage-A", "0.1", "test", "vtest", true},
		{"TestAddImage-A", "0.1", "test", "vtest", true},
	}
	for _, c := range cases {
		_, err := AddImage(c.reponame, c.tag, c.proto, c.protoVersion)
		assert.Equal(t, c.expected, err == nil)
	}

	dbLock.Lock()
	defer dbLock.Unlock()
	AlterTable()
	_, err := AddImage("TestAddImage-B", "0.1", "test", "vtest")
	RecoverTable()
	assert.NotNil(t, err)
}
<|endoftext|>


<commit_before>package httpd

import (
	"bufio"
	"fmt"
	"github.com/Symantec/Dominator/lib/html"
	"net/http"
)

func (s *state) statusHandler(w http.ResponseWriter, req *http.Request) {
	writer := bufio.NewWriter(w)
	defer writer.Flush()
	fmt.Fprintln(writer, "<title>Basic filegen server status page</title>")
	fmt.Fprintln(writer, "")
	fmt.Fprintln(writer, "")
	fmt.Fprintln(writer, "
Basic filegen server status page
")
	fmt.Fprintln(writer, "")
	html.WriteHeaderWithRequest(writer, req)
	fmt.Fprintln(writer, "
")
	for _, htmlWriter := range htmlWriters {
		htmlWriter.WriteHtml(writer)
	}
	fmt.Fprintln(writer, "
")
	fmt.Fprintln(writer, "
")
	html.WriteFooter(writer)
	fmt.Fprintln(writer, "")
}
<commit_msg>Add warning banner to filegen HTTP server if in insecure mode.<commit_after>package httpd

import (
	"bufio"
	"fmt"
	"github.com/Symantec/Dominator/lib/html"
	"github.com/Symantec/Dominator/lib/srpc"
	"net/http"
)

func (s *state) statusHandler(w http.ResponseWriter, req *http.Request) {
	writer := bufio.NewWriter(w)
	defer writer.Flush()
	fmt.Fprintln(writer, "<title>Basic filegen server status page</title>")
	fmt.Fprintln(writer, "")
	fmt.Fprintln(writer, "")
	fmt.Fprintln(writer, "
Basic filegen server status page
")
	if !srpc.CheckTlsRequired() {
		fmt.Fprintln(writer,
			`<h1><font color="red">Running in insecure mode. Anyone can get your files!</font></h1>`)
	}
	fmt.Fprintln(writer, "")
	html.WriteHeaderWithRequest(writer, req)
	fmt.Fprintln(writer, "
")
	for _, htmlWriter := range htmlWriters {
		htmlWriter.WriteHtml(writer)
	}
	fmt.Fprintln(writer, "
")
	fmt.Fprintln(writer, "
")
	html.WriteFooter(writer)
	fmt.Fprintln(writer, "")
}
<|endoftext|>


<commit_before>package rocserv

import (
	"fmt"
	"net/http/httputil"
	"runtime"
	"strings"
	"time"

	xprom "gitlab.pri.ibanyu.com/middleware/seaweed/xstat/xmetric/xprometheus"
	"gitlab.pri.ibanyu.com/middleware/seaweed/xtrace"

	"github.com/gin-gonic/gin"
	"github.com/gin-gonic/gin/binding"
	"github.com/julienschmidt/httprouter"
	"github.com/uber/jaeger-client-go"
)

const (
	HttpHeaderKeyTraceID = "ipalfish-trace-id"

	WildCharacter = ":"
	RoutePath     = "req-simple-path"
)

// HttpServer is the http server, Create an instance of GinServer, by using NewGinServer()
type HttpServer struct {
	*gin.Engine
}

// Context warp gin Context
type Context struct {
	*gin.Context
}

// HandlerFunc ...
type HandlerFunc func(*Context)

// NewHttpServer create http server with gin
func NewHttpServer() *HttpServer {
	// 实例化gin Server
	router := gin.New()
	router.Use(Recovery(), Metric(), Trace())

	return &HttpServer{router}
}

// Use attachs a global middleware to the router
func (s *HttpServer) Use(middleware ...HandlerFunc) {
	s.Engine.Use(mutilWrap(middleware...)...)
}

// GET is a shortcut for router.Handle("GET", path, handle).
func (s *HttpServer) GET(relativePath string, handlers ...HandlerFunc) {
	ws := append([]gin.HandlerFunc{pathHook(relativePath)}, mutilWrap(handlers...)...)
	s.Engine.GET(relativePath, ws...)
}

// WGET is wrap httprouter handle to GET
func (s *HttpServer) WGET(relativePath string, handlers ...httprouter.Handle) {
	s.GET(relativePath, MutilWrapHttpRouter(handlers...)...)
}

// POST is a shortcut for router.Handle("POST", path, handle).
func (s *HttpServer) POST(relativePath string, handlers ...HandlerFunc) {
	ws := append([]gin.HandlerFunc{pathHook(relativePath)}, mutilWrap(handlers...)...)
	s.Engine.POST(relativePath, ws...)
}

// WPOST is wrap httprouter handle to POST
func (s *HttpServer) WPOST(relativePath string, handlers ...httprouter.Handle) {
	s.POST(relativePath, MutilWrapHttpRouter(handlers...)...)
}

// PUT is a shortcut for router.Handle("PUT", path, handle).
func (s *HttpServer) PUT(relativePath string, handlers ...HandlerFunc) {
	ws := append([]gin.HandlerFunc{pathHook(relativePath)}, mutilWrap(handlers...)...)
	s.Engine.PUT(relativePath, ws...)
}

// WPUT is wrap httprouter handle to PUT
func (s *HttpServer) WPUT(relativePath string, handlers ...httprouter.Handle) {
	s.PUT(relativePath, MutilWrapHttpRouter(handlers...)...)
}

// Any registers a route that matches all the HTTP methods.
func (s *HttpServer) Any(relativePath string, handlers ...HandlerFunc) {
	ws := append([]gin.HandlerFunc{pathHook(relativePath)}, mutilWrap(handlers...)...)
	s.Engine.Any(relativePath, ws...)
}

// WAny is wrap httprouter handle to ANY
func (s *HttpServer) WAny(relativePath string, handlers ...httprouter.Handle) {
	s.Any(relativePath, MutilWrapHttpRouter(handlers...)...)
}

// DELETE is a shortcut for router.Handle("DELETE", path, handle).
func (s *HttpServer) DELETE(relativePath string, handlers ...HandlerFunc) {
	ws := append([]gin.HandlerFunc{pathHook(relativePath)}, mutilWrap(handlers...)...)
	s.Engine.DELETE(relativePath, ws...)
}

// WDELETE is wrap httprouter handle to DELETE
func (s *HttpServer) WDELETE(relativePath string, handlers ...httprouter.Handle) {
	s.DELETE(relativePath, MutilWrapHttpRouter(handlers...)...)
}

// PATCH is a shortcut for router.Handle("PATCH", path, handle).
func (s *HttpServer) PATCH(relativePath string, handlers ...HandlerFunc) {
	ws := append([]gin.HandlerFunc{pathHook(relativePath)}, mutilWrap(handlers...)...)
	s.Engine.PATCH(relativePath, ws...)
}

// WPATCH is wrap httprouter handle to PATCH
func (s *HttpServer) WPATCH(relativePath string, handlers ...httprouter.Handle) {
	s.PATCH(relativePath, MutilWrapHttpRouter(handlers...)...)
}

// OPTIONS is a shortcut for router.Handle("OPTIONS", path, handle).
func (s *HttpServer) OPTIONS(relativePath string, handlers ...HandlerFunc) {
	ws := append([]gin.HandlerFunc{pathHook(relativePath)}, mutilWrap(handlers...)...)
	s.Engine.OPTIONS(relativePath, ws...)
}

// WOPTIONS is wrap httprouter handle to OPTIONS
func (s *HttpServer) WOPTIONS(relativePath string, handlers ...httprouter.Handle) {
	s.OPTIONS(relativePath, MutilWrapHttpRouter(handlers...)...)
}

// HEAD is a shortcut for router.Handle("HEAD", path, handle).
func (s *HttpServer) HEAD(relativePath string, handlers ...HandlerFunc) {
	ws := append([]gin.HandlerFunc{pathHook(relativePath)}, mutilWrap(handlers...)...)
	s.Engine.HEAD(relativePath, ws...)
}

// WHEAD is wrap httprouter handle to HEAD
func (s *HttpServer) WHEAD(relativePath string, handlers ...httprouter.Handle) {
	s.HEAD(relativePath, MutilWrapHttpRouter(handlers...)...)
}

// Bind checks the Content-Type to select a binding engine automatically
func (c *Context) Bind(obj interface{}) error {
	b := binding.Default(c.Request.Method, c.ContentType())
	return c.MustBindWith(obj, b)
}

// Metric returns a metric middleware
func Metric() gin.HandlerFunc {
	return func(c *gin.Context) {
		now := time.Now()
		c.Next()
		dt := time.Since(now)
		if path, exist := c.Get(RoutePath); exist {
			if fun, ok := path.(string); ok {
				group, serviceName := GetGroupAndService()
				_metricAPIRequestCount.With(xprom.LabelGroupName, group, xprom.LabelServiceName, serviceName, xprom.LabelAPI, fun).Inc()
				_metricAPIRequestTime.With(xprom.LabelGroupName, group, xprom.LabelServiceName, serviceName, xprom.LabelAPI, fun).Observe(float64(dt / time.Millisecond))
			}
		}
	}
}

// Trace returns a trace middleware
func Trace() gin.HandlerFunc {
	return func(c *gin.Context) {
		if span := xtrace.SpanFromContext(c.Request.Context()); span != nil {
			if sc, ok := span.Context().(jaeger.SpanContext); ok {
				c.Writer.Header()[HttpHeaderKeyTraceID] = []string{fmt.Sprint(sc.TraceID())}
			}
		}
		c.Next()
	}
}

// Recovery returns a middleware that recovers from any panics and writes a 500 if there was one.
func Recovery() gin.HandlerFunc {
	return func(c *gin.Context) {
		defer func() {
			var rawReq []byte
			if err := recover(); err != nil {
				const size = 64 << 10
				buf := make([]byte, size)
				buf = buf[:runtime.Stack(buf, false)]
				if c.Request != nil {
					rawReq, _ = httputil.DumpRequest(c.Request, false)
				}
				pl := fmt.Sprintf("http call panic: %s\n%v\n%s\n", string(rawReq), err, buf)
				fmt.Println(pl)
				c.AbortWithStatus(500)
			}
		}()
		c.Next()
	}
}

func pathHook(relativePath string) gin.HandlerFunc {
	return func(c *gin.Context) {
		values := strings.Split(relativePath, WildCharacter)
		c.Set(RoutePath, values[0])
	}
}

func mutilWrap(handlers ...HandlerFunc) []gin.HandlerFunc {
	var h = make([]gin.HandlerFunc, len(handlers))
	for k, v := range handlers {
		h[k] = wrap(v)
	}
	return h
}

func wrap(h HandlerFunc) gin.HandlerFunc {
	return func(c *gin.Context) {
		h(&Context{c})
	}
}

// MutilWrapHttpRouter wrap many httprouter handle to roc httpserver handle
func MutilWrapHttpRouter(handlers ...httprouter.Handle) []HandlerFunc {
	var h = make([]HandlerFunc, len(handlers))
	for k, v := range handlers {
		h[k] = WrapHttpRouter(v)
	}
	return h
}

// WrapHttpRouter wrap httprouter handle to roc httpserver handle
func WrapHttpRouter(handle httprouter.Handle) HandlerFunc {
	return func(c *Context) {
		params := make([]httprouter.Param, 0)
		for param := range c.Params {
			params = append(params, httprouter.Param{
				Key:   c.Params[param].Key,
				Value: c.Params[param].Value,
			})
		}
		handle(c.Writer, c.Request, params)
	}
}
<commit_msg>fix http trace middleware<commit_after>package rocserv

import (
	"fmt"
	"net/http/httputil"
	"runtime"
	"strings"
	"time"

	xprom "gitlab.pri.ibanyu.com/middleware/seaweed/xstat/xmetric/xprometheus"
	"gitlab.pri.ibanyu.com/middleware/seaweed/xtrace"

	"github.com/gin-gonic/gin"
	"github.com/gin-gonic/gin/binding"
	"github.com/julienschmidt/httprouter"
	"github.com/uber/jaeger-client-go"
)

const (
	HttpHeaderKeyTraceID = "ipalfish-trace-id"

	WildCharacter = ":"
	RoutePath     = "req-simple-path"
)

// HttpServer is the http server, Create an instance of GinServer, by using NewGinServer()
type HttpServer struct {
	*gin.Engine
}

// Context warp gin Context
type Context struct {
	*gin.Context
}

// HandlerFunc ...
type HandlerFunc func(*Context)

// NewHttpServer create http server with gin
func NewHttpServer() *HttpServer {
	// 实例化gin Server
	router := gin.New()
	router.Use(Recovery(), Metric(), Trace())

	return &HttpServer{router}
}

// Use attachs a global middleware to the router
func (s *HttpServer) Use(middleware ...HandlerFunc) {
	s.Engine.Use(mutilWrap(middleware...)...)
}

// GET is a shortcut for router.Handle("GET", path, handle).
func (s *HttpServer) GET(relativePath string, handlers ...HandlerFunc) {
	ws := append([]gin.HandlerFunc{pathHook(relativePath)}, mutilWrap(handlers...)...)
	s.Engine.GET(relativePath, ws...)
}

// WGET is wrap httprouter handle to GET
func (s *HttpServer) WGET(relativePath string, handlers ...httprouter.Handle) {
	s.GET(relativePath, MutilWrapHttpRouter(handlers...)...)
}

// POST is a shortcut for router.Handle("POST", path, handle).
func (s *HttpServer) POST(relativePath string, handlers ...HandlerFunc) {
	ws := append([]gin.HandlerFunc{pathHook(relativePath)}, mutilWrap(handlers...)...)
	s.Engine.POST(relativePath, ws...)
}

// WPOST is wrap httprouter handle to POST
func (s *HttpServer) WPOST(relativePath string, handlers ...httprouter.Handle) {
	s.POST(relativePath, MutilWrapHttpRouter(handlers...)...)
}

// PUT is a shortcut for router.Handle("PUT", path, handle).
func (s *HttpServer) PUT(relativePath string, handlers ...HandlerFunc) {
	ws := append([]gin.HandlerFunc{pathHook(relativePath)}, mutilWrap(handlers...)...)
	s.Engine.PUT(relativePath, ws...)
}

// WPUT is wrap httprouter handle to PUT
func (s *HttpServer) WPUT(relativePath string, handlers ...httprouter.Handle) {
	s.PUT(relativePath, MutilWrapHttpRouter(handlers...)...)
}

// Any registers a route that matches all the HTTP methods.
func (s *HttpServer) Any(relativePath string, handlers ...HandlerFunc) {
	ws := append([]gin.HandlerFunc{pathHook(relativePath)}, mutilWrap(handlers...)...)
	s.Engine.Any(relativePath, ws...)
}

// WAny is wrap httprouter handle to ANY
func (s *HttpServer) WAny(relativePath string, handlers ...httprouter.Handle) {
	s.Any(relativePath, MutilWrapHttpRouter(handlers...)...)
}

// DELETE is a shortcut for router.Handle("DELETE", path, handle).
func (s *HttpServer) DELETE(relativePath string, handlers ...HandlerFunc) {
	ws := append([]gin.HandlerFunc{pathHook(relativePath)}, mutilWrap(handlers...)...)
	s.Engine.DELETE(relativePath, ws...)
}

// WDELETE is wrap httprouter handle to DELETE
func (s *HttpServer) WDELETE(relativePath string, handlers ...httprouter.Handle) {
	s.DELETE(relativePath, MutilWrapHttpRouter(handlers...)...)
}

// PATCH is a shortcut for router.Handle("PATCH", path, handle).
func (s *HttpServer) PATCH(relativePath string, handlers ...HandlerFunc) {
	ws := append([]gin.HandlerFunc{pathHook(relativePath)}, mutilWrap(handlers...)...)
	s.Engine.PATCH(relativePath, ws...)
}

// WPATCH is wrap httprouter handle to PATCH
func (s *HttpServer) WPATCH(relativePath string, handlers ...httprouter.Handle) {
	s.PATCH(relativePath, MutilWrapHttpRouter(handlers...)...)
}

// OPTIONS is a shortcut for router.Handle("OPTIONS", path, handle).
func (s *HttpServer) OPTIONS(relativePath string, handlers ...HandlerFunc) {
	ws := append([]gin.HandlerFunc{pathHook(relativePath)}, mutilWrap(handlers...)...)
	s.Engine.OPTIONS(relativePath, ws...)
}

// WOPTIONS is wrap httprouter handle to OPTIONS
func (s *HttpServer) WOPTIONS(relativePath string, handlers ...httprouter.Handle) {
	s.OPTIONS(relativePath, MutilWrapHttpRouter(handlers...)...)
}

// HEAD is a shortcut for router.Handle("HEAD", path, handle).
func (s *HttpServer) HEAD(relativePath string, handlers ...HandlerFunc) {
	ws := append([]gin.HandlerFunc{pathHook(relativePath)}, mutilWrap(handlers...)...)
	s.Engine.HEAD(relativePath, ws...)
}

// WHEAD is wrap httprouter handle to HEAD
func (s *HttpServer) WHEAD(relativePath string, handlers ...httprouter.Handle) {
	s.HEAD(relativePath, MutilWrapHttpRouter(handlers...)...)
}

// Bind checks the Content-Type to select a binding engine automatically
func (c *Context) Bind(obj interface{}) error {
	b := binding.Default(c.Request.Method, c.ContentType())
	return c.MustBindWith(obj, b)
}

// Metric returns a metric middleware
func Metric() gin.HandlerFunc {
	return func(c *gin.Context) {
		now := time.Now()
		c.Next()
		dt := time.Since(now)
		if path, exist := c.Get(RoutePath); exist {
			if fun, ok := path.(string); ok {
				group, serviceName := GetGroupAndService()
				_metricAPIRequestCount.With(xprom.LabelGroupName, group, xprom.LabelServiceName, serviceName, xprom.LabelAPI, fun).Inc()
				_metricAPIRequestTime.With(xprom.LabelGroupName, group, xprom.LabelServiceName, serviceName, xprom.LabelAPI, fun).Observe(float64(dt / time.Millisecond))
			}
		}
	}
}

// Trace returns a trace middleware
func Trace() gin.HandlerFunc {
	return func(c *gin.Context) {
		span := xtrace.SpanFromContext(c.Request.Context())
		if span == nil {
			newSpan, ctx := xtrace.StartSpanFromContext(c.Request.Context(), c.Request.RequestURI)
			c.Request.WithContext(ctx)
			span = newSpan
		}
		defer span.Finish()

		if sc, ok := span.Context().(jaeger.SpanContext); ok {
			c.Writer.Header()[HttpHeaderKeyTraceID] = []string{fmt.Sprint(sc.TraceID())}
		}

		c.Next()
	}
}

// Recovery returns a middleware that recovers from any panics and writes a 500 if there was one.
func Recovery() gin.HandlerFunc {
	return func(c *gin.Context) {
		defer func() {
			var rawReq []byte
			if err := recover(); err != nil {
				const size = 64 << 10
				buf := make([]byte, size)
				buf = buf[:runtime.Stack(buf, false)]
				if c.Request != nil {
					rawReq, _ = httputil.DumpRequest(c.Request, false)
				}
				pl := fmt.Sprintf("http call panic: %s\n%v\n%s\n", string(rawReq), err, buf)
				fmt.Println(pl)
				c.AbortWithStatus(500)
			}
		}()
		c.Next()
	}
}

func pathHook(relativePath string) gin.HandlerFunc {
	return func(c *gin.Context) {
		values := strings.Split(relativePath, WildCharacter)
		c.Set(RoutePath, values[0])
	}
}

func mutilWrap(handlers ...HandlerFunc) []gin.HandlerFunc {
	var h = make([]gin.HandlerFunc, len(handlers))
	for k, v := range handlers {
		h[k] = wrap(v)
	}
	return h
}

func wrap(h HandlerFunc) gin.HandlerFunc {
	return func(c *gin.Context) {
		h(&Context{c})
	}
}

// MutilWrapHttpRouter wrap many httprouter handle to roc httpserver handle
func MutilWrapHttpRouter(handlers ...httprouter.Handle) []HandlerFunc {
	var h = make([]HandlerFunc, len(handlers))
	for k, v := range handlers {
		h[k] = WrapHttpRouter(v)
	}
	return h
}

// WrapHttpRouter wrap httprouter handle to roc httpserver handle
func WrapHttpRouter(handle httprouter.Handle) HandlerFunc {
	return func(c *Context) {
		params := make([]httprouter.Param, 0)
		for param := range c.Params {
			params = append(params, httprouter.Param{
				Key:   c.Params[param].Key,
				Value: c.Params[param].Value,
			})
		}
		handle(c.Writer, c.Request, params)
	}
}
<|endoftext|>


<commit_before>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package mqtt

import (
	"testing"

	. "github.com/TheThingsNetwork/ttn/utils/testing"
	//"github.com/TheThingsNetwork/ttn/utils/pointer"
)

func TestMQTTSend(t *testing.T) {
	tests := []struct {
		Desc      string          // Test Description
		Packet    []byte          // Handy representation of the packet to send
		Recipient []testRecipient // List of recipient to send

		WantData     []byte  // Expected Data on the recipient
		WantResponse []byte  // Expected Response from the Send method
		WantError    *string // Expected error nature returned by the Send method
	}{}

	for _, test := range tests {
		// Describe
		Desc(t, test.Desc)

		// Build
		// Generate new adapter
		// Generate reception servers

		// Operate
		// Send data to recipients
		// Retrieve data from servers

		// Check
		// Check if data has been received
		// Check if response is valid
		// Check if error is valid
	}
}

// ----- TYPE utilities
type testRecipient struct {
	Response  []byte
	TopicUp   string
	TopicDown string
}

// ----- BUILD utilities

// ----- OPERATE utilities

// ----- CHECK utilities
<commit_msg>[refactor] Write test checks methods<commit_after>// Copyright © 2015 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package mqtt

import (
	"reflect"
	"testing"

	"github.com/TheThingsNetwork/ttn/utils/errors"
	. "github.com/TheThingsNetwork/ttn/utils/testing"
	//"github.com/TheThingsNetwork/ttn/utils/pointer"
)

func TestMQTTSend(t *testing.T) {
	tests := []struct {
		Desc      string          // Test Description
		Packet    []byte          // Handy representation of the packet to send
		Recipient []testRecipient // List of recipient to send

		WantData     []byte  // Expected Data on the recipient
		WantResponse []byte  // Expected Response from the Send method
		WantError    *string // Expected error nature returned by the Send method
	}{}

	for _, test := range tests {
		// Describe
		Desc(t, test.Desc)

		// Build
		// Generate new adapter
		// Generate reception servers

		// Operate
		// Send data to recipients
		// Retrieve data from servers

		// Check
		// Check if data has been received
		// Check if response is valid
		// Check if error is valid
	}
}

// ----- TYPE utilities
type testRecipient struct {
	Response  []byte
	TopicUp   string
	TopicDown string
}

// ----- BUILD utilities

// ----- OPERATE utilities

// ----- CHECK utilities
func checkErrors(t *testing.T, want *string, got error) {
	if got == nil {
		if want == nil {
			Ok(t, "Check errors")
			return
		}
		Ko(t, "Expected error to be {%s} but got nothing", *want)
		return
	}

	if want == nil {
		Ko(t, "Expected no error but got {%v}", got)
		return
	}

	if got.(errors.Failure).Nature == *want {
		Ok(t, "Check errors")
		return
	}
	Ko(t, "Expected error to be {%s} but got {%v}", *want, got)
}

func checkResponses(t *testing.T, want []byte, got []byte) {
	if reflect.DeepEqual(want, got) {
		Ok(t, "Check responses")
		return
	}
	Ko(t, "Received response does not match expectations.\nWant: %s\nGot:  %s", string(want), string(got))
}

func checkData(t *testing.T, want []byte, got []byte) {
	if reflect.DeepEqual(want, got) {
		Ok(t, "Check data")
		return
	}
	Ko(t, "Received data does not match expectations.\nWant: %s\nGot:  %s", string(want), string(got))
}
<|endoftext|>


<commit_before># https://godoc.org/code.google.com/p/go.net/websocket
# https://gist.github.com/jweir/4528042
# https://github.com/golang-samples/websocket/blob/master/simple/main.go

package main

import (
	"log"
	"code.google.com/p/go.net/websocket"
	"io"
	"net/http"
)

func createSubscription(sh * subscriptionHandler, pubChan * string){
	log.Printf("creating channel %s",*pubChan)

	pubsub, err := redis.NewTCPClient(":6379","",-1).PubSubClient()
	haltOnErr(err)

	ch, err := pubsub.Subscribe(*pubChan)
	haltOnErr(err)

}

func echoHandler(ws *websocket.Conn) {

	# can I just plug createSubscription here or... ?
	# io.Copy(ws, ws)
}

func main() {

	http.Handle("/", websocket.Handler(echoHandler))
	err := http.ListenAndServe(":8080", nil)

	if err != nil {
		panic("ListenAndServe: " + err.Error())
	}
}
<commit_msg>notes and comments<commit_after>/*
https://godoc.org/code.google.com/p/go.net/websocket
http://blog.golang.org/spotlight-on-external-go-libraries
https://gist.github.com/jweir/4528042
https://github.com/golang-samples/websocket/blob/master/simple/main.go
*/

package main

import (
	"log"
	"code.google.com/p/go.net/websocket"
	"io"
	"net/http"
)

func createSubscription(sh * subscriptionHandler, pubChan * string){
	log.Printf("creating channel %s",*pubChan)

	pubsub, err := redis.NewTCPClient(":6379","",-1).PubSubClient()
	haltOnErr(err)

	ch, err := pubsub.Subscribe(*pubChan)
	haltOnErr(err)

}

func echoHandler(ws *websocket.Conn) {

     	/* can I just plug createSubscription here or... ?
	io.Copy(ws, ws)
	*/
}

func main() {

	http.Handle("/", websocket.Handler(echoHandler))
	err := http.ListenAndServe(":8080", nil)

	if err != nil {
		panic("ListenAndServe: " + err.Error())
	}
}
<|endoftext|>


<commit_before>package piece

import (
	"chessboard"
	"point"
)

type Piece struct {
	movable []point.Point
}

func NewPiece(movable []point.Point) *Piece {
	piece := new(Piece)
	piece.movable = movable
	return piece
}

func (piece Piece) CanMove(from, to point.Point) bool {
	if chessboard.InBoard(from) == false || chessboard.InBoard(to) == false {
		return false
	}
	diff := from.Diff(to)
	for i := 0; i < len(piece.movable); i++ {
		if diff.Y <= piece.movable[i].Y && diff.X <= piece.movable[i].X {
			return true
		}
	}
	return false
}
<commit_msg>Add black and white letters to piece<commit_after>package piece

import (
	"chessboard"
	"point"
)

type Piece struct {
	movable []point.Point
	black   byte
	white   byte
}

func NewPiece(movable []point.Point, black, white byte) *Piece {
	piece := new(Piece)
	piece.movable = movable
	piece.black = black
	piece.white = white
	return piece
}

func (piece Piece) CanMove(from, to point.Point) bool {
	if chessboard.InBoard(from) == false || chessboard.InBoard(to) == false {
		return false
	}
	diff := from.Diff(to)
	for i := 0; i < len(piece.movable); i++ {
		if diff.Y <= piece.movable[i].Y && diff.X <= piece.movable[i].X {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Pipe adapter to connect code expecting an io.Reader
// with code expecting an io.Writer.

package io

import (
	"os"
	"runtime"
	"sync"
)

type pipeResult struct {
	n   int
	err os.Error
}

// Shared pipe structure.
type pipe struct {
	// Reader sends on cr1, receives on cr2.
	// Writer does the same on cw1, cw2.
	r1, w1 chan []byte
	r2, w2 chan pipeResult

	rclose chan os.Error // read close; error to return to writers
	wclose chan os.Error // write close; error to return to readers

	done chan int // read or write half is done
}

func (p *pipe) run() {
	var (
		rb    []byte      // pending Read
		wb    []byte      // pending Write
		wn    int         // amount written so far from wb
		rerr  os.Error    // if read end is closed, error to send to writers
		werr  os.Error    // if write end is closed, error to send to readers
		r1    chan []byte // p.cr1 or nil depending on whether Read is ok
		w1    chan []byte // p.cw1 or nil depending on whether Write is ok
		ndone int
	)

	// Read and Write are enabled at the start.
	r1 = p.r1
	w1 = p.w1

	for {
		select {
		case <-p.done:
			if ndone++; ndone == 2 {
				// both reader and writer are gone
				return
			}
			continue
		case rerr = <-p.rclose:
			if w1 == nil {
				// finish pending Write
				p.w2 <- pipeResult{wn, rerr}
				wn = 0
				w1 = p.w1 // allow another Write
			}
			if r1 == nil {
				// Close of read side during Read.
				// finish pending Read with os.EINVAL.
				p.r2 <- pipeResult{0, os.EINVAL}
				r1 = p.r1 // allow another Read
			}
			continue
		case werr = <-p.wclose:
			if r1 == nil {
				// finish pending Read
				p.r2 <- pipeResult{0, werr}
				r1 = p.r1 // allow another Read
			}
			if w1 == nil {
				// Close of write side during Write.
				// finish pending Write with os.EINVAL.
				p.w2 <- pipeResult{wn, os.EINVAL}
				wn = 0
				w1 = p.w1 // allow another Write
			}
			continue
		case rb = <-r1:
			if werr != nil {
				// write end is closed
				p.r2 <- pipeResult{0, werr}
				continue
			}
			r1 = nil // disable Read until this one is done
		case wb = <-w1:
			if rerr != nil {
				// read end is closed
				p.w2 <- pipeResult{0, rerr}
				continue
			}
			w1 = nil // disable Write until this one is done
		}

		if r1 == nil && w1 == nil {
			// Have rb and wb.  Execute.
			n := copy(rb, wb)
			wn += n
			wb = wb[n:]

			// Finish Read.
			p.r2 <- pipeResult{n, nil}
			r1 = p.r1 // allow another Read

			// Maybe finish Write.
			if len(wb) == 0 {
				p.w2 <- pipeResult{wn, nil}
				wn = 0
				w1 = p.w1 // allow another Write
			}
		}
	}
}

// Read/write halves of the pipe.
// They are separate structures for two reasons:
//  1.  If one end becomes garbage without being Closed,
//      its finalizer can Close so that the other end
//      does not hang indefinitely.
//  2.  Clients cannot use interface conversions on the
//      read end to find the Write method, and vice versa.

type pipeHalf struct {
	c1     chan []byte
	c2     chan pipeResult
	cclose chan os.Error
	done   chan int

	lock   sync.Mutex
	closed bool

	io       sync.Mutex
	ioclosed bool
}

func (p *pipeHalf) rw(data []byte) (n int, err os.Error) {
	// Run i/o operation.
	// Check ioclosed flag under lock to make sure we're still allowed to do i/o.
	p.io.Lock()
	defer p.io.Unlock()
	if p.ioclosed {
		return 0, os.EINVAL
	}
	p.c1 <- data
	res := <-p.c2
	return res.n, res.err
}

func (p *pipeHalf) close(err os.Error) os.Error {
	// Close pipe half.
	// Only first call to close does anything.
	p.lock.Lock()
	if p.closed {
		p.lock.Unlock()
		return os.EINVAL
	}
	p.closed = true
	p.lock.Unlock()

	// First, send the close notification.
	p.cclose <- err

	// Runner is now responding to rw operations
	// with os.EINVAL.  Cut off future rw operations
	// by setting ioclosed flag.
	p.io.Lock()
	p.ioclosed = true
	p.io.Unlock()

	// With ioclosed set, there will be no more rw operations
	// working on the channels.
	// Tell the runner we won't be bothering it anymore.
	p.done <- 1

	// Successfully torn down; can disable finalizer.
	runtime.SetFinalizer(p, nil)

	return nil
}

func (p *pipeHalf) finalizer() {
	p.close(os.EINVAL)
}

// A PipeReader is the read half of a pipe.
type PipeReader struct {
	pipeHalf
}

// Read implements the standard Read interface:
// it reads data from the pipe, blocking until a writer
// arrives or the write end is closed.
// If the write end is closed with an error, that error is
// returned as err; otherwise err is nil.
func (r *PipeReader) Read(data []byte) (n int, err os.Error) {
	return r.rw(data)
}

// Close closes the reader; subsequent writes to the
// write half of the pipe will return the error os.EPIPE.
func (r *PipeReader) Close() os.Error {
	return r.CloseWithError(nil)
}

// CloseWithError closes the reader; subsequent writes
// to the write half of the pipe will return the error err.
func (r *PipeReader) CloseWithError(err os.Error) os.Error {
	if err == nil {
		err = os.EPIPE
	}
	return r.close(err)
}

// A PipeWriter is the write half of a pipe.
type PipeWriter struct {
	pipeHalf
}

// Write implements the standard Write interface:
// it writes data to the pipe, blocking until readers
// have consumed all the data or the read end is closed.
// If the read end is closed with an error, that err is
// returned as err; otherwise err is os.EPIPE.
func (w *PipeWriter) Write(data []byte) (n int, err os.Error) {
	return w.rw(data)
}

// Close closes the writer; subsequent reads from the
// read half of the pipe will return no bytes and os.EOF.
func (w *PipeWriter) Close() os.Error {
	return w.CloseWithError(nil)
}

// CloseWithError closes the writer; subsequent reads from the
// read half of the pipe will return no bytes and the error err.
func (w *PipeWriter) CloseWithError(err os.Error) os.Error {
	if err == nil {
		err = os.EOF
	}
	return w.close(err)
}

// Pipe creates a synchronous in-memory pipe.
// It can be used to connect code expecting an io.Reader
// with code expecting an io.Writer.
// Reads on one end are matched with writes on the other,
// copying data directly between the two; there is no internal buffering.
func Pipe() (*PipeReader, *PipeWriter) {
	p := &pipe{
		r1:     make(chan []byte),
		r2:     make(chan pipeResult),
		w1:     make(chan []byte),
		w2:     make(chan pipeResult),
		rclose: make(chan os.Error),
		wclose: make(chan os.Error),
		done:   make(chan int),
	}
	go p.run()

	// NOTE: Cannot use composite literal here:
	//	pipeHalf{c1: p.cr1, c2: p.cr2, cclose: p.crclose, cdone: p.cdone}
	// because this implicitly copies the pipeHalf, which copies the inner mutex.

	r := new(PipeReader)
	r.c1 = p.r1
	r.c2 = p.r2
	r.cclose = p.rclose
	r.done = p.done
	// TODO(rsc): Should be able to write
	//	runtime.SetFinalizer(r, (*PipeReader).finalizer)
	// but 6g doesn't see the finalizer method.
	runtime.SetFinalizer(&r.pipeHalf, (*pipeHalf).finalizer)

	w := new(PipeWriter)
	w.c1 = p.w1
	w.c2 = p.w2
	w.cclose = p.wclose
	w.done = p.done
	// TODO(rsc): Should be able to write
	//	runtime.SetFinalizer(w, (*PipeWriter).finalizer)
	// but 6g doesn't see the finalizer method.
	runtime.SetFinalizer(&w.pipeHalf, (*pipeHalf).finalizer)

	return r, w
}
<commit_msg>io: Avoid race condition in pipe.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Pipe adapter to connect code expecting an io.Reader
// with code expecting an io.Writer.

package io

import (
	"os"
	"runtime"
	"sync"
)

type pipeResult struct {
	n   int
	err os.Error
}

// Shared pipe structure.
type pipe struct {
	// Reader sends on cr1, receives on cr2.
	// Writer does the same on cw1, cw2.
	r1, w1 chan []byte
	r2, w2 chan pipeResult

	rclose chan os.Error // read close; error to return to writers
	wclose chan os.Error // write close; error to return to readers

	done chan int // read or write half is done
}

func (p *pipe) run() {
	var (
		rb    []byte      // pending Read
		wb    []byte      // pending Write
		wn    int         // amount written so far from wb
		rerr  os.Error    // if read end is closed, error to send to writers
		werr  os.Error    // if write end is closed, error to send to readers
		r1    chan []byte // p.cr1 or nil depending on whether Read is ok
		w1    chan []byte // p.cw1 or nil depending on whether Write is ok
		ndone int
	)

	// Read and Write are enabled at the start.
	r1 = p.r1
	w1 = p.w1

	for {
		select {
		case <-p.done:
			if ndone++; ndone == 2 {
				// both reader and writer are gone
				return
			}
			continue
		case rerr = <-p.rclose:
			if w1 == nil {
				// finish pending Write
				p.w2 <- pipeResult{wn, rerr}
				wn = 0
				w1 = p.w1 // allow another Write
			}
			if r1 == nil {
				// Close of read side during Read.
				// finish pending Read with os.EINVAL.
				p.r2 <- pipeResult{0, os.EINVAL}
				r1 = p.r1 // allow another Read
			}
			continue
		case werr = <-p.wclose:
			if r1 == nil {
				// finish pending Read
				p.r2 <- pipeResult{0, werr}
				r1 = p.r1 // allow another Read
			}
			if w1 == nil {
				// Close of write side during Write.
				// finish pending Write with os.EINVAL.
				p.w2 <- pipeResult{wn, os.EINVAL}
				wn = 0
				w1 = p.w1 // allow another Write
			}
			continue
		case rb = <-r1:
			if werr != nil {
				// write end is closed
				p.r2 <- pipeResult{0, werr}
				continue
			}
			r1 = nil // disable Read until this one is done
		case wb = <-w1:
			if rerr != nil {
				// read end is closed
				p.w2 <- pipeResult{0, rerr}
				continue
			}
			w1 = nil // disable Write until this one is done
		}

		if r1 == nil && w1 == nil {
			// Have rb and wb.  Execute.
			n := copy(rb, wb)
			wn += n
			wb = wb[n:]

			// Finish Read.
			p.r2 <- pipeResult{n, nil}
			r1 = p.r1 // allow another Read

			// Maybe finish Write.
			if len(wb) == 0 {
				p.w2 <- pipeResult{wn, nil}
				wn = 0
				w1 = p.w1 // allow another Write
			}
		}
	}
}

// Read/write halves of the pipe.
// They are separate structures for two reasons:
//  1.  If one end becomes garbage without being Closed,
//      its finalizer can Close so that the other end
//      does not hang indefinitely.
//  2.  Clients cannot use interface conversions on the
//      read end to find the Write method, and vice versa.

type pipeHalf struct {
	c1     chan []byte
	c2     chan pipeResult
	cclose chan os.Error
	done   chan int

	lock   sync.Mutex
	closed bool

	io       sync.Mutex
	ioclosed bool
}

func (p *pipeHalf) rw(data []byte) (n int, err os.Error) {
	// Run i/o operation.
	// Check ioclosed flag under lock to make sure we're still allowed to do i/o.
	p.io.Lock()
	if p.ioclosed {
		p.io.Unlock()
		return 0, os.EINVAL
	}
	p.io.Unlock()
	p.c1 <- data
	res := <-p.c2
	return res.n, res.err
}

func (p *pipeHalf) close(err os.Error) os.Error {
	// Close pipe half.
	// Only first call to close does anything.
	p.lock.Lock()
	if p.closed {
		p.lock.Unlock()
		return os.EINVAL
	}
	p.closed = true
	p.lock.Unlock()

	// First, send the close notification.
	p.cclose <- err

	// Runner is now responding to rw operations
	// with os.EINVAL.  Cut off future rw operations
	// by setting ioclosed flag.
	p.io.Lock()
	p.ioclosed = true
	p.io.Unlock()

	// With ioclosed set, there will be no more rw operations
	// working on the channels.
	// Tell the runner we won't be bothering it anymore.
	p.done <- 1

	// Successfully torn down; can disable finalizer.
	runtime.SetFinalizer(p, nil)

	return nil
}

func (p *pipeHalf) finalizer() {
	p.close(os.EINVAL)
}

// A PipeReader is the read half of a pipe.
type PipeReader struct {
	pipeHalf
}

// Read implements the standard Read interface:
// it reads data from the pipe, blocking until a writer
// arrives or the write end is closed.
// If the write end is closed with an error, that error is
// returned as err; otherwise err is nil.
func (r *PipeReader) Read(data []byte) (n int, err os.Error) {
	return r.rw(data)
}

// Close closes the reader; subsequent writes to the
// write half of the pipe will return the error os.EPIPE.
func (r *PipeReader) Close() os.Error {
	return r.CloseWithError(nil)
}

// CloseWithError closes the reader; subsequent writes
// to the write half of the pipe will return the error err.
func (r *PipeReader) CloseWithError(err os.Error) os.Error {
	if err == nil {
		err = os.EPIPE
	}
	return r.close(err)
}

// A PipeWriter is the write half of a pipe.
type PipeWriter struct {
	pipeHalf
}

// Write implements the standard Write interface:
// it writes data to the pipe, blocking until readers
// have consumed all the data or the read end is closed.
// If the read end is closed with an error, that err is
// returned as err; otherwise err is os.EPIPE.
func (w *PipeWriter) Write(data []byte) (n int, err os.Error) {
	return w.rw(data)
}

// Close closes the writer; subsequent reads from the
// read half of the pipe will return no bytes and os.EOF.
func (w *PipeWriter) Close() os.Error {
	return w.CloseWithError(nil)
}

// CloseWithError closes the writer; subsequent reads from the
// read half of the pipe will return no bytes and the error err.
func (w *PipeWriter) CloseWithError(err os.Error) os.Error {
	if err == nil {
		err = os.EOF
	}
	return w.close(err)
}

// Pipe creates a synchronous in-memory pipe.
// It can be used to connect code expecting an io.Reader
// with code expecting an io.Writer.
// Reads on one end are matched with writes on the other,
// copying data directly between the two; there is no internal buffering.
func Pipe() (*PipeReader, *PipeWriter) {
	p := &pipe{
		r1:     make(chan []byte),
		r2:     make(chan pipeResult),
		w1:     make(chan []byte),
		w2:     make(chan pipeResult),
		rclose: make(chan os.Error),
		wclose: make(chan os.Error),
		done:   make(chan int),
	}
	go p.run()

	// NOTE: Cannot use composite literal here:
	//	pipeHalf{c1: p.cr1, c2: p.cr2, cclose: p.crclose, cdone: p.cdone}
	// because this implicitly copies the pipeHalf, which copies the inner mutex.

	r := new(PipeReader)
	r.c1 = p.r1
	r.c2 = p.r2
	r.cclose = p.rclose
	r.done = p.done
	// TODO(rsc): Should be able to write
	//	runtime.SetFinalizer(r, (*PipeReader).finalizer)
	// but 6g doesn't see the finalizer method.
	runtime.SetFinalizer(&r.pipeHalf, (*pipeHalf).finalizer)

	w := new(PipeWriter)
	w.c1 = p.w1
	w.c2 = p.w2
	w.cclose = p.wclose
	w.done = p.done
	// TODO(rsc): Should be able to write
	//	runtime.SetFinalizer(w, (*PipeWriter).finalizer)
	// but 6g doesn't see the finalizer method.
	runtime.SetFinalizer(&w.pipeHalf, (*pipeHalf).finalizer)

	return r, w
}
<|endoftext|>


<commit_before>package sftp_test

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"testing"
	"time"

	"github.com/pkg/errors"

	"github.com/kopia/kopia/internal/blobtesting"
	"github.com/kopia/kopia/internal/clock"
	"github.com/kopia/kopia/internal/testlogging"
	"github.com/kopia/kopia/internal/testutil"
	"github.com/kopia/kopia/repo/blob"
	"github.com/kopia/kopia/repo/blob/sftp"
)

const (
	dockerImage  = "atmoz/sftp"
	dialTimeout  = 10 * time.Second
	sftpUsername = "foo"
)

func mustGetLocalTmpDir(t *testing.T) string {
	t.Helper()

	tmpDir, err := ioutil.TempDir(".", ".creds")
	if err != nil {
		t.Fatal(err)
	}

	tmpDir, err = filepath.Abs(tmpDir)
	if err != nil {
		t.Fatal(err)
	}

	t.Cleanup(func() {
		os.RemoveAll(tmpDir)
	})

	return tmpDir
}

// nolint:unparam
func runAndGetOutput(t *testing.T, cmd string, args ...string) ([]byte, error) {
	t.Helper()

	t.Logf("running %v %v", cmd, args)

	var stderr bytes.Buffer

	c := exec.Command(cmd, args...)
	c.Stderr = &stderr

	o, err := c.Output()
	if err != nil {
		return nil, errors.Wrapf(err, "error running %v %v (stdout %s stderr %s)", cmd, args, o, stderr.Bytes())
	}

	t.Logf("output: %s stderr: %s", o, stderr.Bytes())

	return o, nil
}

// nolint:unparam
func mustRunCommand(t *testing.T, cmd string, args ...string) []byte {
	t.Helper()

	v, err := runAndGetOutput(t, cmd, args...)
	if err != nil {
		t.Fatal(err)
	}

	return v
}

func startDockerSFTPServerOrSkip(t *testing.T, idRSA string) (host string, port int, knownHostsFile string) {
	t.Helper()

	tmpDir := mustGetLocalTmpDir(t)
	sshHostED25519Key := filepath.Join(tmpDir, "ssh_host_ed25519_key")
	sshHostRSAKey := filepath.Join(tmpDir, "ssh_host_rsa_key")

	mustRunCommand(t, "ssh-keygen", "-t", "ed25519", "-P", "", "-f", sshHostED25519Key)
	mustRunCommand(t, "ssh-keygen", "-t", "rsa", "-P", "", "-f", sshHostRSAKey)

	// see https://github.com/atmoz/sftp for instructions
	shortContainerID := testutil.RunContainerAndKillOnCloseOrSkip(t,
		"run", "--rm", "-p", "0:22",
		"-v", idRSA+".pub:/home/"+sftpUsername+"/.ssh/keys/id_rsa.pub:ro",
		"-v", sshHostED25519Key+":/etc/ssh/ssh_host_ed25519_key:ro",
		"-v", sshHostRSAKey+":/etc/ssh/ssh_host_rsa_key:ro",
		"-d", dockerImage,
		sftpUsername+"::::upload")
	sftpEndpoint := testutil.GetContainerMappedPortAddress(t, shortContainerID, "22")

	// wait for SFTP server to come up.
	deadline := clock.Now().Add(dialTimeout)
	for clock.Now().Before(deadline) {
		t.Logf("waiting for SFTP server to come up on '%v'...", sftpEndpoint)

		conn, err := net.DialTimeout("tcp", sftpEndpoint, time.Second)
		if err != nil {
			t.Logf("err: %v", err)
			time.Sleep(time.Second)

			continue
		}

		banner := make([]byte, 100)

		n, err := conn.Read(banner)
		if err != nil {
			t.Logf("error reading banner: %v", err)
			conn.Close()
			time.Sleep(time.Second)

			continue
		}

		conn.Close()

		t.Logf("got banner: %s", banner[0:n])

		parts := strings.Split(sftpEndpoint, ":")
		host = parts[0]
		port, _ = strconv.Atoi(parts[1])
		knownHostsFile = filepath.Join(testutil.TempDirectory(t), "known_hosts")

		time.Sleep(3 * time.Second)

		knownHostsData, err := runAndGetOutput(t, "ssh-keyscan", "-t", "rsa", "-p", strconv.Itoa(port), host)
		if err != nil || len(knownHostsData) == 0 {
			t.Logf("error scanning keys: %v", err)
			time.Sleep(time.Second)

			continue
		}

		t.Logf("knownHostsData: %s", knownHostsData)

		ioutil.WriteFile(knownHostsFile, knownHostsData, 0600)

		t.Logf("SFTP server OK on host:%q port:%v. Known hosts file: %v", host, port, knownHostsFile)

		return
	}

	t.Skipf("SFTP server did not start!")

	return //nolint:nakedret
}

func TestSFTPStorageValid(t *testing.T) {
	t.Parallel()
	testutil.ProviderTest(t)

	testutil.TestSkipOnCIUnlessLinuxAMD64(t)

	tmpDir := mustGetLocalTmpDir(t)
	idRSA := filepath.Join(tmpDir, "id_rsa")

	mustRunCommand(t, "ssh-keygen", "-t", "rsa", "-P", "", "-f", idRSA)

	host, port, knownHostsFile := startDockerSFTPServerOrSkip(t, idRSA)

	for _, embedCreds := range []bool{false, true} {
		embedCreds := embedCreds
		t.Run(fmt.Sprintf("Embed=%v", embedCreds), func(t *testing.T) {
			ctx := testlogging.Context(t)

			st, err := createSFTPStorage(ctx, t, host, port, idRSA, knownHostsFile, embedCreds)
			if err != nil {
				t.Fatalf("unable to connect to SSH: %v", err)
			}

			deleteBlobs(ctx, t, st)

			blobtesting.VerifyStorage(ctx, t, st)
			blobtesting.AssertConnectionInfoRoundTrips(ctx, t, st)

			// delete everything again
			deleteBlobs(ctx, t, st)

			if err := st.Close(ctx); err != nil {
				t.Fatalf("err: %v", err)
			}
		})
	}
}

func TestInvalidServerFailsFast(t *testing.T) {
	t.Parallel()
	testutil.ProviderTest(t)

	ctx := testlogging.Context(t)

	tmpDir := mustGetLocalTmpDir(t)
	idRSA := filepath.Join(tmpDir, "id_rsa")
	knownHostsFile := filepath.Join(tmpDir, "known_hosts")

	mustRunCommand(t, "ssh-keygen", "-t", "rsa", "-P", "", "-f", idRSA)
	ioutil.WriteFile(knownHostsFile, nil, 0600)

	t0 := clock.Now()

	if _, err := createSFTPStorage(ctx, t, "no-such-host", 22, idRSA, knownHostsFile, false); err == nil {
		t.Fatalf("unexpected success with bad credentials")
	}

	if dt := clock.Since(t0); dt > 10*time.Second {
		t.Fatalf("opening storage took too long, probably due to retries")
	}
}

func deleteBlobs(ctx context.Context, t *testing.T, st blob.Storage) {
	t.Helper()

	if err := st.ListBlobs(ctx, "", func(bm blob.Metadata) error {
		return st.DeleteBlob(ctx, bm.BlobID)
	}); err != nil {
		t.Fatalf("unable to clear sftp storage: %v", err)
	}
}

func createSFTPStorage(ctx context.Context, t *testing.T, host string, port int, idRSA, knownHostsFile string, embed bool) (blob.Storage, error) {
	t.Helper()

	if _, err := os.Stat(knownHostsFile); err != nil {
		t.Fatalf("skipping test because SFTP known hosts file can't be opened: %v", knownHostsFile)
	}

	opt := &sftp.Options{
		Path:           "/upload",
		Host:           host,
		Username:       sftpUsername,
		Port:           port,
		Keyfile:        idRSA,
		KnownHostsFile: knownHostsFile,
	}

	if embed {
		opt.KeyData = mustReadFileToString(t, opt.Keyfile)
		opt.Keyfile = ""

		opt.KnownHostsData = mustReadFileToString(t, opt.KnownHostsFile)
		opt.KnownHostsFile = ""
	}

	return sftp.New(ctx, opt)
}

func mustReadFileToString(t *testing.T, fname string) string {
	t.Helper()

	data, err := ioutil.ReadFile(fname)
	if err != nil {
		t.Fatal(err)
	}

	return string(data)
}
<commit_msg>sftp: run tests as part of unit tests (#1086)<commit_after>package sftp_test

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"testing"
	"time"

	"github.com/pkg/errors"

	"github.com/kopia/kopia/internal/blobtesting"
	"github.com/kopia/kopia/internal/clock"
	"github.com/kopia/kopia/internal/testlogging"
	"github.com/kopia/kopia/internal/testutil"
	"github.com/kopia/kopia/repo/blob"
	"github.com/kopia/kopia/repo/blob/sftp"
)

const (
	dockerImage  = "atmoz/sftp"
	dialTimeout  = 10 * time.Second
	sftpUsername = "foo"
)

func mustGetLocalTmpDir(t *testing.T) string {
	t.Helper()

	tmpDir, err := ioutil.TempDir(".", ".creds")
	if err != nil {
		t.Fatal(err)
	}

	tmpDir, err = filepath.Abs(tmpDir)
	if err != nil {
		t.Fatal(err)
	}

	t.Cleanup(func() {
		os.RemoveAll(tmpDir)
	})

	return tmpDir
}

// nolint:unparam
func runAndGetOutput(t *testing.T, cmd string, args ...string) ([]byte, error) {
	t.Helper()

	t.Logf("running %v %v", cmd, args)

	var stderr bytes.Buffer

	c := exec.Command(cmd, args...)
	c.Stderr = &stderr

	o, err := c.Output()
	if err != nil {
		return nil, errors.Wrapf(err, "error running %v %v (stdout %s stderr %s)", cmd, args, o, stderr.Bytes())
	}

	t.Logf("output: %s stderr: %s", o, stderr.Bytes())

	return o, nil
}

// nolint:unparam
func mustRunCommand(t *testing.T, cmd string, args ...string) []byte {
	t.Helper()

	v, err := runAndGetOutput(t, cmd, args...)
	if err != nil {
		t.Fatal(err)
	}

	return v
}

func startDockerSFTPServerOrSkip(t *testing.T, idRSA string) (host string, port int, knownHostsFile string) {
	t.Helper()

	tmpDir := mustGetLocalTmpDir(t)
	sshHostED25519Key := filepath.Join(tmpDir, "ssh_host_ed25519_key")
	sshHostRSAKey := filepath.Join(tmpDir, "ssh_host_rsa_key")

	mustRunCommand(t, "ssh-keygen", "-t", "ed25519", "-P", "", "-f", sshHostED25519Key)
	mustRunCommand(t, "ssh-keygen", "-t", "rsa", "-P", "", "-f", sshHostRSAKey)

	// see https://github.com/atmoz/sftp for instructions
	shortContainerID := testutil.RunContainerAndKillOnCloseOrSkip(t,
		"run", "--rm", "-p", "0:22",
		"-v", idRSA+".pub:/home/"+sftpUsername+"/.ssh/keys/id_rsa.pub:ro",
		"-v", sshHostED25519Key+":/etc/ssh/ssh_host_ed25519_key:ro",
		"-v", sshHostRSAKey+":/etc/ssh/ssh_host_rsa_key:ro",
		"-d", dockerImage,
		sftpUsername+"::::upload")
	sftpEndpoint := testutil.GetContainerMappedPortAddress(t, shortContainerID, "22")

	// wait for SFTP server to come up.
	deadline := clock.Now().Add(dialTimeout)
	for clock.Now().Before(deadline) {
		t.Logf("waiting for SFTP server to come up on '%v'...", sftpEndpoint)

		conn, err := net.DialTimeout("tcp", sftpEndpoint, time.Second)
		if err != nil {
			t.Logf("err: %v", err)
			time.Sleep(time.Second)

			continue
		}

		banner := make([]byte, 100)

		n, err := conn.Read(banner)
		if err != nil {
			t.Logf("error reading banner: %v", err)
			conn.Close()
			time.Sleep(time.Second)

			continue
		}

		conn.Close()

		t.Logf("got banner: %s", banner[0:n])

		parts := strings.Split(sftpEndpoint, ":")
		host = parts[0]
		port, _ = strconv.Atoi(parts[1])
		knownHostsFile = filepath.Join(testutil.TempDirectory(t), "known_hosts")

		time.Sleep(3 * time.Second)

		knownHostsData, err := runAndGetOutput(t, "ssh-keyscan", "-t", "rsa", "-p", strconv.Itoa(port), host)
		if err != nil || len(knownHostsData) == 0 {
			t.Logf("error scanning keys: %v", err)
			time.Sleep(time.Second)

			continue
		}

		t.Logf("knownHostsData: %s", knownHostsData)

		ioutil.WriteFile(knownHostsFile, knownHostsData, 0600)

		t.Logf("SFTP server OK on host:%q port:%v. Known hosts file: %v", host, port, knownHostsFile)

		return
	}

	t.Skipf("SFTP server did not start!")

	return //nolint:nakedret
}

func TestSFTPStorageValid(t *testing.T) {
	t.Parallel()

	testutil.TestSkipOnCIUnlessLinuxAMD64(t)

	tmpDir := mustGetLocalTmpDir(t)
	idRSA := filepath.Join(tmpDir, "id_rsa")

	mustRunCommand(t, "ssh-keygen", "-t", "rsa", "-P", "", "-f", idRSA)

	host, port, knownHostsFile := startDockerSFTPServerOrSkip(t, idRSA)

	for _, embedCreds := range []bool{false, true} {
		embedCreds := embedCreds
		t.Run(fmt.Sprintf("Embed=%v", embedCreds), func(t *testing.T) {
			ctx := testlogging.Context(t)

			st, err := createSFTPStorage(ctx, t, host, port, idRSA, knownHostsFile, embedCreds)
			if err != nil {
				t.Fatalf("unable to connect to SSH: %v", err)
			}

			deleteBlobs(ctx, t, st)

			blobtesting.VerifyStorage(ctx, t, st)
			blobtesting.AssertConnectionInfoRoundTrips(ctx, t, st)

			// delete everything again
			deleteBlobs(ctx, t, st)

			if err := st.Close(ctx); err != nil {
				t.Fatalf("err: %v", err)
			}
		})
	}
}

func TestInvalidServerFailsFast(t *testing.T) {
	t.Parallel()

	ctx := testlogging.Context(t)

	tmpDir := mustGetLocalTmpDir(t)
	idRSA := filepath.Join(tmpDir, "id_rsa")
	knownHostsFile := filepath.Join(tmpDir, "known_hosts")

	mustRunCommand(t, "ssh-keygen", "-t", "rsa", "-P", "", "-f", idRSA)
	ioutil.WriteFile(knownHostsFile, nil, 0600)

	t0 := clock.Now()

	if _, err := createSFTPStorage(ctx, t, "no-such-host", 22, idRSA, knownHostsFile, false); err == nil {
		t.Fatalf("unexpected success with bad credentials")
	}

	if dt := clock.Since(t0); dt > 10*time.Second {
		t.Fatalf("opening storage took too long, probably due to retries")
	}
}

func deleteBlobs(ctx context.Context, t *testing.T, st blob.Storage) {
	t.Helper()

	if err := st.ListBlobs(ctx, "", func(bm blob.Metadata) error {
		return st.DeleteBlob(ctx, bm.BlobID)
	}); err != nil {
		t.Fatalf("unable to clear sftp storage: %v", err)
	}
}

func createSFTPStorage(ctx context.Context, t *testing.T, host string, port int, idRSA, knownHostsFile string, embed bool) (blob.Storage, error) {
	t.Helper()

	if _, err := os.Stat(knownHostsFile); err != nil {
		t.Fatalf("skipping test because SFTP known hosts file can't be opened: %v", knownHostsFile)
	}

	opt := &sftp.Options{
		Path:           "/upload",
		Host:           host,
		Username:       sftpUsername,
		Port:           port,
		Keyfile:        idRSA,
		KnownHostsFile: knownHostsFile,
	}

	if embed {
		opt.KeyData = mustReadFileToString(t, opt.Keyfile)
		opt.Keyfile = ""

		opt.KnownHostsData = mustReadFileToString(t, opt.KnownHostsFile)
		opt.KnownHostsFile = ""
	}

	return sftp.New(ctx, opt)
}

func mustReadFileToString(t *testing.T, fname string) string {
	t.Helper()

	data, err := ioutil.ReadFile(fname)
	if err != nil {
		t.Fatal(err)
	}

	return string(data)
}
<|endoftext|>


<commit_before>//----------------------------------------
//
// Copyright © ying32. All Rights Reserved.
//
// Licensed under Apache License 2.0
//
//----------------------------------------

// +build windows linux
// +build !tempdll
// +build !memorydll

// 指令为：target == windows || target == linux && !tempdll && !memorydll

package api

func checkAndReleaseDLL() (bool, string) {
	return false, ""
}
<commit_msg>Fix the error of `tempdll` constraint under macOS<commit_after>//----------------------------------------
//
// Copyright © ying32. All Rights Reserved.
//
// Licensed under Apache License 2.0
//
//----------------------------------------

// +build !tempdll
// +build !memorydll

// 指令为：!tempdll && !memorydll

package api

func checkAndReleaseDLL() (bool, string) {
	return false, ""
}
<|endoftext|>