<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fs_test

import (
	"github.com/jacobsa/comeback/fs"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"io/ioutil"
	"os"
	"path"
	"testing"
)

func TestPerms(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// SetPermissions
////////////////////////////////////////////////////////////////////////

type SetPermissionsTest struct {
	fileSystemTest

	path string
	perms os.FileMode

	err error
}

func init() { RegisterTestSuite(&SetPermissionsTest{}) }

func (t *SetPermissionsTest) call() {
	t.err = t.fileSystem.SetPermissions(t.path, t.perms)
}

func (t *SetPermissionsTest) list() []*fs.DirectoryEntry {
	entries, err := t.fileSystem.ReadDir(t.baseDir)
	AssertEq(nil, err)
	return entries
}

func (t *SetPermissionsTest) NonExistentPath() {
	t.path = path.Join(t.baseDir, "foobar")

	// Call
	t.call()

	ExpectThat(t.err, Error(HasSubstr("foobar")))
	ExpectThat(t.err, Error(HasSubstr("no such")))
}

func (t *SetPermissionsTest) File() {
	t.path = path.Join(t.baseDir, "taco.txt")
	t.perms = 0754

	// Create
	err := ioutil.WriteFile(t.path, []byte(""), 0600)
	AssertEq(nil, err)

	// Call
	t.call()
	AssertEq(nil, t.err)

	// List
	entries := t.list()

	AssertThat(entries, ElementsAre(Any()))
	entry := entries[0]

	AssertEq(fs.TypeFile, entry.Type)
	ExpectEq(0754, entry.Permissions)
}

func (t *SetPermissionsTest) Directory() {
	t.path = path.Join(t.baseDir, "taco")
	t.perms = 0754

	// Create
	err := os.Mkdir(t.path, 0300)
	AssertEq(nil, err)

	// Call
	t.call()
	AssertEq(nil, t.err)

	// List
	entries := t.list()

	AssertThat(entries, ElementsAre(Any()))
	entry := entries[0]

	AssertEq(fs.TypeDirectory, entry.Type)
	ExpectEq(0754, entry.Permissions)
}

func (t *SetPermissionsTest) Symlink() {
	t.path = path.Join(t.baseDir, "taco")
	t.perms = 0754

	// Create
	err := os.Symlink("/foo/burrito", t.path)
	AssertEq(nil, err)

	// Call
	t.call()
	AssertEq(nil, t.err)

	// List
	entries := t.list()

	AssertThat(entries, ElementsAre(Any()))
	entry := entries[0]

	AssertEq(fs.TypeDirectory, entry.Type)
	ExpectEq(0754, entry.Permissions)
}

func (t *SetPermissionsTest) Device() {
	ExpectEq("TODO", "")
}

func (t *SetPermissionsTest) IgnoresOtherBits() {
	ExpectEq("TODO", "")
}
<commit_msg>SetPermissionsTest.IgnoresOtherBits<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fs_test

import (
	"github.com/jacobsa/comeback/fs"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"io/ioutil"
	"os"
	"path"
	"testing"
)

func TestPerms(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// SetPermissions
////////////////////////////////////////////////////////////////////////

type SetPermissionsTest struct {
	fileSystemTest

	path string
	perms os.FileMode

	err error
}

func init() { RegisterTestSuite(&SetPermissionsTest{}) }

func (t *SetPermissionsTest) call() {
	t.err = t.fileSystem.SetPermissions(t.path, t.perms)
}

func (t *SetPermissionsTest) list() []*fs.DirectoryEntry {
	entries, err := t.fileSystem.ReadDir(t.baseDir)
	AssertEq(nil, err)
	return entries
}

func (t *SetPermissionsTest) NonExistentPath() {
	t.path = path.Join(t.baseDir, "foobar")

	// Call
	t.call()

	ExpectThat(t.err, Error(HasSubstr("foobar")))
	ExpectThat(t.err, Error(HasSubstr("no such")))
}

func (t *SetPermissionsTest) File() {
	t.path = path.Join(t.baseDir, "taco.txt")
	t.perms = 0754

	// Create
	err := ioutil.WriteFile(t.path, []byte(""), 0600)
	AssertEq(nil, err)

	// Call
	t.call()
	AssertEq(nil, t.err)

	// List
	entries := t.list()

	AssertThat(entries, ElementsAre(Any()))
	entry := entries[0]

	AssertEq(fs.TypeFile, entry.Type)
	ExpectEq(0754, entry.Permissions)
}

func (t *SetPermissionsTest) Directory() {
	t.path = path.Join(t.baseDir, "taco")
	t.perms = 0754

	// Create
	err := os.Mkdir(t.path, 0300)
	AssertEq(nil, err)

	// Call
	t.call()
	AssertEq(nil, t.err)

	// List
	entries := t.list()

	AssertThat(entries, ElementsAre(Any()))
	entry := entries[0]

	AssertEq(fs.TypeDirectory, entry.Type)
	ExpectEq(0754, entry.Permissions)
}

func (t *SetPermissionsTest) Symlink() {
	t.path = path.Join(t.baseDir, "taco")
	t.perms = 0754

	// Create
	err := os.Symlink("/foo/burrito", t.path)
	AssertEq(nil, err)

	// Call
	t.call()
	AssertEq(nil, t.err)

	// List
	entries := t.list()

	AssertThat(entries, ElementsAre(Any()))
	entry := entries[0]

	AssertEq(fs.TypeDirectory, entry.Type)
	ExpectEq(0754, entry.Permissions)
}

func (t *SetPermissionsTest) IgnoresOtherBits() {
	t.path = path.Join(t.baseDir, "taco.txt")
	t.perms = 0754 | os.ModeNamedPipe | os.ModeTemporary

	// Create
	err := ioutil.WriteFile(t.path, []byte(""), 0600)
	AssertEq(nil, err)

	// Call
	t.call()
	AssertEq(nil, t.err)

	// List
	entries := t.list()

	AssertThat(entries, ElementsAre(Any()))
	entry := entries[0]

	AssertEq(fs.TypeFile, entry.Type)
	ExpectEq(0754, entry.Permissions)
}
<|endoftext|>


<commit_before>package client

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"sync"

	"github.com/pachyderm/pachyderm/src/client/pkg/config"

	log "github.com/sirupsen/logrus"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
	_ "k8s.io/client-go/plugin/pkg/client/auth" // enables support for configs with auth
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/portforward"
	"k8s.io/client-go/transport/spdy"
)

// PortForwarder handles proxying local traffic to a kubernetes pod
type PortForwarder struct {
	core          corev1.CoreV1Interface
	client        rest.Interface
	config        *rest.Config
	namespace     string
	logger        *io.PipeWriter
	stopChansLock *sync.Mutex
	stopChans     []chan struct{}
	shutdown      bool
}

// NewPortForwarder creates a new port forwarder
func NewPortForwarder(namespace string) (*PortForwarder, error) {
	cfg, err := config.Read()
	if err != nil {
		return nil, fmt.Errorf("could not read config: %v", err)
	}
	_, context, err := cfg.ActiveContext()
	if err != nil {
		return nil, fmt.Errorf("could not get active context: %v", err)
	}

	if namespace == "" {
		namespace = context.Namespace
	}
	if namespace == "" {
		namespace = "default"
	}

	kubeConfig := config.KubeConfig(context)

	kubeClientConfig, err := kubeConfig.ClientConfig()
	if err != nil {
		return nil, err
	}

	client, err := kubernetes.NewForConfig(kubeClientConfig)
	if err != nil {
		return nil, err
	}

	core := client.CoreV1()

	return &PortForwarder{
		core:          core,
		client:        core.RESTClient(),
		config:        kubeClientConfig,
		namespace:     namespace,
		logger:        log.StandardLogger().Writer(),
		stopChansLock: &sync.Mutex{},
		stopChans:     []chan struct{}{},
		shutdown:      false,
	}, nil
}

// Run starts the port forwarder. Returns after initialization is begun with
// the locally bound port and any initialization errors.
func (f *PortForwarder) Run(appName string, localPort, remotePort uint16) (uint16, error) {
	podNameSelector := map[string]string{
		"suite": "pachyderm",
		"app":   appName,
	}

	podList, err := f.core.Pods(f.namespace).List(metav1.ListOptions{
		LabelSelector: metav1.FormatLabelSelector(metav1.SetAsLabelSelector(podNameSelector)),
		TypeMeta: metav1.TypeMeta{
			Kind:       "ListOptions",
			APIVersion: "v1",
		},
	})
	if err != nil {
		return 0, err
	}
	if len(podList.Items) == 0 {
		return 0, return fmt.Errorf("no pods found for app %s", appName)
	}

	// Choose a random pod
	podName := podList.Items[rand.Intn(len(podList.Items))].Name

	url := f.client.Post().
		Resource("pods").
		Namespace(f.namespace).
		Name(podName).
		SubResource("portforward").
		URL()

	transport, upgrader, err := spdy.RoundTripperFor(f.config)
	if err != nil {
		return 0, err
	}

	dialer := spdy.NewDialer(upgrader, &http.Client{Transport: transport}, "POST", url)
	ports := []string{fmt.Sprintf("%d:%d", localPort, remotePort)}
	readyChan := make(chan struct{}, 1)
	stopChan := make(chan struct{}, 1)

	// Ensure that the port forwarder isn't already shutdown, and append the
	// shutdown channel so this forwarder can be closed
	f.stopChansLock.Lock()
	if f.shutdown {
		f.stopChansLock.Unlock()
		return 0, fmt.Errorf("port forwarder is shutdown")
	}
	f.stopChans = append(f.stopChans, stopChan)
	f.stopChansLock.Unlock()

	fw, err := portforward.New(dialer, ports, stopChan, readyChan, ioutil.Discard, f.logger)
	if err != nil {
		return 0, err
	}

	errChan := make(chan error, 1)
	go func() { errChan <- fw.ForwardPorts() }()

	select {
	case err = <-errChan:
		return 0, fmt.Errorf("port forwarding failed: %v", err)
	case <-fw.Ready:
	}

	// don't discover the locally bound port if we already know what it is
	if localPort != 0 {
		return localPort, nil
	}

	// discover the locally bound port if we don't know what it is
	bindings, err := fw.GetPorts()
	if err != nil {
		return 0, fmt.Errorf("failed to fetch local bound ports: %v", err)
	}

	for _, binding := range bindings {
		if binding.Remote == remotePort {
			return binding.Local, nil
		}
	}

	return 0, errors.New("failed to discover local bound port")
}

// RunForDaemon creates a port forwarder for the pachd daemon.
func (f *PortForwarder) RunForDaemon(localPort, remotePort uint16) (uint16, error) {
	return f.Run("pachd", localPort, remotePort)
}

// RunForSAMLACS creates a port forwarder for SAML ACS.
func (f *PortForwarder) RunForSAMLACS(localPort uint16) (uint16, error) {
	return f.Run("pachd", localPort, 654)
}

// RunForDashUI creates a port forwarder for the dash UI.
func (f *PortForwarder) RunForDashUI(localPort uint16) (uint16, error) {
	return f.Run("dash", localPort, 8080)
}

// RunForDashWebSocket creates a port forwarder for the dash websocket.
func (f *PortForwarder) RunForDashWebSocket(localPort uint16) (uint16, error) {
	return f.Run("dash", localPort, 8081)
}

// RunForPFS creates a port forwarder for PFS over HTTP.
func (f *PortForwarder) RunForPFS(localPort uint16) (uint16, error) {
	return f.Run("pachd", localPort, 30652)
}

// RunForS3Gateway creates a port forwarder for the s3gateway.
func (f *PortForwarder) RunForS3Gateway(localPort uint16) (uint16, error) {
	return f.Run("pachd", localPort, 600)
}

// Close shuts down port forwarding.
func (f *PortForwarder) Close() {
	defer f.logger.Close()

	f.stopChansLock.Lock()
	defer f.stopChansLock.Unlock()

	if f.shutdown {
		panic("port forwarder already shutdown")
	}

	f.shutdown = true

	for _, stopChan := range f.stopChans {
		close(stopChan)
	}
}
<commit_msg>Fixed typo<commit_after>package client

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"sync"

	"github.com/pachyderm/pachyderm/src/client/pkg/config"

	log "github.com/sirupsen/logrus"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
	_ "k8s.io/client-go/plugin/pkg/client/auth" // enables support for configs with auth
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/portforward"
	"k8s.io/client-go/transport/spdy"
)

// PortForwarder handles proxying local traffic to a kubernetes pod
type PortForwarder struct {
	core          corev1.CoreV1Interface
	client        rest.Interface
	config        *rest.Config
	namespace     string
	logger        *io.PipeWriter
	stopChansLock *sync.Mutex
	stopChans     []chan struct{}
	shutdown      bool
}

// NewPortForwarder creates a new port forwarder
func NewPortForwarder(namespace string) (*PortForwarder, error) {
	cfg, err := config.Read()
	if err != nil {
		return nil, fmt.Errorf("could not read config: %v", err)
	}
	_, context, err := cfg.ActiveContext()
	if err != nil {
		return nil, fmt.Errorf("could not get active context: %v", err)
	}

	if namespace == "" {
		namespace = context.Namespace
	}
	if namespace == "" {
		namespace = "default"
	}

	kubeConfig := config.KubeConfig(context)

	kubeClientConfig, err := kubeConfig.ClientConfig()
	if err != nil {
		return nil, err
	}

	client, err := kubernetes.NewForConfig(kubeClientConfig)
	if err != nil {
		return nil, err
	}

	core := client.CoreV1()

	return &PortForwarder{
		core:          core,
		client:        core.RESTClient(),
		config:        kubeClientConfig,
		namespace:     namespace,
		logger:        log.StandardLogger().Writer(),
		stopChansLock: &sync.Mutex{},
		stopChans:     []chan struct{}{},
		shutdown:      false,
	}, nil
}

// Run starts the port forwarder. Returns after initialization is begun with
// the locally bound port and any initialization errors.
func (f *PortForwarder) Run(appName string, localPort, remotePort uint16) (uint16, error) {
	podNameSelector := map[string]string{
		"suite": "pachyderm",
		"app":   appName,
	}

	podList, err := f.core.Pods(f.namespace).List(metav1.ListOptions{
		LabelSelector: metav1.FormatLabelSelector(metav1.SetAsLabelSelector(podNameSelector)),
		TypeMeta: metav1.TypeMeta{
			Kind:       "ListOptions",
			APIVersion: "v1",
		},
	})
	if err != nil {
		return 0, err
	}
	if len(podList.Items) == 0 {
		return 0, fmt.Errorf("no pods found for app %s", appName)
	}

	// Choose a random pod
	podName := podList.Items[rand.Intn(len(podList.Items))].Name

	url := f.client.Post().
		Resource("pods").
		Namespace(f.namespace).
		Name(podName).
		SubResource("portforward").
		URL()

	transport, upgrader, err := spdy.RoundTripperFor(f.config)
	if err != nil {
		return 0, err
	}

	dialer := spdy.NewDialer(upgrader, &http.Client{Transport: transport}, "POST", url)
	ports := []string{fmt.Sprintf("%d:%d", localPort, remotePort)}
	readyChan := make(chan struct{}, 1)
	stopChan := make(chan struct{}, 1)

	// Ensure that the port forwarder isn't already shutdown, and append the
	// shutdown channel so this forwarder can be closed
	f.stopChansLock.Lock()
	if f.shutdown {
		f.stopChansLock.Unlock()
		return 0, fmt.Errorf("port forwarder is shutdown")
	}
	f.stopChans = append(f.stopChans, stopChan)
	f.stopChansLock.Unlock()

	fw, err := portforward.New(dialer, ports, stopChan, readyChan, ioutil.Discard, f.logger)
	if err != nil {
		return 0, err
	}

	errChan := make(chan error, 1)
	go func() { errChan <- fw.ForwardPorts() }()

	select {
	case err = <-errChan:
		return 0, fmt.Errorf("port forwarding failed: %v", err)
	case <-fw.Ready:
	}

	// don't discover the locally bound port if we already know what it is
	if localPort != 0 {
		return localPort, nil
	}

	// discover the locally bound port if we don't know what it is
	bindings, err := fw.GetPorts()
	if err != nil {
		return 0, fmt.Errorf("failed to fetch local bound ports: %v", err)
	}

	for _, binding := range bindings {
		if binding.Remote == remotePort {
			return binding.Local, nil
		}
	}

	return 0, errors.New("failed to discover local bound port")
}

// RunForDaemon creates a port forwarder for the pachd daemon.
func (f *PortForwarder) RunForDaemon(localPort, remotePort uint16) (uint16, error) {
	return f.Run("pachd", localPort, remotePort)
}

// RunForSAMLACS creates a port forwarder for SAML ACS.
func (f *PortForwarder) RunForSAMLACS(localPort uint16) (uint16, error) {
	return f.Run("pachd", localPort, 654)
}

// RunForDashUI creates a port forwarder for the dash UI.
func (f *PortForwarder) RunForDashUI(localPort uint16) (uint16, error) {
	return f.Run("dash", localPort, 8080)
}

// RunForDashWebSocket creates a port forwarder for the dash websocket.
func (f *PortForwarder) RunForDashWebSocket(localPort uint16) (uint16, error) {
	return f.Run("dash", localPort, 8081)
}

// RunForPFS creates a port forwarder for PFS over HTTP.
func (f *PortForwarder) RunForPFS(localPort uint16) (uint16, error) {
	return f.Run("pachd", localPort, 30652)
}

// RunForS3Gateway creates a port forwarder for the s3gateway.
func (f *PortForwarder) RunForS3Gateway(localPort uint16) (uint16, error) {
	return f.Run("pachd", localPort, 600)
}

// Close shuts down port forwarding.
func (f *PortForwarder) Close() {
	defer f.logger.Close()

	f.stopChansLock.Lock()
	defer f.stopChansLock.Unlock()

	if f.shutdown {
		panic("port forwarder already shutdown")
	}

	f.shutdown = true

	for _, stopChan := range f.stopChans {
		close(stopChan)
	}
}
<|endoftext|>


<commit_before>package tutum
<commit_msg>partial service struct + List services method<commit_after>package tutum

import "encoding/json"

type SListResponse struct {
	Objects []Service `json: "objects"`
}

type Service struct {
	Autodestroy            string   `json:"autodestroy"`
	Autoredeploy           bool     `json:"autoredeploy"`
	Autorestart            string   `json:"autorestart"`
	Containers             []string `json:"containers"`
	Current_num_containers int      `json:"current_num_containers"`
	Uuid                   string   `json:"uuid"`
}

func ListServices() ([]Service, error) {
	url := "service/"
	request := "GET"

	var response SListResponse
	data, err := TutumCall(url, request)
	if err != nil {
		return nil, err
	}
	err = json.Unmarshal(data, &response)
	if err != nil {
		return nil, err
	}
	return response.Objects, nil

}
<|endoftext|>


<commit_before><commit_msg>Fix the CI lint error about receiver name<commit_after><|endoftext|>


<commit_before>package packfile

import (
	"bytes"
	"errors"
	"io"
	stdioutil "io/ioutil"

	"github.com/go-git/go-git/v5/plumbing"
	"github.com/go-git/go-git/v5/plumbing/cache"
	"github.com/go-git/go-git/v5/plumbing/storer"
	"github.com/go-git/go-git/v5/utils/ioutil"
)

var (
	// ErrReferenceDeltaNotFound is returned when the reference delta is not
	// found.
	ErrReferenceDeltaNotFound = errors.New("reference delta not found")

	// ErrNotSeekableSource is returned when the source for the parser is not
	// seekable and a storage was not provided, so it can't be parsed.
	ErrNotSeekableSource = errors.New("parser source is not seekable and storage was not provided")

	// ErrDeltaNotCached is returned when the delta could not be found in cache.
	ErrDeltaNotCached = errors.New("delta could not be found in cache")
)

// Observer interface is implemented by index encoders.
type Observer interface {
	// OnHeader is called when a new packfile is opened.
	OnHeader(count uint32) error
	// OnInflatedObjectHeader is called for each object header read.
	OnInflatedObjectHeader(t plumbing.ObjectType, objSize int64, pos int64) error
	// OnInflatedObjectContent is called for each decoded object.
	OnInflatedObjectContent(h plumbing.Hash, pos int64, crc uint32, content []byte) error
	// OnFooter is called when decoding is done.
	OnFooter(h plumbing.Hash) error
}

// Parser decodes a packfile and calls any observer associated to it. Is used
// to generate indexes.
type Parser struct {
	storage    storer.EncodedObjectStorer
	scanner    *Scanner
	count      uint32
	oi         []*objectInfo
	oiByHash   map[plumbing.Hash]*objectInfo
	oiByOffset map[int64]*objectInfo
	hashOffset map[plumbing.Hash]int64
	checksum   plumbing.Hash

	cache *cache.BufferLRU
	// delta content by offset, only used if source is not seekable
	deltas map[int64][]byte

	ob []Observer
}

// NewParser creates a new Parser. The Scanner source must be seekable.
// If it's not, NewParserWithStorage should be used instead.
func NewParser(scanner *Scanner, ob ...Observer) (*Parser, error) {
	return NewParserWithStorage(scanner, nil, ob...)
}

// NewParserWithStorage creates a new Parser. The scanner source must either
// be seekable or a storage must be provided.
func NewParserWithStorage(
	scanner *Scanner,
	storage storer.EncodedObjectStorer,
	ob ...Observer,
) (*Parser, error) {
	if !scanner.IsSeekable && storage == nil {
		return nil, ErrNotSeekableSource
	}

	var deltas map[int64][]byte
	if !scanner.IsSeekable {
		deltas = make(map[int64][]byte)
	}

	return &Parser{
		storage: storage,
		scanner: scanner,
		ob:      ob,
		count:   0,
		cache:   cache.NewBufferLRUDefault(),
		deltas:  deltas,
	}, nil
}

func (p *Parser) forEachObserver(f func(o Observer) error) error {
	for _, o := range p.ob {
		if err := f(o); err != nil {
			return err
		}
	}
	return nil
}

func (p *Parser) onHeader(count uint32) error {
	return p.forEachObserver(func(o Observer) error {
		return o.OnHeader(count)
	})
}

func (p *Parser) onInflatedObjectHeader(
	t plumbing.ObjectType,
	objSize int64,
	pos int64,
) error {
	return p.forEachObserver(func(o Observer) error {
		return o.OnInflatedObjectHeader(t, objSize, pos)
	})
}

func (p *Parser) onInflatedObjectContent(
	h plumbing.Hash,
	pos int64,
	crc uint32,
	content []byte,
) error {
	return p.forEachObserver(func(o Observer) error {
		return o.OnInflatedObjectContent(h, pos, crc, content)
	})
}

func (p *Parser) onFooter(h plumbing.Hash) error {
	return p.forEachObserver(func(o Observer) error {
		return o.OnFooter(h)
	})
}

// Parse start decoding phase of the packfile.
func (p *Parser) Parse() (plumbing.Hash, error) {
	if err := p.init(); err != nil {
		return plumbing.ZeroHash, err
	}

	if err := p.indexObjects(); err != nil {
		return plumbing.ZeroHash, err
	}

	var err error
	p.checksum, err = p.scanner.Checksum()
	if err != nil && err != io.EOF {
		return plumbing.ZeroHash, err
	}

	if err := p.resolveDeltas(); err != nil {
		return plumbing.ZeroHash, err
	}

	if err := p.onFooter(p.checksum); err != nil {
		return plumbing.ZeroHash, err
	}

	return p.checksum, nil
}

func (p *Parser) init() error {
	_, c, err := p.scanner.Header()
	if err != nil {
		return err
	}

	if err := p.onHeader(c); err != nil {
		return err
	}

	p.count = c
	p.oiByHash = make(map[plumbing.Hash]*objectInfo, p.count)
	p.oiByOffset = make(map[int64]*objectInfo, p.count)
	p.oi = make([]*objectInfo, p.count)

	return nil
}

func (p *Parser) indexObjects() error {
	buf := new(bytes.Buffer)

	for i := uint32(0); i < p.count; i++ {
		buf.Reset()

		oh, err := p.scanner.NextObjectHeader()
		if err != nil {
			return err
		}

		delta := false
		var ota *objectInfo
		switch t := oh.Type; t {
		case plumbing.OFSDeltaObject:
			delta = true

			parent, ok := p.oiByOffset[oh.OffsetReference]
			if !ok {
				return plumbing.ErrObjectNotFound
			}

			ota = newDeltaObject(oh.Offset, oh.Length, t, parent)
			parent.Children = append(parent.Children, ota)
		case plumbing.REFDeltaObject:
			delta = true
			parent, ok := p.oiByHash[oh.Reference]
			if !ok {
				// can't find referenced object in this pack file
				// this must be a "thin" pack.
				parent = &objectInfo{ //Placeholder parent
					SHA1:        oh.Reference,
					ExternalRef: true, // mark as an external reference that must be resolved
					Type:        plumbing.AnyObject,
					DiskType:    plumbing.AnyObject,
				}
				p.oiByHash[oh.Reference] = parent
			}
			ota = newDeltaObject(oh.Offset, oh.Length, t, parent)
			parent.Children = append(parent.Children, ota)

		default:
			ota = newBaseObject(oh.Offset, oh.Length, t)
		}

		_, crc, err := p.scanner.NextObject(buf)
		if err != nil {
			return err
		}

		ota.Crc32 = crc
		ota.Length = oh.Length

		data := buf.Bytes()
		if !delta {
			sha1, err := getSHA1(ota.Type, data)
			if err != nil {
				return err
			}

			ota.SHA1 = sha1
			p.oiByHash[ota.SHA1] = ota
		}

		if p.storage != nil && !delta {
			obj := new(plumbing.MemoryObject)
			obj.SetSize(oh.Length)
			obj.SetType(oh.Type)
			if _, err := obj.Write(data); err != nil {
				return err
			}

			if _, err := p.storage.SetEncodedObject(obj); err != nil {
				return err
			}
		}

		if delta && !p.scanner.IsSeekable {
			p.deltas[oh.Offset] = make([]byte, len(data))
			copy(p.deltas[oh.Offset], data)
		}

		p.oiByOffset[oh.Offset] = ota
		p.oi[i] = ota
	}

	return nil
}

func (p *Parser) resolveDeltas() error {
	buf := &bytes.Buffer{}
	for _, obj := range p.oi {
		buf.Reset()
		err := p.get(obj, buf)
		if err != nil {
			return err
		}
		content := buf.Bytes()

		if err := p.onInflatedObjectHeader(obj.Type, obj.Length, obj.Offset); err != nil {
			return err
		}

		if err := p.onInflatedObjectContent(obj.SHA1, obj.Offset, obj.Crc32, content); err != nil {
			return err
		}

		if !obj.IsDelta() && len(obj.Children) > 0 {
			for _, child := range obj.Children {
				if err := p.resolveObject(stdioutil.Discard, child, content); err != nil {
					return err
				}
			}

			// Remove the delta from the cache.
			if obj.DiskType.IsDelta() && !p.scanner.IsSeekable {
				delete(p.deltas, obj.Offset)
			}
		}
	}

	return nil
}

func (p *Parser) get(o *objectInfo, buf *bytes.Buffer) (err error) {
	if !o.ExternalRef { // skip cache check for placeholder parents
		b, ok := p.cache.Get(o.Offset)
		if ok {
			_, err := buf.Write(b)
			return err
		}
	}

	// If it's not on the cache and is not a delta we can try to find it in the
	// storage, if there's one. External refs must enter here.
	if p.storage != nil && !o.Type.IsDelta() {
		var e plumbing.EncodedObject
		e, err = p.storage.EncodedObject(plumbing.AnyObject, o.SHA1)
		if err != nil {
			return err
		}
		o.Type = e.Type()

		var r io.ReadCloser
		r, err = e.Reader()
		if err != nil {
			return err
		}

		defer ioutil.CheckClose(r, &err)

		_, err = buf.ReadFrom(io.LimitReader(r, e.Size()))
		return err
	}

	if o.ExternalRef {
		// we were not able to resolve a ref in a thin pack
		return ErrReferenceDeltaNotFound
	}

	if o.DiskType.IsDelta() {
		b := bufPool.Get().(*bytes.Buffer)
		defer bufPool.Put(b)
		b.Reset()
		err := p.get(o.Parent, b)
		if err != nil {
			return err
		}
		base := b.Bytes()

		err = p.resolveObject(buf, o, base)
		if err != nil {
			return err
		}
	} else {
		err := p.readData(buf, o)
		if err != nil {
			return err
		}
	}

	if len(o.Children) > 0 {
		data := make([]byte, buf.Len())
		copy(data, buf.Bytes())
		p.cache.Put(o.Offset, data)
	}
	return nil
}

func (p *Parser) resolveObject(
	w io.Writer,
	o *objectInfo,
	base []byte,
) error {
	if !o.DiskType.IsDelta() {
		return nil
	}
	buf := bufPool.Get().(*bytes.Buffer)
	defer bufPool.Put(buf)
	buf.Reset()
	err := p.readData(buf, o)
	if err != nil {
		return err
	}
	data := buf.Bytes()

	data, err = applyPatchBase(o, data, base)
	if err != nil {
		return err
	}

	if p.storage != nil {
		obj := new(plumbing.MemoryObject)
		obj.SetSize(o.Size())
		obj.SetType(o.Type)
		if _, err := obj.Write(data); err != nil {
			return err
		}

		if _, err := p.storage.SetEncodedObject(obj); err != nil {
			return err
		}
	}
	_, err = w.Write(data)
	return err
}

func (p *Parser) readData(w io.Writer, o *objectInfo) error {
	if !p.scanner.IsSeekable && o.DiskType.IsDelta() {
		data, ok := p.deltas[o.Offset]
		if !ok {
			return ErrDeltaNotCached
		}
		_, err := w.Write(data)
		return err
	}

	if _, err := p.scanner.SeekObjectHeader(o.Offset); err != nil {
		return err
	}

	if _, _, err := p.scanner.NextObject(w); err != nil {
		return err
	}
	return nil
}

func applyPatchBase(ota *objectInfo, data, base []byte) ([]byte, error) {
	patched, err := PatchDelta(base, data)
	if err != nil {
		return nil, err
	}

	if ota.SHA1 == plumbing.ZeroHash {
		ota.Type = ota.Parent.Type
		sha1, err := getSHA1(ota.Type, patched)
		if err != nil {
			return nil, err
		}

		ota.SHA1 = sha1
		ota.Length = int64(len(patched))
	}

	return patched, nil
}

func getSHA1(t plumbing.ObjectType, data []byte) (plumbing.Hash, error) {
	hasher := plumbing.NewHasher(t, int64(len(data)))
	if _, err := hasher.Write(data); err != nil {
		return plumbing.ZeroHash, err
	}

	return hasher.Sum(), nil
}

type objectInfo struct {
	Offset      int64
	Length      int64
	Type        plumbing.ObjectType
	DiskType    plumbing.ObjectType
	ExternalRef bool // indicates this is an external reference in a thin pack file

	Crc32 uint32

	Parent   *objectInfo
	Children []*objectInfo
	SHA1     plumbing.Hash
}

func newBaseObject(offset, length int64, t plumbing.ObjectType) *objectInfo {
	return newDeltaObject(offset, length, t, nil)
}

func newDeltaObject(
	offset, length int64,
	t plumbing.ObjectType,
	parent *objectInfo,
) *objectInfo {
	obj := &objectInfo{
		Offset:   offset,
		Length:   length,
		Type:     t,
		DiskType: t,
		Crc32:    0,
		Parent:   parent,
	}

	return obj
}

func (o *objectInfo) IsDelta() bool {
	return o.Type.IsDelta()
}

func (o *objectInfo) Size() int64 {
	return o.Length
}
<commit_msg>resolve external reference deltas<commit_after>package packfile

import (
	"bytes"
	"errors"
	"io"
	stdioutil "io/ioutil"

	"github.com/go-git/go-git/v5/plumbing"
	"github.com/go-git/go-git/v5/plumbing/cache"
	"github.com/go-git/go-git/v5/plumbing/storer"
	"github.com/go-git/go-git/v5/utils/ioutil"
)

var (
	// ErrReferenceDeltaNotFound is returned when the reference delta is not
	// found.
	ErrReferenceDeltaNotFound = errors.New("reference delta not found")

	// ErrNotSeekableSource is returned when the source for the parser is not
	// seekable and a storage was not provided, so it can't be parsed.
	ErrNotSeekableSource = errors.New("parser source is not seekable and storage was not provided")

	// ErrDeltaNotCached is returned when the delta could not be found in cache.
	ErrDeltaNotCached = errors.New("delta could not be found in cache")
)

// Observer interface is implemented by index encoders.
type Observer interface {
	// OnHeader is called when a new packfile is opened.
	OnHeader(count uint32) error
	// OnInflatedObjectHeader is called for each object header read.
	OnInflatedObjectHeader(t plumbing.ObjectType, objSize int64, pos int64) error
	// OnInflatedObjectContent is called for each decoded object.
	OnInflatedObjectContent(h plumbing.Hash, pos int64, crc uint32, content []byte) error
	// OnFooter is called when decoding is done.
	OnFooter(h plumbing.Hash) error
}

// Parser decodes a packfile and calls any observer associated to it. Is used
// to generate indexes.
type Parser struct {
	storage    storer.EncodedObjectStorer
	scanner    *Scanner
	count      uint32
	oi         []*objectInfo
	oiByHash   map[plumbing.Hash]*objectInfo
	oiByOffset map[int64]*objectInfo
	hashOffset map[plumbing.Hash]int64
	checksum   plumbing.Hash

	cache *cache.BufferLRU
	// delta content by offset, only used if source is not seekable
	deltas map[int64][]byte

	ob []Observer
}

// NewParser creates a new Parser. The Scanner source must be seekable.
// If it's not, NewParserWithStorage should be used instead.
func NewParser(scanner *Scanner, ob ...Observer) (*Parser, error) {
	return NewParserWithStorage(scanner, nil, ob...)
}

// NewParserWithStorage creates a new Parser. The scanner source must either
// be seekable or a storage must be provided.
func NewParserWithStorage(
	scanner *Scanner,
	storage storer.EncodedObjectStorer,
	ob ...Observer,
) (*Parser, error) {
	if !scanner.IsSeekable && storage == nil {
		return nil, ErrNotSeekableSource
	}

	var deltas map[int64][]byte
	if !scanner.IsSeekable {
		deltas = make(map[int64][]byte)
	}

	return &Parser{
		storage: storage,
		scanner: scanner,
		ob:      ob,
		count:   0,
		cache:   cache.NewBufferLRUDefault(),
		deltas:  deltas,
	}, nil
}

func (p *Parser) forEachObserver(f func(o Observer) error) error {
	for _, o := range p.ob {
		if err := f(o); err != nil {
			return err
		}
	}
	return nil
}

func (p *Parser) onHeader(count uint32) error {
	return p.forEachObserver(func(o Observer) error {
		return o.OnHeader(count)
	})
}

func (p *Parser) onInflatedObjectHeader(
	t plumbing.ObjectType,
	objSize int64,
	pos int64,
) error {
	return p.forEachObserver(func(o Observer) error {
		return o.OnInflatedObjectHeader(t, objSize, pos)
	})
}

func (p *Parser) onInflatedObjectContent(
	h plumbing.Hash,
	pos int64,
	crc uint32,
	content []byte,
) error {
	return p.forEachObserver(func(o Observer) error {
		return o.OnInflatedObjectContent(h, pos, crc, content)
	})
}

func (p *Parser) onFooter(h plumbing.Hash) error {
	return p.forEachObserver(func(o Observer) error {
		return o.OnFooter(h)
	})
}

// Parse start decoding phase of the packfile.
func (p *Parser) Parse() (plumbing.Hash, error) {
	if err := p.init(); err != nil {
		return plumbing.ZeroHash, err
	}

	if err := p.indexObjects(); err != nil {
		return plumbing.ZeroHash, err
	}

	var err error
	p.checksum, err = p.scanner.Checksum()
	if err != nil && err != io.EOF {
		return plumbing.ZeroHash, err
	}

	if err := p.resolveDeltas(); err != nil {
		return plumbing.ZeroHash, err
	}

	if err := p.onFooter(p.checksum); err != nil {
		return plumbing.ZeroHash, err
	}

	return p.checksum, nil
}

func (p *Parser) init() error {
	_, c, err := p.scanner.Header()
	if err != nil {
		return err
	}

	if err := p.onHeader(c); err != nil {
		return err
	}

	p.count = c
	p.oiByHash = make(map[plumbing.Hash]*objectInfo, p.count)
	p.oiByOffset = make(map[int64]*objectInfo, p.count)
	p.oi = make([]*objectInfo, p.count)

	return nil
}

func (p *Parser) indexObjects() error {
	buf := new(bytes.Buffer)

	for i := uint32(0); i < p.count; i++ {
		buf.Reset()

		oh, err := p.scanner.NextObjectHeader()
		if err != nil {
			return err
		}

		delta := false
		var ota *objectInfo
		switch t := oh.Type; t {
		case plumbing.OFSDeltaObject:
			delta = true

			parent, ok := p.oiByOffset[oh.OffsetReference]
			if !ok {
				return plumbing.ErrObjectNotFound
			}

			ota = newDeltaObject(oh.Offset, oh.Length, t, parent)
			parent.Children = append(parent.Children, ota)
		case plumbing.REFDeltaObject:
			delta = true
			parent, ok := p.oiByHash[oh.Reference]
			if !ok {
				// can't find referenced object in this pack file
				// this must be a "thin" pack.
				parent = &objectInfo{ //Placeholder parent
					SHA1:        oh.Reference,
					ExternalRef: true, // mark as an external reference that must be resolved
					Type:        plumbing.AnyObject,
					DiskType:    plumbing.AnyObject,
				}
				p.oiByHash[oh.Reference] = parent
			}
			ota = newDeltaObject(oh.Offset, oh.Length, t, parent)
			parent.Children = append(parent.Children, ota)

		default:
			ota = newBaseObject(oh.Offset, oh.Length, t)
		}

		_, crc, err := p.scanner.NextObject(buf)
		if err != nil {
			return err
		}

		ota.Crc32 = crc
		ota.Length = oh.Length

		data := buf.Bytes()
		if !delta {
			sha1, err := getSHA1(ota.Type, data)
			if err != nil {
				return err
			}

			ota.SHA1 = sha1
			p.oiByHash[ota.SHA1] = ota
		}

		if p.storage != nil && !delta {
			obj := new(plumbing.MemoryObject)
			obj.SetSize(oh.Length)
			obj.SetType(oh.Type)
			if _, err := obj.Write(data); err != nil {
				return err
			}

			if _, err := p.storage.SetEncodedObject(obj); err != nil {
				return err
			}
		}

		if delta && !p.scanner.IsSeekable {
			p.deltas[oh.Offset] = make([]byte, len(data))
			copy(p.deltas[oh.Offset], data)
		}

		p.oiByOffset[oh.Offset] = ota
		p.oi[i] = ota
	}

	return nil
}

func (p *Parser) resolveDeltas() error {
	buf := &bytes.Buffer{}
	for _, obj := range p.oi {
		buf.Reset()
		err := p.get(obj, buf)
		if err != nil {
			return err
		}
		content := buf.Bytes()

		if err := p.onInflatedObjectHeader(obj.Type, obj.Length, obj.Offset); err != nil {
			return err
		}

		if err := p.onInflatedObjectContent(obj.SHA1, obj.Offset, obj.Crc32, content); err != nil {
			return err
		}

		if !obj.IsDelta() && len(obj.Children) > 0 {
			for _, child := range obj.Children {
				if err := p.resolveObject(stdioutil.Discard, child, content); err != nil {
					return err
				}
				p.resolveExternalRef(child)
			}

			// Remove the delta from the cache.
			if obj.DiskType.IsDelta() && !p.scanner.IsSeekable {
				delete(p.deltas, obj.Offset)
			}
		}
	}

	return nil
}

func (p *Parser) resolveExternalRef(o *objectInfo) {
	if ref, ok := p.oiByHash[o.SHA1]; ok && ref.ExternalRef {
		p.oiByHash[o.SHA1] = o
		o.Children = ref.Children
		for _, c := range o.Children {
			c.Parent = o
		}
	}
}

func (p *Parser) get(o *objectInfo, buf *bytes.Buffer) (err error) {
	if !o.ExternalRef { // skip cache check for placeholder parents
		b, ok := p.cache.Get(o.Offset)
		if ok {
			_, err := buf.Write(b)
			return err
		}
	}

	// If it's not on the cache and is not a delta we can try to find it in the
	// storage, if there's one. External refs must enter here.
	if p.storage != nil && !o.Type.IsDelta() {
		var e plumbing.EncodedObject
		e, err = p.storage.EncodedObject(plumbing.AnyObject, o.SHA1)
		if err != nil {
			return err
		}
		o.Type = e.Type()

		var r io.ReadCloser
		r, err = e.Reader()
		if err != nil {
			return err
		}

		defer ioutil.CheckClose(r, &err)

		_, err = buf.ReadFrom(io.LimitReader(r, e.Size()))
		return err
	}

	if o.ExternalRef {
		// we were not able to resolve a ref in a thin pack
		return ErrReferenceDeltaNotFound
	}

	if o.DiskType.IsDelta() {
		b := bufPool.Get().(*bytes.Buffer)
		defer bufPool.Put(b)
		b.Reset()
		err := p.get(o.Parent, b)
		if err != nil {
			return err
		}
		base := b.Bytes()

		err = p.resolveObject(buf, o, base)
		if err != nil {
			return err
		}
	} else {
		err := p.readData(buf, o)
		if err != nil {
			return err
		}
	}

	if len(o.Children) > 0 {
		data := make([]byte, buf.Len())
		copy(data, buf.Bytes())
		p.cache.Put(o.Offset, data)
	}
	return nil
}

func (p *Parser) resolveObject(
	w io.Writer,
	o *objectInfo,
	base []byte,
) error {
	if !o.DiskType.IsDelta() {
		return nil
	}
	buf := bufPool.Get().(*bytes.Buffer)
	defer bufPool.Put(buf)
	buf.Reset()
	err := p.readData(buf, o)
	if err != nil {
		return err
	}
	data := buf.Bytes()

	data, err = applyPatchBase(o, data, base)
	if err != nil {
		return err
	}

	if p.storage != nil {
		obj := new(plumbing.MemoryObject)
		obj.SetSize(o.Size())
		obj.SetType(o.Type)
		if _, err := obj.Write(data); err != nil {
			return err
		}

		if _, err := p.storage.SetEncodedObject(obj); err != nil {
			return err
		}
	}
	_, err = w.Write(data)
	return err
}

func (p *Parser) readData(w io.Writer, o *objectInfo) error {
	if !p.scanner.IsSeekable && o.DiskType.IsDelta() {
		data, ok := p.deltas[o.Offset]
		if !ok {
			return ErrDeltaNotCached
		}
		_, err := w.Write(data)
		return err
	}

	if _, err := p.scanner.SeekObjectHeader(o.Offset); err != nil {
		return err
	}

	if _, _, err := p.scanner.NextObject(w); err != nil {
		return err
	}
	return nil
}

func applyPatchBase(ota *objectInfo, data, base []byte) ([]byte, error) {
	patched, err := PatchDelta(base, data)
	if err != nil {
		return nil, err
	}

	if ota.SHA1 == plumbing.ZeroHash {
		ota.Type = ota.Parent.Type
		sha1, err := getSHA1(ota.Type, patched)
		if err != nil {
			return nil, err
		}

		ota.SHA1 = sha1
		ota.Length = int64(len(patched))
	}

	return patched, nil
}

func getSHA1(t plumbing.ObjectType, data []byte) (plumbing.Hash, error) {
	hasher := plumbing.NewHasher(t, int64(len(data)))
	if _, err := hasher.Write(data); err != nil {
		return plumbing.ZeroHash, err
	}

	return hasher.Sum(), nil
}

type objectInfo struct {
	Offset      int64
	Length      int64
	Type        plumbing.ObjectType
	DiskType    plumbing.ObjectType
	ExternalRef bool // indicates this is an external reference in a thin pack file

	Crc32 uint32

	Parent   *objectInfo
	Children []*objectInfo
	SHA1     plumbing.Hash
}

func newBaseObject(offset, length int64, t plumbing.ObjectType) *objectInfo {
	return newDeltaObject(offset, length, t, nil)
}

func newDeltaObject(
	offset, length int64,
	t plumbing.ObjectType,
	parent *objectInfo,
) *objectInfo {
	obj := &objectInfo{
		Offset:   offset,
		Length:   length,
		Type:     t,
		DiskType: t,
		Crc32:    0,
		Parent:   parent,
	}

	return obj
}

func (o *objectInfo) IsDelta() bool {
	return o.Type.IsDelta()
}

func (o *objectInfo) Size() int64 {
	return o.Length
}
<|endoftext|>


<commit_before>package main

import (
	"testing"

	. "github.com/bborbe/assert"
	io_mock "github.com/bborbe/io/mock"
)

func TestDo(t *testing.T) {
	var err error
	writer := io_mock.NewWriter()
	input := io_mock.NewReadCloserString("  http://www.exmaple.com  ")
	err = do(writer, input)
	err = AssertThat(err, NilValue())
	if err != nil {
		t.Fatal(err)
	}
	err = AssertThat(string(writer.Content()), Is("http://www.exmaple.com\n"))
	if err != nil {
		t.Fatal(err)
	}
}
<commit_msg>fix url parsing<commit_after>package main

import (
	"testing"

	. "github.com/bborbe/assert"
	io_mock "github.com/bborbe/io/mock"
)

func TestDo(t *testing.T) {
	var err error
	writer := io_mock.NewWriter()
	input := io_mock.NewReadCloserString(" http://www.example.com ")
	err = do(writer, input)
	err = AssertThat(err, NilValue())
	if err != nil {
		t.Fatal(err)
	}
	err = AssertThat(string(writer.Content()), Is("http://www.example.com\n"))
	if err != nil {
		t.Fatal(err)
	}
}

func TestDoBugfix(t *testing.T) {
	var err error
	writer := io_mock.NewWriter()
	input := io_mock.NewReadCloserString("\n2015-04-01T15:40:09 http://www.example.com\n2015-04-01T15:40:09 http://www.example.com\n")
	err = do(writer, input)
	err = AssertThat(err, NilValue())
	if err != nil {
		t.Fatal(err)
	}
	err = AssertThat(string(writer.Content()), Is("http://www.example.com\nhttp://www.example.com\n"))
	if err != nil {
		t.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package micro

import (
	"context"
	"sync"
	"testing"

	"github.com/micro/go-micro/registry/mock"
	proto "github.com/micro/go-micro/server/debug/proto"
)

func TestFunction(t *testing.T) {
	var wg sync.WaitGroup
	wg.Add(1)

	// create service
	fn := NewFunction(
		Name("test.function"),
		Registry(mock.NewRegistry()),
		AfterStart(func() error {
			wg.Done()
			return nil
		}),
	)

	// we can't test fn.Init as it parses the command line
	// fn.Init()

	go func() {
		// wait for start
		wg.Wait()

		// test call debug
		req := fn.Client().NewRequest(
			"test.function",
			"Debug.Health",
			new(proto.HealthRequest),
		)

		rsp := new(proto.HealthResponse)

		err := fn.Client().Call(context.TODO(), req, rsp)
		if err != nil {
			t.Fatal(err)
		}

		if rsp.Status != "ok" {
			t.Fatalf("function response: %s", rsp.Status)
		}
	}()

	// run service
	if err := fn.Run(); err != nil {
		t.Fatal(err)
	}
}
<commit_msg>shutdown broker once done<commit_after>package micro

import (
	"context"
	"sync"
	"testing"

	"github.com/micro/go-micro/registry/mock"
	proto "github.com/micro/go-micro/server/debug/proto"
)

func TestFunction(t *testing.T) {
	var wg sync.WaitGroup
	wg.Add(1)

	// cancellation context
	ctx, cancel := context.WithCancel(context.Background())

	// create service
	fn := NewFunction(
		Name("test.function"),
		Context(ctx),
		Registry(mock.NewRegistry()),
		AfterStart(func() error {
			wg.Done()
			return nil
		}),
	)

	// we can't test fn.Init as it parses the command line
	// fn.Init()

	go func() {
		// wait for start
		wg.Wait()

		// test call debug
		req := fn.Client().NewRequest(
			"test.function",
			"Debug.Health",
			new(proto.HealthRequest),
		)

		rsp := new(proto.HealthResponse)

		err := fn.Client().Call(context.TODO(), req, rsp)
		if err != nil {
			t.Fatal(err)
		}

		if rsp.Status != "ok" {
			t.Fatalf("function response: %s", rsp.Status)
		}

		cancel()
	}()

	// run service
	if err := fn.Run(); err != nil {
		t.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package toolbox

import (
	"fmt"
	"reflect"
	"strings"
)

//GetFunction returns function for provided owner and name, or error
func GetFunction(owner interface{}, name string) (interface{}, error) {
	if owner == nil {
		return nil, fmt.Errorf("failed to lookup %v on %T, owner was nil", name, owner)
	}
	var ownerType = reflect.TypeOf(owner)
	var method, has = ownerType.MethodByName(name)
	if !has {
		var available = make([]string, 0)
		for i := 0; i < ownerType.NumMethod(); i++ {
			available = append(available, ownerType.Method(i).Name)
		}
		return nil, fmt.Errorf("failed to lookup %T.%v, available:[%v]", owner, name, strings.Join(available, ","))
	}
	return reflect.ValueOf(owner).MethodByName(method.Name).Interface(), nil
}

//CallFunction calls passed in function with provided parameters,it returns a function result.
func CallFunction(function interface{}, parameters ...interface{}) []interface{} {
	AssertKind(function, reflect.Func, "function")
	var functionParameters = make([]reflect.Value, 0)
	ProcessSlice(parameters, func(item interface{}) bool {
		functionParameters = append(functionParameters, reflect.ValueOf(item))
		return true
	})

	functionValue := reflect.ValueOf(function)
	var resultValues = functionValue.Call(functionParameters)
	var result = make([]interface{}, len(resultValues))
	for i, resultValue := range resultValues {
		result[i] = resultValue.Interface()
	}
	return result
}

//AsCompatibleFunctionParameters takes incompatible function parameters and converts then into provided function signature compatible
func AsCompatibleFunctionParameters(function interface{}, parameters []interface{}) ([]interface{}, error) {
	return AsFunctionParameters(function, parameters, map[string]interface{}{})
}

//AsFunctionParameters takes incompatible function parameters and converts then into provided function signature compatible
func AsFunctionParameters(function interface{}, parameters []interface{}, parametersKV map[string]interface{}) ([]interface{}, error) {
	AssertKind(function, reflect.Func, "function")
	functionValue := reflect.ValueOf(function)
	funcSignature := GetFuncSignature(function)
	actualMethodSignatureLength := len(funcSignature)
	converter := Converter{}
	if actualMethodSignatureLength != len(parameters) {
		return nil, fmt.Errorf("invalid number of parameters wanted: [%T],  had: %v", function, len(parameters))
	}
	var functionParameters = make([]interface{}, 0)
	for i, parameterValue := range parameters {
		isStruct := IsStruct(funcSignature[i])
		reflectValue := reflect.ValueOf(parameterValue)
		if !isStruct {
			if parameterValue == nil {
				return nil, fmt.Errorf("parameter[%v] was empty", i)
			}
			if reflectValue.Kind() == reflect.Slice && funcSignature[i].Kind() != reflectValue.Kind() {
				return nil, fmt.Errorf("incompatible types expected: %v, but had %v", funcSignature[i].Kind(), reflectValue.Kind())
			} else if !reflectValue.IsValid() {
				if funcSignature[i].Kind() == reflect.Slice {
					parameterValue = reflect.New(funcSignature[i]).Interface()
					reflectValue = reflect.ValueOf(parameterValue)
				}
			}
		}
		if reflectValue.Type() != funcSignature[i] {
			newValuePointer := reflect.New(funcSignature[i])
			var err error
			if IsStruct(funcSignature[i]) && !(IsStruct(parameterValue) || IsMap(parameterValue)) {
				err = converter.AssignConverted(newValuePointer.Interface(), parametersKV)
			} else {
				err = converter.AssignConverted(newValuePointer.Interface(), parameterValue)
			}
			if err != nil {
				return nil, fmt.Errorf("failed to assign convert %v to %v due to %v", parametersKV, newValuePointer.Interface(), err)
			}
			reflectValue = newValuePointer.Elem()
		}
		if functionValue.Type().IsVariadic() && funcSignature[i].Kind() == reflect.Slice && i+1 == len(funcSignature) {
			ProcessSlice(reflectValue.Interface(), func(item interface{}) bool {
				functionParameters = append(functionParameters, item)
				return true
			})
		} else {
			functionParameters = append(functionParameters, reflectValue.Interface())
		}
	}
	return functionParameters, nil
}

//BuildFunctionParameters builds function parameters provided in the parameterValues.
// Parameters value will be converted if needed to expected by the function signature type. It returns function parameters , or error
func BuildFunctionParameters(function interface{}, parameters []string, parameterValues map[string]interface{}) ([]interface{}, error) {
	var functionParameters = make([]interface{}, 0)
	for _, name := range parameters {
		functionParameters = append(functionParameters, parameterValues[name])
	}
	return AsFunctionParameters(function, functionParameters, parameterValues)
}

//GetFuncSignature returns a function signature
func GetFuncSignature(function interface{}) []reflect.Type {
	AssertKind(function, reflect.Func, "function")
	functionValue := reflect.ValueOf(function)
	var result = make([]reflect.Type, 0)
	functionType := functionValue.Type()
	for i := 0; i < functionType.NumIn(); i++ {
		result = append(result, functionType.In(i))
	}
	return result
}
<commit_msg>patched zero value access error<commit_after>package toolbox

import (
	"fmt"
	"reflect"
	"strings"
)

//GetFunction returns function for provided owner and name, or error
func GetFunction(owner interface{}, name string) (interface{}, error) {
	if owner == nil {
		return nil, fmt.Errorf("failed to lookup %v on %T, owner was nil", name, owner)
	}
	var ownerType = reflect.TypeOf(owner)
	var method, has = ownerType.MethodByName(name)
	if !has {
		var available = make([]string, 0)
		for i := 0; i < ownerType.NumMethod(); i++ {
			available = append(available, ownerType.Method(i).Name)
		}
		return nil, fmt.Errorf("failed to lookup %T.%v, available:[%v]", owner, name, strings.Join(available, ","))
	}
	return reflect.ValueOf(owner).MethodByName(method.Name).Interface(), nil
}

//CallFunction calls passed in function with provided parameters,it returns a function result.
func CallFunction(function interface{}, parameters ...interface{}) []interface{} {
	AssertKind(function, reflect.Func, "function")
	var functionParameters = make([]reflect.Value, 0)
	ProcessSlice(parameters, func(item interface{}) bool {
		functionParameters = append(functionParameters, reflect.ValueOf(item))
		return true
	})

	functionValue := reflect.ValueOf(function)
	var resultValues = functionValue.Call(functionParameters)
	var result = make([]interface{}, len(resultValues))
	for i, resultValue := range resultValues {
		result[i] = resultValue.Interface()
	}
	return result
}

//AsCompatibleFunctionParameters takes incompatible function parameters and converts then into provided function signature compatible
func AsCompatibleFunctionParameters(function interface{}, parameters []interface{}) ([]interface{}, error) {
	return AsFunctionParameters(function, parameters, map[string]interface{}{})
}

//AsFunctionParameters takes incompatible function parameters and converts then into provided function signature compatible
func AsFunctionParameters(function interface{}, parameters []interface{}, parametersKV map[string]interface{}) ([]interface{}, error) {
	AssertKind(function, reflect.Func, "function")
	functionValue := reflect.ValueOf(function)
	funcSignature := GetFuncSignature(function)
	actualMethodSignatureLength := len(funcSignature)
	converter := Converter{}
	if actualMethodSignatureLength != len(parameters) {
		return nil, fmt.Errorf("invalid number of parameters wanted: [%T],  had: %v", function, len(parameters))
	}
	var functionParameters = make([]interface{}, 0)
	for i, parameterValue := range parameters {
		isStruct := IsStruct(funcSignature[i])
		if isStruct && parameterValue == nil {
			parameterValue = make(map[string]interface{})
		}
		reflectValue := reflect.ValueOf(parameterValue)
		if !isStruct {
			if parameterValue == nil {
				return nil, fmt.Errorf("parameter[%v] was empty", i)
			}
			if reflectValue.Kind() == reflect.Slice && funcSignature[i].Kind() != reflectValue.Kind() {
				return nil, fmt.Errorf("incompatible types expected: %v, but had %v", funcSignature[i].Kind(), reflectValue.Kind())
			} else if !reflectValue.IsValid() {
				if funcSignature[i].Kind() == reflect.Slice {
					parameterValue = reflect.New(funcSignature[i]).Interface()
					reflectValue = reflect.ValueOf(parameterValue)
				}
			}
		}
		if reflectValue.Type() != funcSignature[i] {
			newValuePointer := reflect.New(funcSignature[i])
			var err error
			if IsStruct(funcSignature[i]) && !(IsStruct(parameterValue) || IsMap(parameterValue)) {
				err = converter.AssignConverted(newValuePointer.Interface(), parametersKV)
			} else {
				err = converter.AssignConverted(newValuePointer.Interface(), parameterValue)
			}
			if err != nil {
				return nil, fmt.Errorf("failed to assign convert %v to %v due to %v", parametersKV, newValuePointer.Interface(), err)
			}
			reflectValue = newValuePointer.Elem()
		}
		if functionValue.Type().IsVariadic() && funcSignature[i].Kind() == reflect.Slice && i+1 == len(funcSignature) {
			ProcessSlice(reflectValue.Interface(), func(item interface{}) bool {
				functionParameters = append(functionParameters, item)
				return true
			})
		} else {
			functionParameters = append(functionParameters, reflectValue.Interface())
		}
	}
	return functionParameters, nil
}

//BuildFunctionParameters builds function parameters provided in the parameterValues.
// Parameters value will be converted if needed to expected by the function signature type. It returns function parameters , or error
func BuildFunctionParameters(function interface{}, parameters []string, parameterValues map[string]interface{}) ([]interface{}, error) {
	var functionParameters = make([]interface{}, 0)
	for _, name := range parameters {
		functionParameters = append(functionParameters, parameterValues[name])
	}
	return AsFunctionParameters(function, functionParameters, parameterValues)
}

//GetFuncSignature returns a function signature
func GetFuncSignature(function interface{}) []reflect.Type {
	AssertKind(function, reflect.Func, "function")
	functionValue := reflect.ValueOf(function)
	var result = make([]reflect.Type, 0)
	functionType := functionValue.Type()
	for i := 0; i < functionType.NumIn(); i++ {
		result = append(result, functionType.In(i))
	}
	return result
}
<|endoftext|>


<commit_before>package stack

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"path"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/distribution/reference"
	dfparser "github.com/docker/docker/builder/dockerfile/parser"
	"github.com/paralin/scratchbuild/arch"
)

// ImageStack represents an image as a stack of layers.
type ImageStack struct {
	// The final reference of the last layer.
	Reference reference.Named
	// Layers, from top to bottom.
	Layers []*ImageLayer
}

// RebaseOnArch attempts to rebase the Stack on a compatible image for the target arch.
func (s *ImageStack) RebaseOnArch(target arch.KnownArch) error {
	for i := len(s.Layers) - 1; i > 0; i-- {
		layer := s.Layers[i]
		refName := layer.Reference.Name()
		if refName == "scratch" {
			continue
		}

		compatImage, ok := arch.CompatibleBaseImage(target, refName)
		if !ok {
			continue
		}

		log.Debugf("Rebasing using %s -> %s", refName, compatImage)
		// base on this
		s.Layers = s.Layers[:i+1]
		if tagged, ok := layer.Reference.(reference.NamedTagged); ok {
			compatImage = fmt.Sprintf("%s:%s", compatImage, tagged.Tag())
		}
		img, _ := ParseImageName(compatImage)
		layer.Reference = img
		layer.Dockerfile = nil
		layer.Path = ""

		if i != 0 {
			previousLayer := s.Layers[i-1]
			previousLayer.RewriteFrom(img)
		}
	}
	return nil
}

// RewriteFrom rewrites the FROM definition in the dockerfile.
func (l *ImageLayer) RewriteFrom(img reference.Reference) {
	if l.Dockerfile == nil {
		return
	}

	ast := l.Dockerfile
	for _, line := range ast.Children {
		if line.Value == "from" {
			line.Next = &dfparser.Node{Value: img.String()}
			line.Original = fmt.Sprintf("FROM %s", line.Next.Value)

			var resultDockerfile bytes.Buffer
			lineIdx := line.StartLine - 1
			scanner := bufio.NewScanner(strings.NewReader(l.OriginalDockerfile))
			i := 0
			for scanner.Scan() {
				text := scanner.Text()
				if i == lineIdx {
					resultDockerfile.WriteString(line.Original)
				} else {
					resultDockerfile.WriteString(text)
				}
				resultDockerfile.WriteString("\n")
				i++
			}
			l.OriginalDockerfile = resultDockerfile.String()
		}
	}
}

// String represents the stack as a string.
func (s *ImageStack) String() string {
	var results bytes.Buffer
	for _, layer := range s.Layers {
		results.WriteString(layer.Reference.String())
		if layer.Dockerfile == nil && layer.Reference.Name() != "scratch" {
			results.WriteString(" [pull]")
		}
		results.WriteString(" ")
	}
	return results.String()
}

// ToDockerfile produces a series of dockerfiles representing the stack.
func (s *ImageStack) ToDockerfile() string {
	var result bytes.Buffer
	for i, image := range s.Layers {
		result.WriteString(image.ToDockerfile())
		if i != len(s.Layers)-1 {
			result.WriteString("\n---\n")
		}
	}
	return result.String()
}

// ImageLayer is a layer in a stack.
type ImageLayer struct {
	// Reference is the name/tag/etc of this image.
	Reference reference.Named
	// Dockerfile is the dockerfile source for this image.
	Dockerfile *dfparser.Node
	// Path is the path to the dockerfile directory for this image layer.
	Path string
	// OriginalDockerfile is the original source for the dockerfile.
	OriginalDockerfile string
}

// ParseDockerfile applies a Dockerfile source to a layer.
func (l *ImageLayer) ParseDockerfile(source string) error {
	res, err := dfparser.Parse(strings.NewReader(source))
	if err != nil {
		return err
	}
	l.Dockerfile = res.AST
	l.OriginalDockerfile = source
	return nil
}

// ToDockerfile produces a Dockerfile equivilent to the original source.
func (l *ImageLayer) ToDockerfile() string {
	return l.OriginalDockerfile
}

// LibraryResolver resolves dockerfile sources for library images.
type LibraryResolver interface {
	// GetLibrarySource clones the source for the Dockerfile
	GetLibrarySource(ref reference.NamedTagged) (string, error)
}

// ImageStackFromDockerfile attempts to determine the full stack of the docker image.
func ImageStackFromPath(buildPath string, dockerfilePath string, targetTag string, resolver LibraryResolver, rebaseArch arch.KnownArch) (*ImageStack, error) {
	imageRef, err := ParseImageName(targetTag)
	if err != nil {
		return nil, err
	}

	if dockerfilePath == "" {
		dockerfilePath = "Dockerfile"
	}

	dockerfilePath = path.Clean(dockerfilePath)
	if !path.IsAbs(dockerfilePath) {
		dockerfilePath = path.Join(buildPath, dockerfilePath)
	}

	sourceBin, err := ioutil.ReadFile(dockerfilePath)
	if err != nil {
		return nil, err
	}
	source := string(sourceBin)

	stack := &ImageStack{Reference: imageRef}
	baseLayer := &ImageLayer{Reference: imageRef, Path: buildPath}
	if err := baseLayer.ParseDockerfile(source); err != nil {
		return nil, err
	}
	stack.Layers = []*ImageLayer{baseLayer}
	return stack, stack.processDockerfile(resolver, rebaseArch)
}

// processDockerfile processes the next dockerfile in the stack.
func (s *ImageStack) processDockerfile(resolver LibraryResolver, rebaseArch arch.KnownArch) error {
	layer := s.Layers[len(s.Layers)-1]
	if layer.Dockerfile == nil || layer.Reference.Name() == "scratch" {
		return nil
	}

	lines := layer.Dockerfile.Children
	var nlayer *ImageLayer
	for _, line := range lines {
		if line.Value == "from" {
			if line.Next == nil || line.Next.Value == "" {
				return fmt.Errorf("Dockerfile FROM line is invalid: %s", line.Original)
			}
			ref, err := ParseImageName(line.Next.Value)
			if err != nil {
				return fmt.Errorf("Error parsing FROM line: %v", err)
			}
			nlayer = &ImageLayer{Reference: ref}
			break
		}
	}

	if nlayer == nil {
		return fmt.Errorf("Dockerfile did not have a FROM line.")
	}

	s.Layers = append(s.Layers, nlayer)

	if nlayer.Reference.Name() == "scratch" {
		return nil
	}

	le := log.WithField("image", nlayer.Reference.String())
	tagged, ok := nlayer.Reference.(reference.NamedTagged)
	if !ok {
		le.Debug("No tag given, assuming latest")
		t, err := reference.WithTag(nlayer.Reference, "latest")
		if err != nil {
			return err
		}
		tagged = t
	}

	if rebaseArch != arch.NONE {
		layerRefName := tagged.Name()
		cbi, ok := arch.CompatibleBaseImage(rebaseArch, layerRefName)
		cbi = fmt.Sprintf("%s:%s", cbi, tagged.Tag())
		if ok {
			nlayer.Dockerfile = nil
			ref, err := ParseImageName(cbi)
			if err != nil {
				return err
			}
			nlayer.Reference = ref
			nlayer.Path = ""

			if len(s.Layers) > 1 {
				previousLayer := s.Layers[len(s.Layers)-2]
				previousLayer.RewriteFrom(ref)
			}
			return nil
		}
	}

	// Attempt to find the source for this image.
	if !strings.HasPrefix(tagged.Name(), "library/") {
		le.Debug("Not a library image, cannot determine Dockerfile source.")
		return nil
	}

	srcPath, err := resolver.GetLibrarySource(tagged)
	if err != nil {
		le.WithError(err).Warn("Unable to resolve library source")
		return nil
	}
	nlayer.Path = srcPath

	dfPath := path.Join(srcPath, "Dockerfile")
	src, err := ioutil.ReadFile(dfPath)
	if err != nil {
		le.WithField("path", dfPath).WithError(err).Warn("Unable to find Dockerfile")
		return nil
	}

	if err := nlayer.ParseDockerfile(string(src)); err != nil {
		le.WithError(err).Warn("Unable to parse library dockerfile")
		return nil
	}

	return s.processDockerfile(resolver, rebaseArch)
}
<commit_msg>fix: check library/ in docker.io/library/alpine<commit_after>package stack

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"path"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/distribution/reference"
	dfparser "github.com/docker/docker/builder/dockerfile/parser"
	"github.com/paralin/scratchbuild/arch"
)

// ImageStack represents an image as a stack of layers.
type ImageStack struct {
	// The final reference of the last layer.
	Reference reference.Named
	// Layers, from top to bottom.
	Layers []*ImageLayer
}

// RebaseOnArch attempts to rebase the Stack on a compatible image for the target arch.
func (s *ImageStack) RebaseOnArch(target arch.KnownArch) error {
	for i := len(s.Layers) - 1; i > 0; i-- {
		layer := s.Layers[i]
		refName := layer.Reference.Name()
		if refName == "scratch" {
			continue
		}

		compatImage, ok := arch.CompatibleBaseImage(target, refName)
		if !ok {
			continue
		}

		log.Debugf("Rebasing using %s -> %s", refName, compatImage)
		// base on this
		s.Layers = s.Layers[:i+1]
		if tagged, ok := layer.Reference.(reference.NamedTagged); ok {
			compatImage = fmt.Sprintf("%s:%s", compatImage, tagged.Tag())
		}
		img, _ := ParseImageName(compatImage)
		layer.Reference = img
		layer.Dockerfile = nil
		layer.Path = ""

		if i != 0 {
			previousLayer := s.Layers[i-1]
			previousLayer.RewriteFrom(img)
		}
	}
	return nil
}

// RewriteFrom rewrites the FROM definition in the dockerfile.
func (l *ImageLayer) RewriteFrom(img reference.Reference) {
	if l.Dockerfile == nil {
		return
	}

	ast := l.Dockerfile
	for _, line := range ast.Children {
		if line.Value == "from" {
			line.Next = &dfparser.Node{Value: img.String()}
			line.Original = fmt.Sprintf("FROM %s", line.Next.Value)

			var resultDockerfile bytes.Buffer
			lineIdx := line.StartLine - 1
			scanner := bufio.NewScanner(strings.NewReader(l.OriginalDockerfile))
			i := 0
			for scanner.Scan() {
				text := scanner.Text()
				if i == lineIdx {
					resultDockerfile.WriteString(line.Original)
				} else {
					resultDockerfile.WriteString(text)
				}
				resultDockerfile.WriteString("\n")
				i++
			}
			l.OriginalDockerfile = resultDockerfile.String()
		}
	}
}

// String represents the stack as a string.
func (s *ImageStack) String() string {
	var results bytes.Buffer
	for _, layer := range s.Layers {
		results.WriteString(layer.Reference.String())
		if layer.Dockerfile == nil && layer.Reference.Name() != "scratch" {
			results.WriteString(" [pull]")
		}
		results.WriteString(" ")
	}
	return results.String()
}

// ToDockerfile produces a series of dockerfiles representing the stack.
func (s *ImageStack) ToDockerfile() string {
	var result bytes.Buffer
	for i, image := range s.Layers {
		result.WriteString(image.ToDockerfile())
		if i != len(s.Layers)-1 {
			result.WriteString("\n---\n")
		}
	}
	return result.String()
}

// ImageLayer is a layer in a stack.
type ImageLayer struct {
	// Reference is the name/tag/etc of this image.
	Reference reference.Named
	// Dockerfile is the dockerfile source for this image.
	Dockerfile *dfparser.Node
	// Path is the path to the dockerfile directory for this image layer.
	Path string
	// OriginalDockerfile is the original source for the dockerfile.
	OriginalDockerfile string
}

// ParseDockerfile applies a Dockerfile source to a layer.
func (l *ImageLayer) ParseDockerfile(source string) error {
	res, err := dfparser.Parse(strings.NewReader(source))
	if err != nil {
		return err
	}
	l.Dockerfile = res.AST
	l.OriginalDockerfile = source
	return nil
}

// ToDockerfile produces a Dockerfile equivilent to the original source.
func (l *ImageLayer) ToDockerfile() string {
	return l.OriginalDockerfile
}

// LibraryResolver resolves dockerfile sources for library images.
type LibraryResolver interface {
	// GetLibrarySource clones the source for the Dockerfile
	GetLibrarySource(ref reference.NamedTagged) (string, error)
}

// ImageStackFromDockerfile attempts to determine the full stack of the docker image.
func ImageStackFromPath(buildPath string, dockerfilePath string, targetTag string, resolver LibraryResolver, rebaseArch arch.KnownArch) (*ImageStack, error) {
	imageRef, err := ParseImageName(targetTag)
	if err != nil {
		return nil, err
	}

	if dockerfilePath == "" {
		dockerfilePath = "Dockerfile"
	}

	dockerfilePath = path.Clean(dockerfilePath)
	if !path.IsAbs(dockerfilePath) {
		dockerfilePath = path.Join(buildPath, dockerfilePath)
	}

	sourceBin, err := ioutil.ReadFile(dockerfilePath)
	if err != nil {
		return nil, err
	}
	source := string(sourceBin)

	stack := &ImageStack{Reference: imageRef}
	baseLayer := &ImageLayer{Reference: imageRef, Path: buildPath}
	if err := baseLayer.ParseDockerfile(source); err != nil {
		return nil, err
	}
	stack.Layers = []*ImageLayer{baseLayer}
	return stack, stack.processDockerfile(resolver, rebaseArch)
}

// processDockerfile processes the next dockerfile in the stack.
func (s *ImageStack) processDockerfile(resolver LibraryResolver, rebaseArch arch.KnownArch) error {
	layer := s.Layers[len(s.Layers)-1]
	if layer.Dockerfile == nil || layer.Reference.Name() == "scratch" {
		return nil
	}

	lines := layer.Dockerfile.Children
	var nlayer *ImageLayer
	for _, line := range lines {
		if line.Value == "from" {
			if line.Next == nil || line.Next.Value == "" {
				return fmt.Errorf("Dockerfile FROM line is invalid: %s", line.Original)
			}
			ref, err := ParseImageName(line.Next.Value)
			if err != nil {
				return fmt.Errorf("Error parsing FROM line: %v", err)
			}
			nlayer = &ImageLayer{Reference: ref}
			break
		}
	}

	if nlayer == nil {
		return fmt.Errorf("Dockerfile did not have a FROM line.")
	}

	s.Layers = append(s.Layers, nlayer)

	if nlayer.Reference.Name() == "scratch" {
		return nil
	}

	le := log.WithField("image", nlayer.Reference.String())
	tagged, ok := nlayer.Reference.(reference.NamedTagged)
	if !ok {
		le.Debug("No tag given, assuming latest")
		t, err := reference.WithTag(nlayer.Reference, "latest")
		if err != nil {
			return err
		}
		tagged = t
	}

	if rebaseArch != arch.NONE {
		layerRefName := tagged.Name()
		cbi, ok := arch.CompatibleBaseImage(rebaseArch, layerRefName)
		cbi = fmt.Sprintf("%s:%s", cbi, tagged.Tag())
		if ok {
			nlayer.Dockerfile = nil
			ref, err := ParseImageName(cbi)
			if err != nil {
				return err
			}
			nlayer.Reference = ref
			nlayer.Path = ""

			if len(s.Layers) > 1 {
				previousLayer := s.Layers[len(s.Layers)-2]
				previousLayer.RewriteFrom(ref)
			}
			return nil
		}
	}

	// Attempt to find the source for this image.
	if !strings.Contains(tagged.Name(), "library/") {
		le.Debug("Not a library image, cannot determine Dockerfile source.")
		return nil
	}

	srcPath, err := resolver.GetLibrarySource(tagged)
	if err != nil {
		le.WithError(err).Warn("Unable to resolve library source")
		return nil
	}
	nlayer.Path = srcPath

	dfPath := path.Join(srcPath, "Dockerfile")
	src, err := ioutil.ReadFile(dfPath)
	if err != nil {
		le.WithField("path", dfPath).WithError(err).Warn("Unable to find Dockerfile")
		return nil
	}

	if err := nlayer.ParseDockerfile(string(src)); err != nil {
		le.WithError(err).Warn("Unable to parse library dockerfile")
		return nil
	}

	return s.processDockerfile(resolver, rebaseArch)
}
<|endoftext|>


<commit_before>//   Copyright 2018 MSolution.IO
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package ebs

import (
	"context"
	"crypto/md5"
	"encoding/base64"
	"encoding/json"
	"sync"
	"time"

	"github.com/trackit/jsonlog"

	taws "github.com/trackit/trackit-server/aws"
	"github.com/trackit/trackit-server/aws/usageReports"
	"github.com/trackit/trackit-server/es"
)

const MonitorSnapshotStsSessionName = "monitor-snapshot"

type (
	// SnapshotReport is saved in ES to have all the information of an EBS snapshot
	SnapshotReport struct {
		utils.ReportBase
		Snapshot Snapshot `json:"snapshot"`
	}

	// SnapshotBase contains basics information of an EBS snapshot
	SnapshotBase struct {
		Id          string    `json:"id"`
		Description string    `json:"description"`
		State       string    `json:"state"`
		Encrypted   bool      `json:"encrypted"`
		StartTime   time.Time `json:"startTime"`
	}

	// Snapshot contains all the information of an EBS snapshot
	Snapshot struct {
		SnapshotBase
		Tags []utils.Tag `json:"tags"`
		Volume Volume `json:"volume"`
		Costs map[string]float64 `json:"costs"`
	}

	// Volume contains information about an EBS volume
	Volume struct {
		Id    string  `json:"id"`
		Size int64 `json:"size"`
	}
)

// importSnapshotsToEs imports EBS snapshots in ElasticSearch.
// It calls createIndexEs if the index doesn't exist.
func importSnapshotsToEs(ctx context.Context, aa taws.AwsAccount, snapshots []SnapshotReport) error {
	logger := jsonlog.LoggerFromContextOrDefault(ctx)
	logger.Info("Updating EBS snapshots for AWS account.", map[string]interface{}{
		"awsAccount": aa,
	})
	index := es.IndexNameForUserId(aa.UserId, IndexPrefixEBSReport)
	bp, err := utils.GetBulkProcessor(ctx)
	if err != nil {
		logger.Error("Failed to get bulk processor.", err.Error())
		return err
	}
	for _, snapshot := range snapshots {
		id, err := generateId(snapshot)
		if err != nil {
			logger.Error("Error when marshaling snapshot var", err.Error())
			return err
		}
		bp = utils.AddDocToBulkProcessor(bp, snapshot, TypeEBSReport, index, id)
	}
	bp.Flush()
	err = bp.Close()
	if err != nil {
		logger.Error("Fail to put EBS snapshots in ES", err.Error())
		return err
	}
	logger.Info("EBS snapshots put in ES", nil)
	return nil
}

func generateId(snapshot SnapshotReport) (string, error) {
	ji, err := json.Marshal(struct {
		Account    string    `json:"account"`
		ReportDate time.Time `json:"reportDate"`
		Id         string    `json:"id"`
	}{
		snapshot.Account,
		snapshot.ReportDate,
		snapshot.Snapshot.Id,
	})
	if err != nil {
		return "", err
	}
	hash := md5.Sum(ji)
	hash64 := base64.URLEncoding.EncodeToString(hash[:])
	return hash64, nil
}

// merge function from https://blog.golang.org/pipelines#TOC_4
// It allows to merge many chans to one.
func merge(cs ...<-chan Snapshot) <-chan Snapshot {
	var wg sync.WaitGroup
	out := make(chan Snapshot)

	// Start an output goroutine for each input channel in cs. The output
	// copies values from c to out until c is closed, then calls wg.Done.
	output := func(c <-chan Snapshot) {
		for n := range c {
			out <- n
		}
		wg.Done()
	}
	wg.Add(len(cs))
	for _, c := range cs {
		go output(c)
	}

	// Start a goroutine to close out once all the output goroutines are
	// done. This must start after the wg.Add call.
	go func() {
		wg.Wait()
		close(out)
	}()
	return out
}
<commit_msg>alignment of type in struct<commit_after>//   Copyright 2018 MSolution.IO
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package ebs

import (
	"context"
	"crypto/md5"
	"encoding/base64"
	"encoding/json"
	"sync"
	"time"

	"github.com/trackit/jsonlog"

	taws "github.com/trackit/trackit-server/aws"
	"github.com/trackit/trackit-server/aws/usageReports"
	"github.com/trackit/trackit-server/es"
)

const MonitorSnapshotStsSessionName = "monitor-snapshot"

type (
	// SnapshotReport is saved in ES to have all the information of an EBS snapshot
	SnapshotReport struct {
		utils.ReportBase
		Snapshot Snapshot `json:"snapshot"`
	}

	// SnapshotBase contains basics information of an EBS snapshot
	SnapshotBase struct {
		Id          string    `json:"id"`
		Description string    `json:"description"`
		State       string    `json:"state"`
		Encrypted   bool      `json:"encrypted"`
		StartTime   time.Time `json:"startTime"`
	}

	// Snapshot contains all the information of an EBS snapshot
	Snapshot struct {
		SnapshotBase
		Tags   []utils.Tag        `json:"tags"`
		Volume Volume             `json:"volume"`
		Costs  map[string]float64 `json:"costs"`
	}

	// Volume contains information about an EBS volume
	Volume struct {
		Id    string  `json:"id"`
		Size  int64   `json:"size"`
	}
)

// importSnapshotsToEs imports EBS snapshots in ElasticSearch.
// It calls createIndexEs if the index doesn't exist.
func importSnapshotsToEs(ctx context.Context, aa taws.AwsAccount, snapshots []SnapshotReport) error {
	logger := jsonlog.LoggerFromContextOrDefault(ctx)
	logger.Info("Updating EBS snapshots for AWS account.", map[string]interface{}{
		"awsAccount": aa,
	})
	index := es.IndexNameForUserId(aa.UserId, IndexPrefixEBSReport)
	bp, err := utils.GetBulkProcessor(ctx)
	if err != nil {
		logger.Error("Failed to get bulk processor.", err.Error())
		return err
	}
	for _, snapshot := range snapshots {
		id, err := generateId(snapshot)
		if err != nil {
			logger.Error("Error when marshaling snapshot var", err.Error())
			return err
		}
		bp = utils.AddDocToBulkProcessor(bp, snapshot, TypeEBSReport, index, id)
	}
	bp.Flush()
	err = bp.Close()
	if err != nil {
		logger.Error("Fail to put EBS snapshots in ES", err.Error())
		return err
	}
	logger.Info("EBS snapshots put in ES", nil)
	return nil
}

func generateId(snapshot SnapshotReport) (string, error) {
	ji, err := json.Marshal(struct {
		Account    string    `json:"account"`
		ReportDate time.Time `json:"reportDate"`
		Id         string    `json:"id"`
	}{
		snapshot.Account,
		snapshot.ReportDate,
		snapshot.Snapshot.Id,
	})
	if err != nil {
		return "", err
	}
	hash := md5.Sum(ji)
	hash64 := base64.URLEncoding.EncodeToString(hash[:])
	return hash64, nil
}

// merge function from https://blog.golang.org/pipelines#TOC_4
// It allows to merge many chans to one.
func merge(cs ...<-chan Snapshot) <-chan Snapshot {
	var wg sync.WaitGroup
	out := make(chan Snapshot)

	// Start an output goroutine for each input channel in cs. The output
	// copies values from c to out until c is closed, then calls wg.Done.
	output := func(c <-chan Snapshot) {
		for n := range c {
			out <- n
		}
		wg.Done()
	}
	wg.Add(len(cs))
	for _, c := range cs {
		go output(c)
	}

	// Start a goroutine to close out once all the output goroutines are
	// done. This must start after the wg.Add call.
	go func() {
		wg.Wait()
		close(out)
	}()
	return out
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	// Uncomment the following line to load the gcp plugin (only required to authenticate against GKE clusters).
	// _ "k8s.io/client-go/plugin/pkg/client/auth/gcp"

	"knative.dev/pkg/injection/sharedmain"

	awssqssource "knative.dev/eventing-contrib/awssqs/pkg/client/injection/reconciler/sources/v1alpha1/awssqssource/stub"
)

func main() {
	sharedmain.Main("awssqs-controller",
		awssqssource.NewController,
	)
}
<commit_msg>Fixing issue with awssqs controller pointing to stub class (#1013)<commit_after>/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	// Uncomment the following line to load the gcp plugin (only required to authenticate against GKE clusters).
	// _ "k8s.io/client-go/plugin/pkg/client/auth/gcp"

	"knative.dev/pkg/injection/sharedmain"

	awssqssource "knative.dev/eventing-contrib/awssqs/pkg/reconciler"
)

func main() {
	sharedmain.Main("awssqs-controller",
		awssqssource.NewController,
	)
}
<|endoftext|>


<commit_before>package weddingseats

import (
	"net/http"
	"sync"

	"code.google.com/p/goauth2/oauth"
	"github.com/gorilla/sessions"
)

// global vars
var (
	config       *Configuration
	configLock   = new(sync.RWMutex) // so we can hot-reload config
	FACEBOOK_CFG = new(oauth.Config)
	SessionStore *sessions.CookieStore
)

func check(e error) {
	if e != nil {
		panic(e)
	}
}

func init() {
	// locate and read our configuration
	err := ReadConfig("conf.json")
	check(err)

	FACEBOOK_CFG.ClientId = config.Facebook.ClientId
	FACEBOOK_CFG.ClientSecret = config.Facebook.ClientSecret
	FACEBOOK_CFG.AuthURL = config.Facebook.AuthURL
	FACEBOOK_CFG.TokenURL = config.Facebook.TokenURL
	FACEBOOK_CFG.RedirectURL = config.Facebook.RedirectURL
	FACEBOOK_CFG.Scope = config.Facebook.Scope

	// setup session storage
	SessionStore = sessions.NewCookieStore([]byte(config.CookieSecret))
	SessionStore.Options = &sessions.Options{
		Secure: false,
	}

	// setup URL handlers
	http.HandleFunc("/", HandleIndex)
	http.HandleFunc("/tz", HandleGender)
	http.HandleFunc("/facebook_start", HandleFacebookStart)
	http.HandleFunc("/facebook_authorized", HandleFacebookAuthorized)
}
<commit_msg>read config from .prod (for deploys to GAE)<commit_after>package weddingseats

import (
	"net/http"
	"sync"

	"code.google.com/p/goauth2/oauth"
	"github.com/gorilla/sessions"
)

// global vars
var (
	config       *Configuration
	configLock   = new(sync.RWMutex) // so we can hot-reload config
	FACEBOOK_CFG = new(oauth.Config)
	SessionStore *sessions.CookieStore
)

func check(e error) {
	if e != nil {
		panic(e)
	}
}

func init() {
	// locate and read our configuration
	err := ReadConfig("conf.json.prod")
	check(err)

	FACEBOOK_CFG.ClientId = config.Facebook.ClientId
	FACEBOOK_CFG.ClientSecret = config.Facebook.ClientSecret
	FACEBOOK_CFG.AuthURL = config.Facebook.AuthURL
	FACEBOOK_CFG.TokenURL = config.Facebook.TokenURL
	FACEBOOK_CFG.RedirectURL = config.Facebook.RedirectURL
	FACEBOOK_CFG.Scope = config.Facebook.Scope

	// setup session storage
	SessionStore = sessions.NewCookieStore([]byte(config.CookieSecret))
	SessionStore.Options = &sessions.Options{
		Secure: false,
	}

	// setup URL handlers
	http.HandleFunc("/", HandleIndex)
	http.HandleFunc("/tz", HandleGender)
	http.HandleFunc("/facebook_start", HandleFacebookStart)
	http.HandleFunc("/facebook_authorized", HandleFacebookAuthorized)
}
<|endoftext|>


<commit_before>// Copyright 2015 PLUMgrid
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gbp

import (
	"bytes"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"

	"github.com/jmoiron/sqlx"
	_ "github.com/mattn/go-sqlite3"
)

var filterImplC string = `
struct icmp_t {
	unsigned char type;
	unsigned char code;
	unsigned short checksum;
} BPF_PACKET_HEADER;

BPF_TABLE("hash", u32, u32, endpoints, 1024);

struct match {
	u32 src_tag;
	u32 dst_tag;
	u16 sport;
	u16 dport;
	u8 proto;
	u8 direction;
	u8 pad[2];
};
BPF_TABLE("hash", struct match, int, rules, 1024);

static int handle_tx(void *skb, struct metadata *md) {
	u8 *cursor = 0;
	struct match m = {};
	int ret = RX_DROP;
	u32 dst_tag = 0, src_tag = 0;
	if (md->data[0].type == 1)
		src_tag = md->data[0].value;

	ethernet: {
		struct ethernet_t *ethernet = cursor_advance(cursor, sizeof(*ethernet));
		u16 ethertype = ethernet->type;
		switch (ethertype) {
			case ETH_P_IP: goto ip;
			case ETH_P_IPV6: goto ip6;
			case ETH_P_ARP: goto arp;
			default: goto DONE;
		}
	}

	ip: {
		struct ip_t *ip = cursor_advance(cursor, sizeof(*ip));
		u32 dst_ip = ip->dst;
		m.proto = ip->nextp;

		u32 *tag = endpoints.lookup(&dst_ip);
		if (!tag)
			goto DONE;
		dst_tag = *tag;
		switch (m.proto) {
			case 1: goto icmp;
			case 6: goto tcp;
			case 17: goto udp;
			default: goto DONE;
		}
	}

	ip6: {
		goto DONE;
	}

	icmp: {
		struct icmp_t *icmp = cursor_advance(cursor, sizeof(*icmp));
		goto EOP;
	}

	tcp: {
		struct tcp_t *tcp = cursor_advance(cursor, sizeof(*tcp));
		m.dport = tcp->dst_port;
		m.sport = tcp->src_port;
		goto EOP;
	}

	udp: {
		struct udp_t *udp = cursor_advance(cursor, sizeof(*udp));
		m.dport = udp->dport;
		m.sport = udp->sport;
		goto EOP;
	}

	arp: {
		return RX_OK;
	}

EOP: ;
	int *result;
	struct match m1 = {src_tag, dst_tag, m.sport, m.dport, m.proto, 0};
	result = rules.lookup(&m1);
	if (result) {
		ret = *result;
		goto DONE;
	}
	struct match m2 = {src_tag, dst_tag, m.sport, 0, m.proto, 1/*2*/};
	result = rules.lookup(&m2);
	if (result) {
		ret = *result;
		goto DONE;
	}
	struct match m3 = {src_tag, dst_tag, 0, m.dport, m.proto, 1};
	result = rules.lookup(&m3);
	if (result) {
		ret = *result;
		goto DONE;
	}

DONE:
	return ret;
}

static int handle_rx(void *skb, struct metadata *md) {
	u8 *cursor = 0;
	u32 src_tag = 0;
	int ret = RX_OK;

	ethernet: {
		struct ethernet_t *ethernet = cursor_advance(cursor, sizeof(*ethernet));
		u16 ethertype = ethernet->type;
		switch (ethertype) {
			case ETH_P_IP: goto ip;
			case ETH_P_IPV6: goto ip6;
			case ETH_P_ARP: goto arp;
			default: goto DONE;
		}
	}

	ip: {
		struct ip_t *ip = cursor_advance(cursor, sizeof(*ip));
		u32 src_ip = ip->src;

		u32 *tag = endpoints.lookup(&src_ip);
		if (!tag)
			goto DONE;
		src_tag = *tag;
		goto DONE;
	}

	ip6: {
		goto DONE;
	}

	arp: {
		goto DONE;
	}

DONE:
	if (src_tag != 0) {
		md->data[0].type = 1;
		md->data[0].value = src_tag;
	}

	return ret;
}
`

type moduleEntry struct {
	Id          string                 `json:"id"`
	ModuleType  string                 `json:"module_type"`
	DisplayName string                 `json:"display_name"`
	Perm        string                 `json:"permissions"`
	Config      map[string]interface{} `json:"config"`
}

type tableEntry struct {
	Key   string `json:"key"`
	Value string `json:"value"`
}

type Dataplane struct {
	client  *http.Client
	baseUrl string
	id      string
	db      *sqlx.DB
}

func NewDataplane(sqlUrl string) *Dataplane {
	client := &http.Client{}
	d := &Dataplane{
		client: client,
		db:     sqlx.MustConnect("sqlite3", sqlUrl),
	}
	d.db.Exec(`
CREATE TABLE endpoints (
	ip CHAR(50)     PRIMARY KEY NOT NULL,
	tenant CHAR(40) NOT NULL,
	epg CHAR(40)    NOT NULL
);
`)
	return d
}

func (d *Dataplane) postObject(url string, requestObj interface{}, responseObj interface{}) (err error) {
	b, err := json.Marshal(requestObj)
	if err != nil {
		return
	}
	resp, err := d.client.Post(d.baseUrl+url, "application/json", bytes.NewReader(b))
	if err != nil {
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		var body []byte
		if body, err = ioutil.ReadAll(resp.Body); err != nil {
			Error.Print(string(body))
		}
		return fmt.Errorf("module server returned %s", resp.Status)
	}
	if responseObj != nil {
		err = json.NewDecoder(resp.Body).Decode(responseObj)
	}
	return
}

func (d *Dataplane) Init(baseUrl string) error {
	d.baseUrl = baseUrl
	req := map[string]interface{}{
		"module_type":  "bpf",
		"display_name": "gbp",
		"config": map[string]interface{}{
			"code":     filterImplC,
			"handlers": []string{"handle_rx", "handle_tx"},
		},
	}
	var module moduleEntry
	err := d.postObject("/modules/", req, &module)
	if err != nil {
		return err
	}
	d.id = module.Id
	return nil
}

func (d *Dataplane) Id() string {
	return d.id
}

func (d *Dataplane) Close() error {
	return nil
}

func epgToId(epgName string) int {
	// TODO: store in the DB a name->id mapping
	switch epgName {
	case "clients":
		return 1
	case "webservers":
		return 2
	}
	return 0
}

func (d *Dataplane) ParsePolicy(policy *Policy) (err error) {
	Debug.Println("ParsePolicy")
	consumerId := epgToId(policy.ConsumerEpgId)
	providerId := epgToId(policy.ProviderEpgId)
	for _, ruleGroupConstrained := range policy.PolicyRuleGroups {
		for _, ruleGroup := range ruleGroupConstrained.PolicyRuleGroups {
			for _, rule := range ruleGroup.ResolvedRules {
				for _, classifier := range rule.Classifiers {
					m := classifier.ToMatch()
					id1, id2 := consumerId, providerId
					if m.Direction == 2 {
						id1, id2 = providerId, consumerId
					}
					k := fmt.Sprintf("{ %d %d %d %d %d %d [0 0]}",
						id1, id2, m.SourcePort, m.DestPort, m.Proto, 1 /*m.Direction*/)
					v := "2" // drop
					if rule.IsAllow() {
						v = "0"
					}
					Debug.Printf("rule %s %s\n", k, v)
					obj := &tableEntry{Key: k, Value: v}
					err = d.postObject("/modules/"+d.id+"/tables/rules/entries/", obj, nil)
					if err != nil {
						return
					}
				}
			}
		}
	}
	return
}

type EndpointEntry struct {
	Ip     string `db:"ip" json:"ip"`
	Tenant string `db:"tenant" json:"tenant"`
	Epg    string `db:"epg" json:"epg"`
}

func (d *Dataplane) Endpoints() <-chan *EndpointEntry {
	ch := make(chan *EndpointEntry)
	rows, err := d.db.Queryx(`SELECT * FROM endpoints`)
	if err != nil {
		return nil
	}
	go func() {
		defer close(ch)
		for rows.Next() {
			var e EndpointEntry
			err = rows.StructScan(&e)
			if err != nil {
				return
			}
			ch <- &e
		}
	}()
	return ch
}

func ipStrToKey(ipStr string) (string, error) {
	ip := net.ParseIP(ipStr)
	if ip == nil {
		return "", fmt.Errorf("ipStrToKey: ipStr is not a valid IP address")
	}
	if ip.To4() != nil {
		return fmt.Sprintf("%d", binary.BigEndian.Uint32(ip.To4())), nil
	}
	return "", fmt.Errorf("ipStrToKey: IPv6 support not implemented")
}
func (d *Dataplane) AddEndpoint(ipStr, tenant, epg string) (err error) {
	ipKey, err := ipStrToKey(ipStr)
	if err != nil {
		return
	}

	tx, err := d.db.Begin()
	if err != nil {
		return
	}
	defer func() {
		if err != nil {
			Info.Printf("AddEndpoint: rolling back %s\n", ipStr)
			tx.Rollback()
		} else {
			err = tx.Commit()
		}
	}()
	_, err = tx.Exec(`INSERT INTO endpoints (ip, tenant, epg) VALUES (?, ?, ?)`, ipStr, tenant, epg)
	if err != nil {
		return
	}
	obj := &tableEntry{
		Key:   ipKey,
		Value: fmt.Sprintf("%d", epgToId(epg)),
	}
	err = d.postObject("/modules/"+d.id+"/tables/endpoints/entries/", obj, nil)
	if err != nil {
		return
	}
	return
}

func (d *Dataplane) DeleteEndpoint(ip string) (err error) {
	tx, err := d.db.Begin()
	if err != nil {
		return
	}
	defer func() {
		if err != nil {
			tx.Rollback()
		} else {
			err = tx.Commit()
		}
	}()
	_, err = tx.Exec(`DELETE FROM endpoints WHERE ip=?`, ip)
	if err != nil {
		return
	}
	//err = d.deleteObject("/modules/"+d.id+"/tables/endpoints/entries/"+ip, nil, nil)
	return
}
<commit_msg>Fixup duplicate icmp_t header in gbp<commit_after>// Copyright 2015 PLUMgrid
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gbp

import (
	"bytes"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"

	"github.com/jmoiron/sqlx"
	_ "github.com/mattn/go-sqlite3"
)

var filterImplC string = `
BPF_TABLE("hash", u32, u32, endpoints, 1024);

struct match {
	u32 src_tag;
	u32 dst_tag;
	u16 sport;
	u16 dport;
	u8 proto;
	u8 direction;
	u8 pad[2];
};
BPF_TABLE("hash", struct match, int, rules, 1024);

static int handle_tx(void *skb, struct metadata *md) {
	u8 *cursor = 0;
	struct match m = {};
	int ret = RX_DROP;
	u32 dst_tag = 0, src_tag = 0;
	if (md->data[0].type == 1)
		src_tag = md->data[0].value;

	ethernet: {
		struct ethernet_t *ethernet = cursor_advance(cursor, sizeof(*ethernet));
		u16 ethertype = ethernet->type;
		switch (ethertype) {
			case ETH_P_IP: goto ip;
			case ETH_P_IPV6: goto ip6;
			case ETH_P_ARP: goto arp;
			default: goto DONE;
		}
	}

	ip: {
		struct ip_t *ip = cursor_advance(cursor, sizeof(*ip));
		u32 dst_ip = ip->dst;
		m.proto = ip->nextp;

		u32 *tag = endpoints.lookup(&dst_ip);
		if (!tag)
			goto DONE;
		dst_tag = *tag;
		switch (m.proto) {
			case 1: goto icmp;
			case 6: goto tcp;
			case 17: goto udp;
			default: goto DONE;
		}
	}

	ip6: {
		goto DONE;
	}

	icmp: {
		struct icmp_t *icmp = cursor_advance(cursor, sizeof(*icmp));
		goto EOP;
	}

	tcp: {
		struct tcp_t *tcp = cursor_advance(cursor, sizeof(*tcp));
		m.dport = tcp->dst_port;
		m.sport = tcp->src_port;
		goto EOP;
	}

	udp: {
		struct udp_t *udp = cursor_advance(cursor, sizeof(*udp));
		m.dport = udp->dport;
		m.sport = udp->sport;
		goto EOP;
	}

	arp: {
		return RX_OK;
	}

EOP: ;
	int *result;
	struct match m1 = {src_tag, dst_tag, m.sport, m.dport, m.proto, 0};
	result = rules.lookup(&m1);
	if (result) {
		ret = *result;
		goto DONE;
	}
	struct match m2 = {src_tag, dst_tag, m.sport, 0, m.proto, 1/*2*/};
	result = rules.lookup(&m2);
	if (result) {
		ret = *result;
		goto DONE;
	}
	struct match m3 = {src_tag, dst_tag, 0, m.dport, m.proto, 1};
	result = rules.lookup(&m3);
	if (result) {
		ret = *result;
		goto DONE;
	}

DONE:
	return ret;
}

static int handle_rx(void *skb, struct metadata *md) {
	u8 *cursor = 0;
	u32 src_tag = 0;
	int ret = RX_OK;

	ethernet: {
		struct ethernet_t *ethernet = cursor_advance(cursor, sizeof(*ethernet));
		u16 ethertype = ethernet->type;
		switch (ethertype) {
			case ETH_P_IP: goto ip;
			case ETH_P_IPV6: goto ip6;
			case ETH_P_ARP: goto arp;
			default: goto DONE;
		}
	}

	ip: {
		struct ip_t *ip = cursor_advance(cursor, sizeof(*ip));
		u32 src_ip = ip->src;

		u32 *tag = endpoints.lookup(&src_ip);
		if (!tag)
			goto DONE;
		src_tag = *tag;
		goto DONE;
	}

	ip6: {
		goto DONE;
	}

	arp: {
		goto DONE;
	}

DONE:
	if (src_tag != 0) {
		md->data[0].type = 1;
		md->data[0].value = src_tag;
	}

	return ret;
}
`

type moduleEntry struct {
	Id          string                 `json:"id"`
	ModuleType  string                 `json:"module_type"`
	DisplayName string                 `json:"display_name"`
	Perm        string                 `json:"permissions"`
	Config      map[string]interface{} `json:"config"`
}

type tableEntry struct {
	Key   string `json:"key"`
	Value string `json:"value"`
}

type Dataplane struct {
	client  *http.Client
	baseUrl string
	id      string
	db      *sqlx.DB
}

func NewDataplane(sqlUrl string) *Dataplane {
	client := &http.Client{}
	d := &Dataplane{
		client: client,
		db:     sqlx.MustConnect("sqlite3", sqlUrl),
	}
	d.db.Exec(`
CREATE TABLE endpoints (
	ip CHAR(50)     PRIMARY KEY NOT NULL,
	tenant CHAR(40) NOT NULL,
	epg CHAR(40)    NOT NULL
);
`)
	return d
}

func (d *Dataplane) postObject(url string, requestObj interface{}, responseObj interface{}) (err error) {
	b, err := json.Marshal(requestObj)
	if err != nil {
		return
	}
	resp, err := d.client.Post(d.baseUrl+url, "application/json", bytes.NewReader(b))
	if err != nil {
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		var body []byte
		if body, err = ioutil.ReadAll(resp.Body); err != nil {
			Error.Print(string(body))
		}
		return fmt.Errorf("module server returned %s", resp.Status)
	}
	if responseObj != nil {
		err = json.NewDecoder(resp.Body).Decode(responseObj)
	}
	return
}

func (d *Dataplane) Init(baseUrl string) error {
	d.baseUrl = baseUrl
	req := map[string]interface{}{
		"module_type":  "bpf",
		"display_name": "gbp",
		"config": map[string]interface{}{
			"code":     filterImplC,
			"handlers": []string{"handle_rx", "handle_tx"},
		},
	}
	var module moduleEntry
	err := d.postObject("/modules/", req, &module)
	if err != nil {
		return err
	}
	d.id = module.Id
	return nil
}

func (d *Dataplane) Id() string {
	return d.id
}

func (d *Dataplane) Close() error {
	return nil
}

func epgToId(epgName string) int {
	// TODO: store in the DB a name->id mapping
	switch epgName {
	case "clients":
		return 1
	case "webservers":
		return 2
	}
	return 0
}

func (d *Dataplane) ParsePolicy(policy *Policy) (err error) {
	Debug.Println("ParsePolicy")
	consumerId := epgToId(policy.ConsumerEpgId)
	providerId := epgToId(policy.ProviderEpgId)
	for _, ruleGroupConstrained := range policy.PolicyRuleGroups {
		for _, ruleGroup := range ruleGroupConstrained.PolicyRuleGroups {
			for _, rule := range ruleGroup.ResolvedRules {
				for _, classifier := range rule.Classifiers {
					m := classifier.ToMatch()
					id1, id2 := consumerId, providerId
					if m.Direction == 2 {
						id1, id2 = providerId, consumerId
					}
					k := fmt.Sprintf("{ %d %d %d %d %d %d [0 0]}",
						id1, id2, m.SourcePort, m.DestPort, m.Proto, 1 /*m.Direction*/)
					v := "2" // drop
					if rule.IsAllow() {
						v = "0"
					}
					Debug.Printf("rule %s %s\n", k, v)
					obj := &tableEntry{Key: k, Value: v}
					err = d.postObject("/modules/"+d.id+"/tables/rules/entries/", obj, nil)
					if err != nil {
						return
					}
				}
			}
		}
	}
	return
}

type EndpointEntry struct {
	Ip     string `db:"ip" json:"ip"`
	Tenant string `db:"tenant" json:"tenant"`
	Epg    string `db:"epg" json:"epg"`
}

func (d *Dataplane) Endpoints() <-chan *EndpointEntry {
	ch := make(chan *EndpointEntry)
	rows, err := d.db.Queryx(`SELECT * FROM endpoints`)
	if err != nil {
		return nil
	}
	go func() {
		defer close(ch)
		for rows.Next() {
			var e EndpointEntry
			err = rows.StructScan(&e)
			if err != nil {
				return
			}
			ch <- &e
		}
	}()
	return ch
}

func ipStrToKey(ipStr string) (string, error) {
	ip := net.ParseIP(ipStr)
	if ip == nil {
		return "", fmt.Errorf("ipStrToKey: ipStr is not a valid IP address")
	}
	if ip.To4() != nil {
		return fmt.Sprintf("%d", binary.BigEndian.Uint32(ip.To4())), nil
	}
	return "", fmt.Errorf("ipStrToKey: IPv6 support not implemented")
}
func (d *Dataplane) AddEndpoint(ipStr, tenant, epg string) (err error) {
	ipKey, err := ipStrToKey(ipStr)
	if err != nil {
		return
	}

	tx, err := d.db.Begin()
	if err != nil {
		return
	}
	defer func() {
		if err != nil {
			Info.Printf("AddEndpoint: rolling back %s\n", ipStr)
			tx.Rollback()
		} else {
			err = tx.Commit()
		}
	}()
	_, err = tx.Exec(`INSERT INTO endpoints (ip, tenant, epg) VALUES (?, ?, ?)`, ipStr, tenant, epg)
	if err != nil {
		return
	}
	obj := &tableEntry{
		Key:   ipKey,
		Value: fmt.Sprintf("%d", epgToId(epg)),
	}
	err = d.postObject("/modules/"+d.id+"/tables/endpoints/entries/", obj, nil)
	if err != nil {
		return
	}
	return
}

func (d *Dataplane) DeleteEndpoint(ip string) (err error) {
	tx, err := d.db.Begin()
	if err != nil {
		return
	}
	defer func() {
		if err != nil {
			tx.Rollback()
		} else {
			err = tx.Commit()
		}
	}()
	_, err = tx.Exec(`DELETE FROM endpoints WHERE ip=?`, ip)
	if err != nil {
		return
	}
	//err = d.deleteObject("/modules/"+d.id+"/tables/endpoints/entries/"+ip, nil, nil)
	return
}
<|endoftext|>