<commit_before>/*
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package executor

import (
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/GoogleContainerTools/kaniko/pkg/cache"
	"github.com/GoogleContainerTools/kaniko/pkg/config"
	"github.com/GoogleContainerTools/kaniko/pkg/constants"
	"github.com/GoogleContainerTools/kaniko/pkg/creds"
	"github.com/GoogleContainerTools/kaniko/pkg/timing"
	"github.com/GoogleContainerTools/kaniko/pkg/version"
	"github.com/google/go-containerregistry/pkg/name"
	"github.com/google/go-containerregistry/pkg/v1"
	"github.com/google/go-containerregistry/pkg/v1/empty"
	"github.com/google/go-containerregistry/pkg/v1/layout"
	"github.com/google/go-containerregistry/pkg/v1/mutate"
	"github.com/google/go-containerregistry/pkg/v1/remote"
	"github.com/google/go-containerregistry/pkg/v1/tarball"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

type withUserAgent struct {
	t http.RoundTripper
}

const (
	UpstreamClientUaKey = "UPSTREAM_CLIENT_TYPE"
)

func (w *withUserAgent) RoundTrip(r *http.Request) (*http.Response, error) {
	ua := []string{fmt.Sprintf("kaniko/%s", version.Version())}
	if upstream := os.Getenv(UpstreamClientUaKey); upstream != "" {
		ua = append(ua, upstream)
	}
	r.Header.Set("User-Agent", strings.Join(ua, ","))
	return w.t.RoundTrip(r)
}

// CheckPushPermissionos checks that the configured credentials can be used to
// push to every specified destination.
func CheckPushPermissions(opts *config.KanikoOptions) error {
	if opts.NoPush {
		return nil
	}

	checked := map[string]bool{}
	for _, destination := range opts.Destinations {
		destRef, err := name.NewTag(destination, name.WeakValidation)
		if err != nil {
			return errors.Wrap(err, "getting tag for destination")
		}
		if checked[destRef.Context().RepositoryStr()] {
			continue
		}

		registryName := destRef.Repository.Registry.Name()
		tr := makeTransport(opts, registryName)
		if err := remote.CheckPushPermission(destRef, creds.GetKeychain(), tr); err != nil {
			return errors.Wrapf(err, "checking push permission for %q", destRef)
		}
		checked[destRef.Context().RepositoryStr()] = true
	}
	return nil
}

// DoPush is responsible for pushing image to the destinations specified in opts
func DoPush(image v1.Image, opts *config.KanikoOptions) error {
	t := timing.Start("Total Push Time")

	if opts.DigestFile != "" {
		digest, err := image.Digest()
		if err != nil {
			return errors.Wrap(err, "error fetching digest")
		}
		digestByteArray := []byte(digest.String())
		err = ioutil.WriteFile(opts.DigestFile, digestByteArray, 0644)
		if err != nil {
			return errors.Wrap(err, "writing digest to file failed")
		}
	}

	if opts.OCILayoutPath != "" {
		path, err := layout.Write(opts.OCILayoutPath, empty.Index)
		if err != nil {
			return errors.Wrap(err, "writing empty layout")
		}
		if err := path.AppendImage(image); err != nil {
			return errors.Wrap(err, "appending image")
		}
	}

	destRefs := []name.Tag{}
	for _, destination := range opts.Destinations {
		destRef, err := name.NewTag(destination, name.WeakValidation)
		if err != nil {
			return errors.Wrap(err, "getting tag for destination")
		}
		destRefs = append(destRefs, destRef)
	}

	if opts.TarPath != "" {
		tagToImage := map[name.Tag]v1.Image{}
		for _, destRef := range destRefs {
			tagToImage[destRef] = image
		}
		return tarball.MultiWriteToFile(opts.TarPath, tagToImage)
	}

	if opts.NoPush {
		logrus.Info("Skipping push to container registry due to --no-push flag")
		return nil
	}

	// continue pushing unless an error occurs
	for _, destRef := range destRefs {
		registryName := destRef.Repository.Registry.Name()
		if opts.Insecure || opts.InsecureRegistries.Contains(registryName) {
			newReg, err := name.NewRegistry(registryName, name.WeakValidation, name.Insecure)
			if err != nil {
				return errors.Wrap(err, "getting new insecure registry")
			}
			destRef.Repository.Registry = newReg
		}

		pushAuth, err := creds.GetKeychain().Resolve(destRef.Context().Registry)
		if err != nil {
			return errors.Wrap(err, "resolving pushAuth")
		}

		tr := makeTransport(opts, registryName)
		rt := &withUserAgent{t: tr}

		if err := remote.Write(destRef, image, remote.WithAuth(pushAuth), remote.WithTransport(rt)); err != nil {
			return errors.Wrap(err, fmt.Sprintf("failed to push to destination %s", destRef))
		}
	}
	timing.DefaultRun.Stop(t)
	return nil
}

func makeTransport(opts *config.KanikoOptions, registryName string) http.RoundTripper {
	// Create a transport to set our user-agent.
	tr := http.DefaultTransport
	if opts.SkipTLSVerify || opts.SkipTLSVerifyRegistries.Contains(registryName) {
		tr.(*http.Transport).TLSClientConfig = &tls.Config{
			InsecureSkipVerify: true,
		}
	}
	return tr
}

// pushLayerToCache pushes layer (tagged with cacheKey) to opts.Cache
// if opts.Cache doesn't exist, infer the cache from the given destination
func pushLayerToCache(opts *config.KanikoOptions, cacheKey string, tarPath string, createdBy string) error {
	layer, err := tarball.LayerFromFile(tarPath)
	if err != nil {
		return err
	}
	cache, err := cache.Destination(opts, cacheKey)
	if err != nil {
		return errors.Wrap(err, "getting cache destination")
	}
	logrus.Infof("Pushing layer %s to cache now", cache)
	empty := empty.Image
	empty, err = mutate.CreatedAt(empty, v1.Time{Time: time.Now()})
	if err != nil {
		return errors.Wrap(err, "setting empty image created time")
	}

	empty, err = mutate.Append(empty,
		mutate.Addendum{
			Layer: layer,
			History: v1.History{
				Author:    constants.Author,
				CreatedBy: createdBy,
			},
		},
	)
	if err != nil {
		return errors.Wrap(err, "appending layer onto empty image")
	}
	cacheOpts := *opts
	cacheOpts.TarPath = ""   // tarPath doesn't make sense for Docker layers
	cacheOpts.NoPush = false // we want to push cached layers
	cacheOpts.Destinations = []string{cache}
	cacheOpts.InsecureRegistries = opts.InsecureRegistries
	cacheOpts.SkipTLSVerifyRegistries = opts.SkipTLSVerifyRegistries
	return DoPush(empty, &cacheOpts)
}
<commit_msg>change schema to http when Insecure flag is enabled<commit_after>/*
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package executor

import (
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/GoogleContainerTools/kaniko/pkg/cache"
	"github.com/GoogleContainerTools/kaniko/pkg/config"
	"github.com/GoogleContainerTools/kaniko/pkg/constants"
	"github.com/GoogleContainerTools/kaniko/pkg/creds"
	"github.com/GoogleContainerTools/kaniko/pkg/timing"
	"github.com/GoogleContainerTools/kaniko/pkg/version"
	"github.com/google/go-containerregistry/pkg/name"
	"github.com/google/go-containerregistry/pkg/v1"
	"github.com/google/go-containerregistry/pkg/v1/empty"
	"github.com/google/go-containerregistry/pkg/v1/layout"
	"github.com/google/go-containerregistry/pkg/v1/mutate"
	"github.com/google/go-containerregistry/pkg/v1/remote"
	"github.com/google/go-containerregistry/pkg/v1/tarball"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

type withUserAgent struct {
	t http.RoundTripper
}

const (
	UpstreamClientUaKey = "UPSTREAM_CLIENT_TYPE"
)

func (w *withUserAgent) RoundTrip(r *http.Request) (*http.Response, error) {
	ua := []string{fmt.Sprintf("kaniko/%s", version.Version())}
	if upstream := os.Getenv(UpstreamClientUaKey); upstream != "" {
		ua = append(ua, upstream)
	}
	r.Header.Set("User-Agent", strings.Join(ua, ","))
	return w.t.RoundTrip(r)
}

// CheckPushPermissionos checks that the configured credentials can be used to
// push to every specified destination.
func CheckPushPermissions(opts *config.KanikoOptions) error {
	if opts.NoPush {
		return nil
	}

	checked := map[string]bool{}
	for _, destination := range opts.Destinations {
		destRef, err := name.NewTag(destination, name.WeakValidation)
		if err != nil {
			return errors.Wrap(err, "getting tag for destination")
		}
		if checked[destRef.Context().RepositoryStr()] {
			continue
		}

		registryName := destRef.Repository.Registry.Name()
		if opts.Insecure || opts.InsecureRegistries.Contains(registryName) {
			newReg, err := name.NewRegistry(registryName, name.WeakValidation, name.Insecure)
			if err != nil {
				return errors.Wrap(err, "getting new insecure registry")
			}
			destRef.Repository.Registry = newReg
		}
		tr := makeTransport(opts, registryName)
		if err := remote.CheckPushPermission(destRef, creds.GetKeychain(), tr); err != nil {
			return errors.Wrapf(err, "checking push permission for %q", destRef)
		}
		checked[destRef.Context().RepositoryStr()] = true
	}
	return nil
}

// DoPush is responsible for pushing image to the destinations specified in opts
func DoPush(image v1.Image, opts *config.KanikoOptions) error {
	t := timing.Start("Total Push Time")

	if opts.DigestFile != "" {
		digest, err := image.Digest()
		if err != nil {
			return errors.Wrap(err, "error fetching digest")
		}
		digestByteArray := []byte(digest.String())
		err = ioutil.WriteFile(opts.DigestFile, digestByteArray, 0644)
		if err != nil {
			return errors.Wrap(err, "writing digest to file failed")
		}
	}

	if opts.OCILayoutPath != "" {
		path, err := layout.Write(opts.OCILayoutPath, empty.Index)
		if err != nil {
			return errors.Wrap(err, "writing empty layout")
		}
		if err := path.AppendImage(image); err != nil {
			return errors.Wrap(err, "appending image")
		}
	}

	destRefs := []name.Tag{}
	for _, destination := range opts.Destinations {
		destRef, err := name.NewTag(destination, name.WeakValidation)
		if err != nil {
			return errors.Wrap(err, "getting tag for destination")
		}
		destRefs = append(destRefs, destRef)
	}

	if opts.TarPath != "" {
		tagToImage := map[name.Tag]v1.Image{}
		for _, destRef := range destRefs {
			tagToImage[destRef] = image
		}
		return tarball.MultiWriteToFile(opts.TarPath, tagToImage)
	}

	if opts.NoPush {
		logrus.Info("Skipping push to container registry due to --no-push flag")
		return nil
	}

	// continue pushing unless an error occurs
	for _, destRef := range destRefs {
		registryName := destRef.Repository.Registry.Name()
		if opts.Insecure || opts.InsecureRegistries.Contains(registryName) {
			newReg, err := name.NewRegistry(registryName, name.WeakValidation, name.Insecure)
			if err != nil {
				return errors.Wrap(err, "getting new insecure registry")
			}
			destRef.Repository.Registry = newReg
		}

		pushAuth, err := creds.GetKeychain().Resolve(destRef.Context().Registry)
		if err != nil {
			return errors.Wrap(err, "resolving pushAuth")
		}

		tr := makeTransport(opts, registryName)
		rt := &withUserAgent{t: tr}

		if err := remote.Write(destRef, image, remote.WithAuth(pushAuth), remote.WithTransport(rt)); err != nil {
			return errors.Wrap(err, fmt.Sprintf("failed to push to destination %s", destRef))
		}
	}
	timing.DefaultRun.Stop(t)
	return nil
}

func makeTransport(opts *config.KanikoOptions, registryName string) http.RoundTripper {
	// Create a transport to set our user-agent.
	tr := http.DefaultTransport
	if opts.SkipTLSVerify || opts.SkipTLSVerifyRegistries.Contains(registryName) {
		tr.(*http.Transport).TLSClientConfig = &tls.Config{
			InsecureSkipVerify: true,
		}
	}
	return tr
}

// pushLayerToCache pushes layer (tagged with cacheKey) to opts.Cache
// if opts.Cache doesn't exist, infer the cache from the given destination
func pushLayerToCache(opts *config.KanikoOptions, cacheKey string, tarPath string, createdBy string) error {
	layer, err := tarball.LayerFromFile(tarPath)
	if err != nil {
		return err
	}
	cache, err := cache.Destination(opts, cacheKey)
	if err != nil {
		return errors.Wrap(err, "getting cache destination")
	}
	logrus.Infof("Pushing layer %s to cache now", cache)
	empty := empty.Image
	empty, err = mutate.CreatedAt(empty, v1.Time{Time: time.Now()})
	if err != nil {
		return errors.Wrap(err, "setting empty image created time")
	}

	empty, err = mutate.Append(empty,
		mutate.Addendum{
			Layer: layer,
			History: v1.History{
				Author:    constants.Author,
				CreatedBy: createdBy,
			},
		},
	)
	if err != nil {
		return errors.Wrap(err, "appending layer onto empty image")
	}
	cacheOpts := *opts
	cacheOpts.TarPath = ""   // tarPath doesn't make sense for Docker layers
	cacheOpts.NoPush = false // we want to push cached layers
	cacheOpts.Destinations = []string{cache}
	cacheOpts.InsecureRegistries = opts.InsecureRegistries
	cacheOpts.SkipTLSVerifyRegistries = opts.SkipTLSVerifyRegistries
	return DoPush(empty, &cacheOpts)
}
<|endoftext|>


<commit_before>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2018 Red Hat, Inc.
 *
 */

package hooks

import (
	"context"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"net"
	"sort"
	"strings"
	"sync"
	"time"

	"google.golang.org/grpc"

	v1 "kubevirt.io/kubevirt/pkg/api/v1"
	hooksInfo "kubevirt.io/kubevirt/pkg/hooks/info"
	hooksV1alpha1 "kubevirt.io/kubevirt/pkg/hooks/v1alpha1"
	hooksV1alpha2 "kubevirt.io/kubevirt/pkg/hooks/v1alpha2"
	"kubevirt.io/kubevirt/pkg/log"
	virtwrapApi "kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/api"
)

type callBackClient struct {
	SocketPath          string
	Version             string
	subsribedHookPoints []*hooksInfo.HookPoint
}

var manager *Manager
var once sync.Once

type Manager struct {
	callbacksPerHookPoint map[string][]*callBackClient
}

func GetManager() *Manager {
	once.Do(func() {
		manager = &Manager{callbacksPerHookPoint: make(map[string][]*callBackClient)}
	})
	return manager
}

func (m *Manager) Collect(numberOfRequestedHookSidecars uint, timeout time.Duration) error {
	callbacksPerHookPoint, err := collectSideCarSockets(numberOfRequestedHookSidecars, timeout)
	if err != nil {
		return err
	}
	log.Log.Info("Collected all requested hook sidecar sockets")

	sortCallbacksPerHookPoint(callbacksPerHookPoint)
	log.Log.Infof("Sorted all collected sidecar sockets per hook point based on their priority and name: %v", callbacksPerHookPoint)

	m.callbacksPerHookPoint = callbacksPerHookPoint

	return nil
}

// TODO: Handle sockets in parallel, when a socket appears, run a goroutine trying to read Info from it
func collectSideCarSockets(numberOfRequestedHookSidecars uint, timeout time.Duration) (map[string][]*callBackClient, error) {
	callbacksPerHookPoint := make(map[string][]*callBackClient)
	processedSockets := make(map[string]bool)

	timeoutCh := time.After(timeout)

	for uint(len(processedSockets)) < numberOfRequestedHookSidecars {
		sockets, err := ioutil.ReadDir(HookSocketsSharedDirectory)
		if err != nil {
			return nil, err
		}

		for _, socket := range sockets {
			select {
			case <-timeoutCh:
				return nil, fmt.Errorf("Failed to collect all expected sidecar hook sockets within given timeout")
			default:
				if _, processed := processedSockets[socket.Name()]; processed {
					continue
				}

				callBackClient, notReady, err := processSideCarSocket(HookSocketsSharedDirectory + "/" + socket.Name())
				if notReady {
					log.Log.Info("Sidecar server might not be ready yet, retrying in the next iteration")
					continue
				} else if err != nil {
					log.Log.Reason(err).Infof("Failed to process sidecar socket: %s", socket.Name())
					return nil, err
				}

				for _, subsribedHookPoint := range callBackClient.subsribedHookPoints {
					callbacksPerHookPoint[subsribedHookPoint.GetName()] = append(callbacksPerHookPoint[subsribedHookPoint.GetName()], callBackClient)
				}

				processedSockets[socket.Name()] = true
			}
		}

		time.Sleep(time.Second)
	}

	return callbacksPerHookPoint, nil
}

func processSideCarSocket(socketPath string) (*callBackClient, bool, error) {
	conn, err := dialSocket(socketPath)
	if err != nil {
		log.Log.Reason(err).Infof("Failed to Dial hook socket: %s", socketPath)
		return nil, true, nil
	}
	defer conn.Close()

	infoClient := hooksInfo.NewInfoClient(conn)
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	info, err := infoClient.Info(ctx, &hooksInfo.InfoParams{})
	if err != nil {
		return nil, false, err
	}

	versionsSet := make(map[string]bool)
	for _, version := range info.GetVersions() {
		versionsSet[version] = true
	}

	if _, found := versionsSet[hooksV1alpha2.Version]; found {
		return &callBackClient{
			SocketPath:          socketPath,
			Version:             hooksV1alpha2.Version,
			subsribedHookPoints: info.GetHookPoints(),
		}, false, nil
	} else if _, found := versionsSet[hooksV1alpha1.Version]; found {
		return &callBackClient{
			SocketPath:          socketPath,
			Version:             hooksV1alpha1.Version,
			subsribedHookPoints: info.GetHookPoints(),
		}, false, nil
	} else {
		return nil, false,
			fmt.Errorf("Hook sidecar does not expose a supported version. Exposed versions: %v, supported versions: %v",
				info.GetVersions(), []string{hooksV1alpha1.Version, hooksV1alpha2.Version})
	}
}

func sortCallbacksPerHookPoint(callbacksPerHookPoint map[string][]*callBackClient) {
	for _, callbacks := range callbacksPerHookPoint {
		for _, callback := range callbacks {
			sort.Slice(callbacks, func(i, j int) bool {
				if callback.subsribedHookPoints[i].Priority == callback.subsribedHookPoints[j].Priority {
					return strings.Compare(callback.subsribedHookPoints[i].Name, callback.subsribedHookPoints[j].Name) < 0
				} else {
					return callback.subsribedHookPoints[i].Priority > callback.subsribedHookPoints[j].Priority
				}
			})
		}
	}
}

func (m *Manager) OnDefineDomain(domainSpec *virtwrapApi.DomainSpec, vmi *v1.VirtualMachineInstance) (string, error) {
	domainSpecXML, err := xml.Marshal(domainSpec)
	if err != nil {
		return "", fmt.Errorf("Failed to marshal domain spec: %v", domainSpec)
	}
	if callbacks, found := m.callbacksPerHookPoint[hooksInfo.OnDefineDomainHookPointName]; found {
		for _, callback := range callbacks {
			if callback.Version == hooksV1alpha1.Version {

				vmiJSON, err := json.Marshal(vmi)
				if err != nil {
					return "", fmt.Errorf("Failed to marshal VMI spec: %v", vmi)
				}

				conn, err := dialSocket(callback.SocketPath)
				if err != nil {
					log.Log.Reason(err).Infof("Failed to Dial hook socket: %s", callback.SocketPath)
					return "", err
				}
				defer conn.Close()

				client := hooksV1alpha1.NewCallbacksClient(conn)

				ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
				defer cancel()
				result, err := client.OnDefineDomain(ctx, &hooksV1alpha1.OnDefineDomainParams{
					DomainXML: domainSpecXML,
					Vmi:       vmiJSON,
				})
				if err != nil {
					return "", err
				}
				domainSpecXML = result.GetDomainXML()
			} else {
				panic("Should never happen, version compatibility check is done during Info call")
			}
		}
	}
	return string(domainSpecXML), nil
}

func (m *Manager) PreCloudInitIso(vmi *v1.VirtualMachineInstance, source *v1.CloudInitNoCloudSource) (*v1.CloudInitNoCloudSource, error) {
	if callbacks, found := m.callbacksPerHookPoint[hooksInfo.PreCloudInitIsoHookPointName]; found {
		for _, callback := range callbacks {
			if callback.Version == hooksV1alpha2.Version {
				var resultSource *v1.CloudInitNoCloudSource
				vmiJSON, err := json.Marshal(vmi)
				if err != nil {
					return source, fmt.Errorf("Failed to marshal VMI spec: %v", vmi)
				}

				cloudInitData, err := json.Marshal(source)
				if err != nil {
					return source, fmt.Errorf("Failed to marshal CloudInitNoCloudSource: %v", source)
				}

				conn, err := dialSocket(callback.SocketPath)
				if err != nil {
					log.Log.Reason(err).Infof("Failed to Dial hook socket: %s", callback.SocketPath)
					return source, err
				}
				defer conn.Close()

				client := hooksV1alpha2.NewCallbacksClient(conn)
				ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
				defer cancel()
				result, err := client.PreCloudInitIso(ctx, &hooksV1alpha2.PreCloudInitIsoParams{
					CloudInitData: cloudInitData,
					Vmi:           vmiJSON,
				})
				if err != nil {
					return source, err
				}

				err = json.Unmarshal(result.GetCloudInitData(), &resultSource)
				if err != nil {
					return source, err
				}
				return resultSource, nil
			} else {
				panic("Should never happen, version compatibility check is done during Info call")
			}
		}
	}
	return source, nil
}

func dialSocket(socketPath string) (*grpc.ClientConn, error) {
	return grpc.Dial(
		socketPath,
		grpc.WithInsecure(),
		grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) {
			return net.DialTimeout("unix", addr, timeout)
		}),
		grpc.WithTimeout(time.Second),
	)
}
<commit_msg>Better variable names<commit_after>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2018 Red Hat, Inc.
 *
 */

package hooks

import (
	"context"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"net"
	"sort"
	"strings"
	"sync"
	"time"

	"google.golang.org/grpc"

	v1 "kubevirt.io/kubevirt/pkg/api/v1"
	hooksInfo "kubevirt.io/kubevirt/pkg/hooks/info"
	hooksV1alpha1 "kubevirt.io/kubevirt/pkg/hooks/v1alpha1"
	hooksV1alpha2 "kubevirt.io/kubevirt/pkg/hooks/v1alpha2"
	"kubevirt.io/kubevirt/pkg/log"
	virtwrapApi "kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/api"
)

type callBackClient struct {
	SocketPath          string
	Version             string
	subsribedHookPoints []*hooksInfo.HookPoint
}

var manager *Manager
var once sync.Once

type Manager struct {
	callbacksPerHookPoint map[string][]*callBackClient
}

func GetManager() *Manager {
	once.Do(func() {
		manager = &Manager{callbacksPerHookPoint: make(map[string][]*callBackClient)}
	})
	return manager
}

func (m *Manager) Collect(numberOfRequestedHookSidecars uint, timeout time.Duration) error {
	callbacksPerHookPoint, err := collectSideCarSockets(numberOfRequestedHookSidecars, timeout)
	if err != nil {
		return err
	}
	log.Log.Info("Collected all requested hook sidecar sockets")

	sortCallbacksPerHookPoint(callbacksPerHookPoint)
	log.Log.Infof("Sorted all collected sidecar sockets per hook point based on their priority and name: %v", callbacksPerHookPoint)

	m.callbacksPerHookPoint = callbacksPerHookPoint

	return nil
}

// TODO: Handle sockets in parallel, when a socket appears, run a goroutine trying to read Info from it
func collectSideCarSockets(numberOfRequestedHookSidecars uint, timeout time.Duration) (map[string][]*callBackClient, error) {
	callbacksPerHookPoint := make(map[string][]*callBackClient)
	processedSockets := make(map[string]bool)

	timeoutCh := time.After(timeout)

	for uint(len(processedSockets)) < numberOfRequestedHookSidecars {
		sockets, err := ioutil.ReadDir(HookSocketsSharedDirectory)
		if err != nil {
			return nil, err
		}

		for _, socket := range sockets {
			select {
			case <-timeoutCh:
				return nil, fmt.Errorf("Failed to collect all expected sidecar hook sockets within given timeout")
			default:
				if _, processed := processedSockets[socket.Name()]; processed {
					continue
				}

				callBackClient, notReady, err := processSideCarSocket(HookSocketsSharedDirectory + "/" + socket.Name())
				if notReady {
					log.Log.Info("Sidecar server might not be ready yet, retrying in the next iteration")
					continue
				} else if err != nil {
					log.Log.Reason(err).Infof("Failed to process sidecar socket: %s", socket.Name())
					return nil, err
				}

				for _, subsribedHookPoint := range callBackClient.subsribedHookPoints {
					callbacksPerHookPoint[subsribedHookPoint.GetName()] = append(callbacksPerHookPoint[subsribedHookPoint.GetName()], callBackClient)
				}

				processedSockets[socket.Name()] = true
			}
		}

		time.Sleep(time.Second)
	}

	return callbacksPerHookPoint, nil
}

func processSideCarSocket(socketPath string) (*callBackClient, bool, error) {
	conn, err := dialSocket(socketPath)
	if err != nil {
		log.Log.Reason(err).Infof("Failed to Dial hook socket: %s", socketPath)
		return nil, true, nil
	}
	defer conn.Close()

	infoClient := hooksInfo.NewInfoClient(conn)
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	info, err := infoClient.Info(ctx, &hooksInfo.InfoParams{})
	if err != nil {
		return nil, false, err
	}

	versionsSet := make(map[string]bool)
	for _, version := range info.GetVersions() {
		versionsSet[version] = true
	}

	if _, found := versionsSet[hooksV1alpha2.Version]; found {
		return &callBackClient{
			SocketPath:          socketPath,
			Version:             hooksV1alpha2.Version,
			subsribedHookPoints: info.GetHookPoints(),
		}, false, nil
	} else if _, found := versionsSet[hooksV1alpha1.Version]; found {
		return &callBackClient{
			SocketPath:          socketPath,
			Version:             hooksV1alpha1.Version,
			subsribedHookPoints: info.GetHookPoints(),
		}, false, nil
	} else {
		return nil, false,
			fmt.Errorf("Hook sidecar does not expose a supported version. Exposed versions: %v, supported versions: %v",
				info.GetVersions(), []string{hooksV1alpha1.Version, hooksV1alpha2.Version})
	}
}

func sortCallbacksPerHookPoint(callbacksPerHookPoint map[string][]*callBackClient) {
	for _, callbacks := range callbacksPerHookPoint {
		for _, callback := range callbacks {
			sort.Slice(callbacks, func(i, j int) bool {
				if callback.subsribedHookPoints[i].Priority == callback.subsribedHookPoints[j].Priority {
					return strings.Compare(callback.subsribedHookPoints[i].Name, callback.subsribedHookPoints[j].Name) < 0
				} else {
					return callback.subsribedHookPoints[i].Priority > callback.subsribedHookPoints[j].Priority
				}
			})
		}
	}
}

func (m *Manager) OnDefineDomain(domainSpec *virtwrapApi.DomainSpec, vmi *v1.VirtualMachineInstance) (string, error) {
	domainSpecXML, err := xml.Marshal(domainSpec)
	if err != nil {
		return "", fmt.Errorf("Failed to marshal domain spec: %v", domainSpec)
	}
	if callbacks, found := m.callbacksPerHookPoint[hooksInfo.OnDefineDomainHookPointName]; found {
		for _, callback := range callbacks {
			if callback.Version == hooksV1alpha1.Version {

				vmiJSON, err := json.Marshal(vmi)
				if err != nil {
					return "", fmt.Errorf("Failed to marshal VMI spec: %v", vmi)
				}

				conn, err := dialSocket(callback.SocketPath)
				if err != nil {
					log.Log.Reason(err).Infof("Failed to Dial hook socket: %s", callback.SocketPath)
					return "", err
				}
				defer conn.Close()

				client := hooksV1alpha1.NewCallbacksClient(conn)

				ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
				defer cancel()
				result, err := client.OnDefineDomain(ctx, &hooksV1alpha1.OnDefineDomainParams{
					DomainXML: domainSpecXML,
					Vmi:       vmiJSON,
				})
				if err != nil {
					return "", err
				}
				domainSpecXML = result.GetDomainXML()
			} else {
				panic("Should never happen, version compatibility check is done during Info call")
			}
		}
	}
	return string(domainSpecXML), nil
}

func (m *Manager) PreCloudInitIso(vmi *v1.VirtualMachineInstance, cloudInitData *v1.CloudInitNoCloudSource) (*v1.CloudInitNoCloudSource, error) {
	if callbacks, found := m.callbacksPerHookPoint[hooksInfo.PreCloudInitIsoHookPointName]; found {
		for _, callback := range callbacks {
			if callback.Version == hooksV1alpha2.Version {
				var resultSource *v1.CloudInitNoCloudSource
				vmiJSON, err := json.Marshal(vmi)
				if err != nil {
					return cloudInitData, fmt.Errorf("Failed to marshal VMI spec: %v", vmi)
				}

				cloudInitDataJSON, err := json.Marshal(cloudInitData)
				if err != nil {
					return cloudInitData, fmt.Errorf("Failed to marshal CloudInitNoCloudSource: %v", cloudInitData)
				}

				conn, err := dialSocket(callback.SocketPath)
				if err != nil {
					log.Log.Reason(err).Infof("Failed to Dial hook socket: %s", callback.SocketPath)
					return cloudInitData, err
				}
				defer conn.Close()

				client := hooksV1alpha2.NewCallbacksClient(conn)
				ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
				defer cancel()
				result, err := client.PreCloudInitIso(ctx, &hooksV1alpha2.PreCloudInitIsoParams{
					CloudInitData: cloudInitDataJSON,
					Vmi:           vmiJSON,
				})
				if err != nil {
					return cloudInitData, err
				}

				err = json.Unmarshal(result.GetCloudInitData(), &resultSource)
				if err != nil {
					return cloudInitData, err
				}
				return resultSource, nil
			} else {
				panic("Should never happen, version compatibility check is done during Info call")
			}
		}
	}
	return cloudInitData, nil
}

func dialSocket(socketPath string) (*grpc.ClientConn, error) {
	return grpc.Dial(
		socketPath,
		grpc.WithInsecure(),
		grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) {
			return net.DialTimeout("unix", addr, timeout)
		}),
		grpc.WithTimeout(time.Second),
	)
}
<|endoftext|>


<commit_before>package myslave

import (
	"github.com/siddontang/go-mysql/client"
	"github.com/siddontang/go-mysql/mysql"
)

// BinlogRowImage checks MySQL binlog row image, must be in FULL, MINIMAL, NOBLOB.
func (m *MySlave) BinlogRowImage() (string, error) {
	if m.c.String("flavor", "mysql") != mysql.MySQLFlavor {
		return "", nil
	}

	if res, err := m.Execute(`SHOW GLOBAL VARIABLES LIKE "binlog_row_image"`); err != nil {
		return "", err
	} else {
		// MySQL has binlog row image from 5.6, so older will return empty
		return res.GetString(0, 1)
	}
}

// AssertValidRowFormat asserts the mysql master binlog format is ROW.
func (m *MySlave) AssertValidRowFormat() error {
	res, err := m.Execute(`SHOW GLOBAL VARIABLES LIKE "binlog_format";`)
	if err != nil {
		return err
	}

	if f, err := res.GetString(0, 1); err != nil {
		return err
	} else if f != "ROW" {
		return ErrInvalidRowFormat
	}

	return nil
}

// Execute executes a SQL against the mysql master.
func (m *MySlave) Execute(cmd string, args ...interface{}) (rr *mysql.Result, err error) {
	const maxRetry = 3
	for i := 0; i < maxRetry; i++ {
		if m.conn == nil {
			m.conn, err = client.Connect(m.masterAddr, m.user, m.passwd, "")
			if err != nil {
				return nil, err
			}
		}

		rr, err = m.conn.Execute(cmd, args...)
		if err != nil && !mysql.ErrorEqual(err, mysql.ErrBadConn) {
			return
		} else if mysql.ErrorEqual(err, mysql.ErrBadConn) {
			m.conn.Close()
			m.conn = nil
			continue
		} else {
			return
		}
	}

	return
}

// MasterPosition returns the latest mysql master binlog position info.
func (m *MySlave) MasterPosition() (*mysql.Position, error) {
	rr, err := m.conn.Execute("SHOW MASTER STATUS")
	if err != nil {
		return nil, err
	}

	name, _ := rr.GetString(0, 0)
	pos, _ := rr.GetInt(0, 1)
	return &mysql.Position{
		Name: name,
		Pos:  uint32(pos),
	}, nil
}
<commit_msg>myslave fetches all binlog files on master<commit_after>package myslave

import (
	"github.com/siddontang/go-mysql/client"
	"github.com/siddontang/go-mysql/mysql"
)

// BinlogRowImage checks MySQL binlog row image, must be in FULL, MINIMAL, NOBLOB.
func (m *MySlave) BinlogRowImage() (string, error) {
	if m.c.String("flavor", "mysql") != mysql.MySQLFlavor {
		return "", nil
	}

	if res, err := m.Execute(`SHOW GLOBAL VARIABLES LIKE "binlog_row_image"`); err != nil {
		return "", err
	} else {
		// MySQL has binlog row image from 5.6, so older will return empty
		return res.GetString(0, 1)
	}
}

// AssertValidRowFormat asserts the mysql master binlog format is ROW.
func (m *MySlave) AssertValidRowFormat() error {
	res, err := m.Execute(`SHOW GLOBAL VARIABLES LIKE "binlog_format";`)
	if err != nil {
		return err
	}

	if f, err := res.GetString(0, 1); err != nil {
		return err
	} else if f != "ROW" {
		return ErrInvalidRowFormat
	}

	return nil
}

// Execute executes a SQL against the mysql master.
func (m *MySlave) Execute(cmd string, args ...interface{}) (rr *mysql.Result, err error) {
	const maxRetry = 3
	for i := 0; i < maxRetry; i++ {
		if m.conn == nil {
			m.conn, err = client.Connect(m.masterAddr, m.user, m.passwd, "")
			if err != nil {
				return nil, err
			}
		}

		rr, err = m.conn.Execute(cmd, args...)
		if err != nil && !mysql.ErrorEqual(err, mysql.ErrBadConn) {
			return
		} else if mysql.ErrorEqual(err, mysql.ErrBadConn) {
			m.conn.Close()
			m.conn = nil
			continue
		} else {
			return
		}
	}

	return
}

// MasterPosition returns the latest mysql master binlog position info.
func (m *MySlave) MasterPosition() (*mysql.Position, error) {
	rr, err := m.conn.Execute("SHOW MASTER STATUS")
	if err != nil {
		return nil, err
	}

	name, _ := rr.GetString(0, 0)
	pos, _ := rr.GetInt(0, 1)
	return &mysql.Position{
		Name: name,
		Pos:  uint32(pos),
	}, nil
}

// MasterBinlogs returns all binlog files on master.
func (m *MySlave) MasterBinlogs() ([]string, error) {
	rr, err := m.conn.Execute("SHOW BINARY LOGS")
	if err != nil {
		return nil, err
	}

	names := make([]string, rr.RowNumber())
	for i := 0; i < rr.RowNumber(); i++ {
		name, err := rr.GetString(i, 0) // [0] is Log_name, [1] is File_size
		if err != nil {
			return nil, err
		}

		names[i] = name
	}

	return names, nil
}
<|endoftext|>


<commit_before>// Copyright 2016-2018 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package option

import (
	"fmt"
	"sort"
	"strings"

	"github.com/cilium/cilium/api/v1/models"
	"github.com/cilium/cilium/pkg/color"
	"github.com/cilium/cilium/pkg/lock"
)

// VerifyFunc validates option key with value and may return an error if the
// option should not be applied
type VerifyFunc func(key string, value string) error

// ParseFunc parses the option value and may return an error if the option
// cannot be parsed or applied.
type ParseFunc func(value string) (int, error)

// FormatFunc formats the specified value as a colored textual representation
// of the option.
type FormatFunc func(value int) string

// Option is the structure used to specify the semantics of a configurable
// boolean option
type Option struct {
	// Define is the name of the #define used for BPF programs
	Define string
	// Description is a short human readable description
	Description string
	// Immutable marks an option which is read-only
	Immutable bool
	// Requires is a list of required options, such options will be
	// automatically enabled as required.
	Requires []string
	// Parse is called to parse the option. If not specified, defaults to
	// NormalizeBool().
	Parse ParseFunc
	// FormatFunc is called to format the value for an option. If not
	// specified, defaults to formatting 0 as "Disabled" and other values
	// as "Enabled".
	Format FormatFunc
	// Verify is called prior to applying the option
	Verify VerifyFunc
}

const (
	OptionDisabled = iota
	OptionEnabled
)

// RequiresOption returns true if the option requires the specified option `name`.
func (o Option) RequiresOption(name string) bool {
	for _, o := range o.Requires {
		if o == name {
			return true
		}
	}

	return false
}

type OptionLibrary map[string]*Option

func (l OptionLibrary) Lookup(name string) (string, *Option) {
	nameLower := strings.ToLower(name)

	for k := range l {
		if strings.ToLower(k) == nameLower {
			return k, l[k]
		}
	}

	return "", nil
}

func (l OptionLibrary) Define(name string) string {
	if _, ok := l[name]; ok {
		return l[name].Define
	}

	return name
}

func NormalizeBool(value string) (int, error) {
	switch strings.ToLower(value) {
	case "true", "on", "enable", "enabled", "1":
		return OptionEnabled, nil
	case "false", "off", "disable", "disabled", "0":
		return OptionDisabled, nil
	default:
		return OptionDisabled, fmt.Errorf("Invalid option value %s", value)
	}
}

func (l OptionLibrary) Validate(name string, value string) error {
	key, spec := l.Lookup(name)
	if key == "" {
		return fmt.Errorf("Unknown option %s", name)
	}

	if spec.Immutable {
		return fmt.Errorf("Specified option is immutable (read-only)")
	}

	if spec.Verify != nil {
		return spec.Verify(key, value)
	}

	return nil
}

type OptionMap map[string]int

func (om OptionMap) DeepCopy() OptionMap {
	cpy := make(OptionMap, len(om))
	for k, v := range om {
		cpy[k] = v
	}
	return cpy
}

// IntOptions member functions with external access do not require
// locking by the caller, while functions with internal access presume
// the caller to have taken care of any locking needed.
type IntOptions struct {
	optsMU  lock.RWMutex   // Protects all variables from this structure below this line
	Opts    OptionMap      `json:"map"`
	Library *OptionLibrary `json:"-"`
}

// GetImmutableModel returns the set of immutable options as a ConfigurationMap API model.
func (o *IntOptions) GetImmutableModel() *models.ConfigurationMap {
	immutableCfg := make(models.ConfigurationMap)
	return &immutableCfg
}

// GetMutableModel returns the set of mutable options as a ConfigurationMap API model.
func (o *IntOptions) GetMutableModel() *models.ConfigurationMap {
	mutableCfg := make(models.ConfigurationMap)
	o.optsMU.RLock()
	for k, v := range o.Opts {
		_, config := o.Library.Lookup(k)
		if config.Format == nil {
			if v == OptionDisabled {
				mutableCfg[k] = fmt.Sprintf("Disabled")
			} else {
				mutableCfg[k] = fmt.Sprintf("Enabled")
			}
		} else {
			mutableCfg[k] = config.Format(v)
		}
	}
	o.optsMU.RUnlock()

	return &mutableCfg
}

func (o *IntOptions) DeepCopy() *IntOptions {
	o.optsMU.RLock()
	cpy := &IntOptions{
		Opts:    o.Opts.DeepCopy(),
		Library: o.Library,
	}
	o.optsMU.RUnlock()
	return cpy
}

func NewIntOptions(lib *OptionLibrary) *IntOptions {
	return &IntOptions{
		Opts:    OptionMap{},
		Library: lib,
	}
}

func (o *IntOptions) GetValue(key string) int {
	value, exists := o.Opts[key]
	if !exists {
		return OptionDisabled
	}
	return value
}

func (o *IntOptions) IsEnabled(key string) bool {
	o.optsMU.RLock()
	defer o.optsMU.RUnlock()
	return o.GetValue(key) != OptionDisabled
}

// SetValidated sets the option `key` to the specified value. The caller is
// expected to have validated the input to this function.
func (o *IntOptions) SetValidated(key string, value int) {
	o.optsMU.Lock()
	o.Opts[key] = value
	o.optsMU.Unlock()
}

// SetBool sets the specified option to Enabled.
func (o *IntOptions) SetBool(key string, value bool) {
	intValue := OptionDisabled
	if value {
		intValue = OptionEnabled
	}
	o.optsMU.Lock()
	o.Opts[key] = intValue
	o.optsMU.Unlock()
}

func (o *IntOptions) Delete(key string) {
	o.optsMU.Lock()
	delete(o.Opts, key)
	o.optsMU.Unlock()
}

func (o *IntOptions) SetIfUnset(key string, value int) {
	o.optsMU.Lock()
	if _, exists := o.Opts[key]; !exists {
		o.Opts[key] = value
	}
	o.optsMU.Unlock()
}

func (o *IntOptions) InheritDefault(parent *IntOptions, key string) {
	o.optsMU.RLock()
	o.Opts[key] = parent.GetValue(key)
	o.optsMU.RUnlock()
}

func ParseOption(arg string, lib *OptionLibrary) (string, int, error) {
	result := OptionEnabled

	if arg[0] == '!' {
		result = OptionDisabled
		arg = arg[1:]
	}

	optionSplit := strings.SplitN(arg, "=", 2)
	arg = optionSplit[0]
	if len(optionSplit) > 1 {
		if result == OptionDisabled {
			return "", OptionDisabled, fmt.Errorf("Invalid boolean format")
		}

		return ParseKeyValue(lib, arg, optionSplit[1], result)
	}

	return "", OptionDisabled, fmt.Errorf("Invalid option format")
}

func ParseKeyValue(lib *OptionLibrary, arg, value string, defaultValue int) (string, int, error) {
	result := defaultValue

	key, spec := lib.Lookup(arg)
	if key == "" {
		return "", OptionDisabled, fmt.Errorf("Unknown option %q", arg)
	}

	var err error
	if spec.Parse != nil {
		result, err = spec.Parse(value)
	} else {
		result, err = NormalizeBool(value)
	}
	if err != nil {
		return "", OptionDisabled, err
	}

	if spec.Immutable {
		return "", OptionDisabled, fmt.Errorf("Specified option is immutable (read-only)")
	}

	return key, result, nil
}

// getFmtOpt returns #define name if option exists and is set to true in endpoint's Opts
// map or #undef name if option does not exist or exists but is set to false
func (o *IntOptions) getFmtOpt(name string) string {
	define := o.Library.Define(name)
	if define == "" {
		return ""
	}

	value := o.GetValue(name)
	if value != OptionDisabled {
		return fmt.Sprintf("#define %s %d", o.Library.Define(name), value)
	}
	return "#undef " + o.Library.Define(name)
}

func (o *IntOptions) GetFmtList() string {
	txt := ""

	o.optsMU.RLock()
	opts := []string{}
	for k := range o.Opts {
		opts = append(opts, k)
	}
	sort.Strings(opts)

	for _, k := range opts {
		def := o.getFmtOpt(k)
		if def != "" {
			txt += def + "\n"
		}
	}
	o.optsMU.RUnlock()

	return txt
}

func (o *IntOptions) Dump() {
	if o == nil {
		return
	}

	o.optsMU.RLock()
	opts := []string{}
	for k := range o.Opts {
		opts = append(opts, k)
	}
	sort.Strings(opts)

	for _, k := range opts {
		var text string
		_, option := o.Library.Lookup(k)
		if option == nil || option.Format == nil {
			if o.Opts[k] == OptionDisabled {
				text = color.Red("Disabled")
			} else {
				text = color.Green("Enabled")
			}
		} else {
			text = option.Format(o.Opts[k])
		}

		fmt.Printf("%-24s %s\n", k, text)
	}
	o.optsMU.RUnlock()
}

// Validate validates a given configuration map based on the option library
func (o *IntOptions) Validate(n models.ConfigurationMap) error {
	o.optsMU.RLock()
	defer o.optsMU.RUnlock()
	for k, v := range n {
		_, newVal, err := ParseKeyValue(o.Library, k, v, OptionDisabled)
		if err != nil {
			return err
		}

		// Ignore validation if value is identical
		if oldVal, ok := o.Opts[k]; ok && oldVal == newVal {
			continue
		}

		if err := o.Library.Validate(k, v); err != nil {
			return err
		}
	}

	return nil
}

// ChangedFunc is called by `Apply()` for each option changed
type ChangedFunc func(key string, value int, data interface{})

// enable enables the option `name` with all its dependencies
func (o *IntOptions) enable(name string) {
	if o.Library != nil {
		if _, opt := o.Library.Lookup(name); opt != nil {
			for _, dependency := range opt.Requires {
				o.enable(dependency)
			}
		}
	}

	o.Opts[name] = OptionEnabled
}

// set enables the option `name` with all its dependencies, and sets the
// integer level of the option to `value`.
func (o *IntOptions) set(name string, value int) {
	o.enable(name)
	o.Opts[name] = value
}

// disable disables the option `name`. All options which depend on the option
// to be disabled will be disabled. Options which have previously been enabled
// as a dependency will not be automatically disabled.
func (o *IntOptions) disable(name string) {
	o.Opts[name] = OptionDisabled

	if o.Library != nil {
		// Disable all options which have a dependency on the option
		// that was just disabled
		for key, opt := range *o.Library {
			if opt.RequiresOption(name) && o.Opts[key] != OptionDisabled {
				o.disable(key)
			}
		}
	}
}

type changedOptions struct {
	key   string
	value int
}

// ApplyValidated takes a configuration map and applies the changes. For an
// option which is changed, the `ChangedFunc` function is called with the
// `data` argument passed in as well. Returns the number of options changed if
// any.
//
// The caller is expected to have validated the configuration options prior to
// calling this function.
func (o *IntOptions) ApplyValidated(n models.ConfigurationMap, changed ChangedFunc, data interface{}) int {
	changes := []changedOptions{}

	o.optsMU.Lock()
	for k, v := range n {
		val, ok := o.Opts[k]

		// Ignore the error here because the option was already validated.
		_, optVal, _ := ParseKeyValue(o.Library, k, v, OptionDisabled)
		if optVal == OptionDisabled {
			/* Only disable if enabled already */
			if ok && val != OptionDisabled {
				o.disable(k)
				changes = append(changes, changedOptions{key: k, value: optVal})
			}
		} else {
			/* Only enable if not enabled already */
			if !ok || val == OptionDisabled {
				o.set(k, optVal)
				changes = append(changes, changedOptions{key: k, value: optVal})
			}
		}
	}
	o.optsMU.Unlock()

	for _, change := range changes {
		changed(change.key, change.value, data)
	}

	return len(changes)
}
<commit_msg>pkg/option: fix race in IntOptions GetValue method<commit_after>// Copyright 2016-2018 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package option

import (
	"fmt"
	"sort"
	"strings"

	"github.com/cilium/cilium/api/v1/models"
	"github.com/cilium/cilium/pkg/color"
	"github.com/cilium/cilium/pkg/lock"
)

// VerifyFunc validates option key with value and may return an error if the
// option should not be applied
type VerifyFunc func(key string, value string) error

// ParseFunc parses the option value and may return an error if the option
// cannot be parsed or applied.
type ParseFunc func(value string) (int, error)

// FormatFunc formats the specified value as a colored textual representation
// of the option.
type FormatFunc func(value int) string

// Option is the structure used to specify the semantics of a configurable
// boolean option
type Option struct {
	// Define is the name of the #define used for BPF programs
	Define string
	// Description is a short human readable description
	Description string
	// Immutable marks an option which is read-only
	Immutable bool
	// Requires is a list of required options, such options will be
	// automatically enabled as required.
	Requires []string
	// Parse is called to parse the option. If not specified, defaults to
	// NormalizeBool().
	Parse ParseFunc
	// FormatFunc is called to format the value for an option. If not
	// specified, defaults to formatting 0 as "Disabled" and other values
	// as "Enabled".
	Format FormatFunc
	// Verify is called prior to applying the option
	Verify VerifyFunc
}

const (
	OptionDisabled = iota
	OptionEnabled
)

// RequiresOption returns true if the option requires the specified option `name`.
func (o Option) RequiresOption(name string) bool {
	for _, o := range o.Requires {
		if o == name {
			return true
		}
	}

	return false
}

type OptionLibrary map[string]*Option

func (l OptionLibrary) Lookup(name string) (string, *Option) {
	nameLower := strings.ToLower(name)

	for k := range l {
		if strings.ToLower(k) == nameLower {
			return k, l[k]
		}
	}

	return "", nil
}

func (l OptionLibrary) Define(name string) string {
	if _, ok := l[name]; ok {
		return l[name].Define
	}

	return name
}

func NormalizeBool(value string) (int, error) {
	switch strings.ToLower(value) {
	case "true", "on", "enable", "enabled", "1":
		return OptionEnabled, nil
	case "false", "off", "disable", "disabled", "0":
		return OptionDisabled, nil
	default:
		return OptionDisabled, fmt.Errorf("Invalid option value %s", value)
	}
}

func (l OptionLibrary) Validate(name string, value string) error {
	key, spec := l.Lookup(name)
	if key == "" {
		return fmt.Errorf("Unknown option %s", name)
	}

	if spec.Immutable {
		return fmt.Errorf("Specified option is immutable (read-only)")
	}

	if spec.Verify != nil {
		return spec.Verify(key, value)
	}

	return nil
}

type OptionMap map[string]int

func (om OptionMap) DeepCopy() OptionMap {
	cpy := make(OptionMap, len(om))
	for k, v := range om {
		cpy[k] = v
	}
	return cpy
}

// IntOptions member functions with external access do not require
// locking by the caller, while functions with internal access presume
// the caller to have taken care of any locking needed.
type IntOptions struct {
	optsMU  lock.RWMutex   // Protects all variables from this structure below this line
	Opts    OptionMap      `json:"map"`
	Library *OptionLibrary `json:"-"`
}

// GetImmutableModel returns the set of immutable options as a ConfigurationMap API model.
func (o *IntOptions) GetImmutableModel() *models.ConfigurationMap {
	immutableCfg := make(models.ConfigurationMap)
	return &immutableCfg
}

// GetMutableModel returns the set of mutable options as a ConfigurationMap API model.
func (o *IntOptions) GetMutableModel() *models.ConfigurationMap {
	mutableCfg := make(models.ConfigurationMap)
	o.optsMU.RLock()
	for k, v := range o.Opts {
		_, config := o.Library.Lookup(k)
		if config.Format == nil {
			if v == OptionDisabled {
				mutableCfg[k] = fmt.Sprintf("Disabled")
			} else {
				mutableCfg[k] = fmt.Sprintf("Enabled")
			}
		} else {
			mutableCfg[k] = config.Format(v)
		}
	}
	o.optsMU.RUnlock()

	return &mutableCfg
}

func (o *IntOptions) DeepCopy() *IntOptions {
	o.optsMU.RLock()
	cpy := &IntOptions{
		Opts:    o.Opts.DeepCopy(),
		Library: o.Library,
	}
	o.optsMU.RUnlock()
	return cpy
}

func NewIntOptions(lib *OptionLibrary) *IntOptions {
	return &IntOptions{
		Opts:    OptionMap{},
		Library: lib,
	}
}

func (o *IntOptions) getValue(key string) int {
	value, exists := o.Opts[key]
	if !exists {
		return OptionDisabled
	}
	return value
}

func (o *IntOptions) GetValue(key string) int {
	o.optsMU.RLock()
	v := o.getValue(key)
	o.optsMU.RUnlock()
	return v
}

func (o *IntOptions) IsEnabled(key string) bool {
	return o.GetValue(key) != OptionDisabled
}

// SetValidated sets the option `key` to the specified value. The caller is
// expected to have validated the input to this function.
func (o *IntOptions) SetValidated(key string, value int) {
	o.optsMU.Lock()
	o.Opts[key] = value
	o.optsMU.Unlock()
}

// SetBool sets the specified option to Enabled.
func (o *IntOptions) SetBool(key string, value bool) {
	intValue := OptionDisabled
	if value {
		intValue = OptionEnabled
	}
	o.optsMU.Lock()
	o.Opts[key] = intValue
	o.optsMU.Unlock()
}

func (o *IntOptions) Delete(key string) {
	o.optsMU.Lock()
	delete(o.Opts, key)
	o.optsMU.Unlock()
}

func (o *IntOptions) SetIfUnset(key string, value int) {
	o.optsMU.Lock()
	if _, exists := o.Opts[key]; !exists {
		o.Opts[key] = value
	}
	o.optsMU.Unlock()
}

func (o *IntOptions) InheritDefault(parent *IntOptions, key string) {
	o.optsMU.RLock()
	o.Opts[key] = parent.GetValue(key)
	o.optsMU.RUnlock()
}

func ParseOption(arg string, lib *OptionLibrary) (string, int, error) {
	result := OptionEnabled

	if arg[0] == '!' {
		result = OptionDisabled
		arg = arg[1:]
	}

	optionSplit := strings.SplitN(arg, "=", 2)
	arg = optionSplit[0]
	if len(optionSplit) > 1 {
		if result == OptionDisabled {
			return "", OptionDisabled, fmt.Errorf("Invalid boolean format")
		}

		return ParseKeyValue(lib, arg, optionSplit[1], result)
	}

	return "", OptionDisabled, fmt.Errorf("Invalid option format")
}

func ParseKeyValue(lib *OptionLibrary, arg, value string, defaultValue int) (string, int, error) {
	result := defaultValue

	key, spec := lib.Lookup(arg)
	if key == "" {
		return "", OptionDisabled, fmt.Errorf("Unknown option %q", arg)
	}

	var err error
	if spec.Parse != nil {
		result, err = spec.Parse(value)
	} else {
		result, err = NormalizeBool(value)
	}
	if err != nil {
		return "", OptionDisabled, err
	}

	if spec.Immutable {
		return "", OptionDisabled, fmt.Errorf("Specified option is immutable (read-only)")
	}

	return key, result, nil
}

// getFmtOpt returns #define name if option exists and is set to true in endpoint's Opts
// map or #undef name if option does not exist or exists but is set to false
func (o *IntOptions) getFmtOpt(name string) string {
	define := o.Library.Define(name)
	if define == "" {
		return ""
	}

	value := o.getValue(name)
	if value != OptionDisabled {
		return fmt.Sprintf("#define %s %d", o.Library.Define(name), value)
	}
	return "#undef " + o.Library.Define(name)
}

func (o *IntOptions) GetFmtList() string {
	txt := ""

	o.optsMU.RLock()
	opts := []string{}
	for k := range o.Opts {
		opts = append(opts, k)
	}
	sort.Strings(opts)

	for _, k := range opts {
		def := o.getFmtOpt(k)
		if def != "" {
			txt += def + "\n"
		}
	}
	o.optsMU.RUnlock()

	return txt
}

func (o *IntOptions) Dump() {
	if o == nil {
		return
	}

	o.optsMU.RLock()
	opts := []string{}
	for k := range o.Opts {
		opts = append(opts, k)
	}
	sort.Strings(opts)

	for _, k := range opts {
		var text string
		_, option := o.Library.Lookup(k)
		if option == nil || option.Format == nil {
			if o.Opts[k] == OptionDisabled {
				text = color.Red("Disabled")
			} else {
				text = color.Green("Enabled")
			}
		} else {
			text = option.Format(o.Opts[k])
		}

		fmt.Printf("%-24s %s\n", k, text)
	}
	o.optsMU.RUnlock()
}

// Validate validates a given configuration map based on the option library
func (o *IntOptions) Validate(n models.ConfigurationMap) error {
	o.optsMU.RLock()
	defer o.optsMU.RUnlock()
	for k, v := range n {
		_, newVal, err := ParseKeyValue(o.Library, k, v, OptionDisabled)
		if err != nil {
			return err
		}

		// Ignore validation if value is identical
		if oldVal, ok := o.Opts[k]; ok && oldVal == newVal {
			continue
		}

		if err := o.Library.Validate(k, v); err != nil {
			return err
		}
	}

	return nil
}

// ChangedFunc is called by `Apply()` for each option changed
type ChangedFunc func(key string, value int, data interface{})

// enable enables the option `name` with all its dependencies
func (o *IntOptions) enable(name string) {
	if o.Library != nil {
		if _, opt := o.Library.Lookup(name); opt != nil {
			for _, dependency := range opt.Requires {
				o.enable(dependency)
			}
		}
	}

	o.Opts[name] = OptionEnabled
}

// set enables the option `name` with all its dependencies, and sets the
// integer level of the option to `value`.
func (o *IntOptions) set(name string, value int) {
	o.enable(name)
	o.Opts[name] = value
}

// disable disables the option `name`. All options which depend on the option
// to be disabled will be disabled. Options which have previously been enabled
// as a dependency will not be automatically disabled.
func (o *IntOptions) disable(name string) {
	o.Opts[name] = OptionDisabled

	if o.Library != nil {
		// Disable all options which have a dependency on the option
		// that was just disabled
		for key, opt := range *o.Library {
			if opt.RequiresOption(name) && o.Opts[key] != OptionDisabled {
				o.disable(key)
			}
		}
	}
}

type changedOptions struct {
	key   string
	value int
}

// ApplyValidated takes a configuration map and applies the changes. For an
// option which is changed, the `ChangedFunc` function is called with the
// `data` argument passed in as well. Returns the number of options changed if
// any.
//
// The caller is expected to have validated the configuration options prior to
// calling this function.
func (o *IntOptions) ApplyValidated(n models.ConfigurationMap, changed ChangedFunc, data interface{}) int {
	changes := []changedOptions{}

	o.optsMU.Lock()
	for k, v := range n {
		val, ok := o.Opts[k]

		// Ignore the error here because the option was already validated.
		_, optVal, _ := ParseKeyValue(o.Library, k, v, OptionDisabled)
		if optVal == OptionDisabled {
			/* Only disable if enabled already */
			if ok && val != OptionDisabled {
				o.disable(k)
				changes = append(changes, changedOptions{key: k, value: optVal})
			}
		} else {
			/* Only enable if not enabled already */
			if !ok || val == OptionDisabled {
				o.set(k, optVal)
				changes = append(changes, changedOptions{key: k, value: optVal})
			}
		}
	}
	o.optsMU.Unlock()

	for _, change := range changes {
		changed(change.key, change.value, data)
	}

	return len(changes)
}
<|endoftext|>


<commit_before>package server

import (
	"os"
	"net"

	"junta/util"
	"junta/paxos"
	"junta/proto"
	"junta/store"
	"strconv"
	"time"
)

type ReadFromWriteToer interface {
	ReadFrom([]byte) (int, net.Addr, os.Error)
	WriteTo([]byte, net.Addr) (int, os.Error)
	LocalAddr() net.Addr
}

const packetSize = 3000

type conn struct {
	net.Conn
	s *Server
}

type Manager interface {
	PutFrom(string, paxos.Msg)
	Propose(string) (uint64, string, os.Error)
	Alpha() int
}

type Server struct {
	Addr string
	St *store.Store
	Mg Manager
	Self string
}

func (sv *Server) ListenAndServe() os.Error {
	logger := util.NewLogger("server %s", sv.Addr)

	logger.Log("binding")
	l, err := net.Listen("tcp", sv.Addr)
	if err != nil {
		logger.Log(err)
		return err
	}
	defer l.Close()
	logger.Log("listening")

	err = sv.Serve(l)
	if err != nil {
		logger.Logf("%s: %s", l, err)
}
	return err
}

func (sv *Server) ListenAndServeUdp(outs chan paxos.Packet) os.Error {
	logger := util.NewLogger("udp server %s", sv.Addr)

	logger.Log("binding")
	u, err := net.ListenPacket("udp", sv.Addr)
	if err != nil {
		logger.Log(err)
		return err
	}
	defer u.Close()
	logger.Log("listening")

	err = sv.ServeUdp(u, outs)
	if err != nil {
		logger.Logf("%s: %s", u, err)
	}
	return err
}

func (sv *Server) ServeUdp(u ReadFromWriteToer, outs chan paxos.Packet) os.Error {
	recvd := make(chan paxos.Packet)
	sent := make(chan paxos.Packet)

	logger := util.NewLogger("udp server %s", u.LocalAddr())
	go func() {
		logger.Log("reading messages...")
		for {
			msg, addr, err := paxos.ReadMsg(u, packetSize)
			if err != nil {
				logger.Log(err)
				continue
			}
			logger.Logf("read %v from %s", msg, addr)
			recvd <- paxos.Packet{msg, addr}
			sv.Mg.PutFrom(addr, msg)
		}
	}()

	go func() {
		logger.Log("sending messages...")
		for pk := range outs {
			logger.Logf("sending %v", pk)
			udpAddr, err := net.ResolveUDPAddr(pk.Addr)
			if err != nil {
				logger.Log(err)
				continue
			}

			_, err = u.WriteTo(pk.Msg.WireBytes(), udpAddr)
			if err != nil {
				logger.Log(err)
				continue
			}
			sent <- paxos.Packet{pk.Msg, pk.Addr}
		}
	}()

	needsAck := make(map[string]bool)
	resend := make(chan paxos.Packet)
	for {
		select {
		case pk := <-recvd:
			if pk.Msg.HasFlags(paxos.Ack) {
				logger.Logf("got ack %s %v", pk.Addr, pk.Msg)
				needsAck[pk.Id()] = false
			} else {
				logger.Logf("sending ack %s %v", pk.Addr, pk.Msg)
				udpAddr, err := net.ResolveUDPAddr(pk.Addr)
				if err != nil {
					break
				}
				ack := pk.Msg.Dup().SetFlags(paxos.Ack)
				u.WriteTo(ack.WireBytes(), udpAddr)
			}
		case pk := <-sent:
			needsAck[pk.Id()] = true
			logger.Logf("needs ack %s %v", pk.Addr, pk.Msg)
			go func() {
				time.Sleep(100000000) // ns == 0.1s
				resend <- pk
			}()
		case pk := <-resend:
			if needsAck[pk.Id()] {
				logger.Logf("resending %s %v", pk.Addr, pk.Msg)
				go func() {
					outs <- pk
				}()
			} else {
				needsAck[pk.Id()] = false, false
			}
		}
	}

	panic("not reached")
}

func (s *Server) Serve(l net.Listener) os.Error {
	for {
		rw, e := l.Accept()
		if e != nil {
			return e
		}
		c := &conn{rw, s}
		go c.serve()
	}

	panic("not reached")
}

func (sv *Server) leader() string {
	parts, cas := sv.St.Lookup("/j/junta/leader")
	if cas == store.Dir && cas == store.Missing {
		return ""
	}
	return parts[0]
}

func (sv *Server) setOnce(path, body, cas string) (uint64, os.Error) {
	mut, err := store.EncodeSet(path, body, cas)
	if err != nil {
		return 0, err
	}

	seqn, v, err := sv.Mg.Propose(mut)
	if err != nil {
		return 0, err
	}

	// We failed, but only because of a competing proposal. The client should
	// retry.
	if v != mut {
		return 0, os.EAGAIN
	}

	return seqn, nil
}

func (sv *Server) Set(path, body, cas string) (seqn uint64, err os.Error) {
	err = os.EAGAIN
	for err == os.EAGAIN {
		seqn, err = sv.setOnce(path, body, cas)
	}
	return
}

func (sv *Server) delOnce(path, cas string) (uint64, os.Error) {
	mut, err := store.EncodeDel(path, cas)
	if err != nil {
		return 0, err
	}

	seqn, v, err := sv.Mg.Propose(mut)
	if err != nil {
		return 0, err
	}

	// We failed, but only because of a competing proposal. The client should
	// retry.
	if v != mut {
		return 0, os.EAGAIN
	}

	return seqn, nil
}

func (sv *Server) Del(path, cas string) (seqn uint64, err os.Error) {
	err = os.EAGAIN
	for err == os.EAGAIN {
		seqn, err = sv.delOnce(path, cas)
	}
	return
}

func (sv *Server) WaitForPathSet(path string) (body string, err os.Error) {
	evs := make(chan store.Event)
	defer close(evs)
	sv.St.Watch(path, evs)

	parts, cas := sv.St.Lookup(path)
	if cas != store.Dir && cas != store.Missing {
		return parts[0], nil
	}

	for ev := range evs {
		if ev.IsSet() {
			return ev.Body, nil
		}
	}

	panic("not reached")
}

// Repeatedly propose nop values until a successful read from `done`.
func (sv *Server) AdvanceUntil(done chan int) {
	for _, ok := <-done; !ok; _, ok = <-done {
		sv.Mg.Propose(store.Nop)
	}
}

func (c *conn) serve() {
	pc := proto.NewConn(c)
	logger := util.NewLogger("%v", c.RemoteAddr())
	logger.Log("accepted connection")
	for {
		rid, parts, err := pc.ReadRequest()
		if err != nil {
			if err == os.EOF {
				logger.Log("connection closed by peer")
			} else {
				logger.Log(err)
			}
			return
		}

		rlogger := util.NewLogger("%v - req [%d]", c.RemoteAddr(), rid)
		rlogger.Logf("received <%v>", parts)

		if len(parts) == 0 {
			rlogger.Log("zero parts supplied")
			pc.SendError(rid, proto.InvalidCommand + ": no command")
			continue
		}

		switch parts[0] {
		default:
			rlogger.Logf("unknown command <%s>", parts[0])
			pc.SendError(rid, proto.InvalidCommand + " " + parts[0])
		case "set":
			if len(parts) != 4 {
				rlogger.Logf("invalid set command: %#v", parts)
				pc.SendError(rid, "wrong number of parts")
				break
			}

			leader := c.s.leader()
			if c.s.Self == leader {
				rlogger.Logf("set %q=%q (cas %q)", parts[1], parts[2], parts[3])
				seqn, err := c.s.Set(parts[1], parts[2], parts[3])
				if err != nil {
					rlogger.Logf("bad: %s", err)
					pc.SendError(rid, err.String())
				} else {
					rlogger.Logf("good")
					pc.SendResponse(rid, strconv.Uitoa64(seqn))
				}
			} else {
				rlogger.Logf("redirect to %s", leader)
				pc.SendRedirect(rid, leader)
			}
		case "del":
			if len(parts) != 3 {
				rlogger.Logf("invalid del command: %v", parts)
				pc.SendError(rid, "wrong number of parts")
				break
			}
			rlogger.Logf("del %q (cas %q)", parts[1], parts[2])
			_, err := c.s.Del(parts[1], parts[2])
			if err != nil {
				rlogger.Logf("bad: %s", err)
				pc.SendError(rid, err.String())
			} else {
				rlogger.Logf("good")
				pc.SendResponse(rid, "true")
			}
		case "wait-for-path-set": // TODO this is for demo purposes only
			if len(parts) != 2 {
				rlogger.Logf("invalid wait-for-path-set command: %v", parts)
				pc.SendError(rid, "wrong number of parts")
				break
			}
			rlogger.Logf("wait-for-path-set %q", parts[1])
			body, err := c.s.WaitForPathSet(parts[1])
			if err != nil {
				rlogger.Logf("bad: %s", err)
				pc.SendError(rid, err.String())
			} else {
				rlogger.Logf("good %q", body)
				pc.SendResponse(rid, body)
			}
		case "join":
			// join abc123 1.2.3.4:999
			if len(parts) != 3 {
				rlogger.Logf("invalid join command: %v", parts)
				pc.SendError(rid, "wrong number of parts")
				break
			}

			who, addr := parts[1], parts[2]
			rlogger.Logf("membership requested for %s at %s", who, addr)

			key := "/j/junta/members/" + who

			seqn, err := c.s.Set(key, addr, store.Missing)
			if err != nil {
				rlogger.Logf("bad: %s", err)
				pc.SendError(rid, err.String())
			} else {
				rlogger.Logf("good")
				done := make(chan int)
				go c.s.AdvanceUntil(done)
				c.s.St.Sync(seqn + uint64(c.s.Mg.Alpha()))
				close(done)
				seqn, snap := c.s.St.Snapshot()
				pc.SendResponse(rid, strconv.Uitoa64(seqn), snap)
			}
		}
	}
}
<commit_msg>refactor<commit_after>package server

import (
	"os"
	"net"

	"junta/util"
	"junta/paxos"
	"junta/proto"
	"junta/store"
	"strconv"
	"time"
)

type ReadFromWriteToer interface {
	ReadFrom([]byte) (int, net.Addr, os.Error)
	WriteTo([]byte, net.Addr) (int, os.Error)
	LocalAddr() net.Addr
}

const packetSize = 3000

type conn struct {
	net.Conn
	s *Server
}

type Manager interface {
	PutFrom(string, paxos.Msg)
	Propose(string) (uint64, string, os.Error)
	Alpha() int
}

type Server struct {
	Addr string
	St *store.Store
	Mg Manager
	Self string
}

func (sv *Server) ListenAndServe() os.Error {
	logger := util.NewLogger("server %s", sv.Addr)

	logger.Log("binding")
	l, err := net.Listen("tcp", sv.Addr)
	if err != nil {
		logger.Log(err)
		return err
	}
	defer l.Close()
	logger.Log("listening")

	err = sv.Serve(l)
	if err != nil {
		logger.Logf("%s: %s", l, err)
}
	return err
}

func (sv *Server) ListenAndServeUdp(outs chan paxos.Packet) os.Error {
	logger := util.NewLogger("udp server %s", sv.Addr)

	logger.Log("binding")
	u, err := net.ListenPacket("udp", sv.Addr)
	if err != nil {
		logger.Log(err)
		return err
	}
	defer u.Close()
	logger.Log("listening")

	err = sv.ServeUdp(u, outs)
	if err != nil {
		logger.Logf("%s: %s", u, err)
	}
	return err
}

func (sv *Server) ServeUdp(u ReadFromWriteToer, outs chan paxos.Packet) os.Error {
	recvd := make(chan paxos.Packet)
	sent := make(chan paxos.Packet)

	logger := util.NewLogger("udp server %s", u.LocalAddr())
	go func() {
		logger.Log("reading messages...")
		for {
			msg, addr, err := paxos.ReadMsg(u, packetSize)
			if err != nil {
				logger.Log(err)
				continue
			}
			logger.Logf("read %v from %s", msg, addr)
			recvd <- paxos.Packet{msg, addr}
			sv.Mg.PutFrom(addr, msg)
		}
	}()

	go func() {
		logger.Log("sending messages...")
		for pk := range outs {
			logger.Logf("sending %v", pk)
			udpAddr, err := net.ResolveUDPAddr(pk.Addr)
			if err != nil {
				logger.Log(err)
				continue
			}

			_, err = u.WriteTo(pk.Msg.WireBytes(), udpAddr)
			if err != nil {
				logger.Log(err)
				continue
			}
			sent <- paxos.Packet{pk.Msg, pk.Addr}
		}
	}()

	needsAck := make(map[string]bool)
	resend := make(chan paxos.Packet)
	for {
		select {
		case pk := <-recvd:
			if pk.Msg.HasFlags(paxos.Ack) {
				logger.Logf("got ack %s %v", pk.Addr, pk.Msg)
				needsAck[pk.Id()] = false
			} else {
				logger.Logf("sending ack %s %v", pk.Addr, pk.Msg)
				udpAddr, err := net.ResolveUDPAddr(pk.Addr)
				if err != nil {
					break
				}
				ack := pk.Msg.Dup().SetFlags(paxos.Ack)
				u.WriteTo(ack.WireBytes(), udpAddr)
			}
		case pk := <-sent:
			needsAck[pk.Id()] = true
			logger.Logf("needs ack %s %v", pk.Addr, pk.Msg)
			go func() {
				time.Sleep(100000000) // ns == 0.1s
				resend <- pk
			}()
		case pk := <-resend:
			if needsAck[pk.Id()] {
				logger.Logf("resending %s %v", pk.Addr, pk.Msg)
				go func() {
					outs <- pk
				}()
			} else {
				needsAck[pk.Id()] = false, false
			}
		}
	}

	panic("not reached")
}

func (s *Server) Serve(l net.Listener) os.Error {
	for {
		rw, e := l.Accept()
		if e != nil {
			return e
		}
		c := &conn{rw, s}
		go c.serve()
	}

	panic("not reached")
}

func (sv *Server) leader() string {
	parts, cas := sv.St.Lookup("/j/junta/leader")
	if cas == store.Dir && cas == store.Missing {
		return ""
	}
	return parts[0]
}

func (sv *Server) setOnce(path, body, cas string) (uint64, os.Error) {
	mut, err := store.EncodeSet(path, body, cas)
	if err != nil {
		return 0, err
	}

	seqn, v, err := sv.Mg.Propose(mut)
	if err != nil {
		return 0, err
	}

	// We failed, but only because of a competing proposal. The client should
	// retry.
	if v != mut {
		return 0, os.EAGAIN
	}

	return seqn, nil
}

func (sv *Server) Set(path, body, cas string) (seqn uint64, err os.Error) {
	err = os.EAGAIN
	for err == os.EAGAIN {
		seqn, err = sv.setOnce(path, body, cas)
	}
	return
}

func (sv *Server) delOnce(path, cas string) (uint64, os.Error) {
	mut, err := store.EncodeDel(path, cas)
	if err != nil {
		return 0, err
	}

	seqn, v, err := sv.Mg.Propose(mut)
	if err != nil {
		return 0, err
	}

	// We failed, but only because of a competing proposal. The client should
	// retry.
	if v != mut {
		return 0, os.EAGAIN
	}

	return seqn, nil
}

func (sv *Server) Del(path, cas string) (seqn uint64, err os.Error) {
	err = os.EAGAIN
	for err == os.EAGAIN {
		seqn, err = sv.delOnce(path, cas)
	}
	return
}

func (sv *Server) WaitForPathSet(path string) (body string, err os.Error) {
	evs := make(chan store.Event)
	defer close(evs)
	sv.St.Watch(path, evs)

	parts, cas := sv.St.Lookup(path)
	if cas != store.Dir && cas != store.Missing {
		return parts[0], nil
	}

	for ev := range evs {
		if ev.IsSet() {
			return ev.Body, nil
		}
	}

	panic("not reached")
}

// Repeatedly propose nop values until a successful read from `done`.
func (sv *Server) AdvanceUntil(done chan int) {
	for _, ok := <-done; !ok; _, ok = <-done {
		sv.Mg.Propose(store.Nop)
	}
}

func (c *conn) serve() {
	pc := proto.NewConn(c)
	logger := util.NewLogger("%v", c.RemoteAddr())
	logger.Log("accepted connection")
	for {
		rid, parts, err := pc.ReadRequest()
		if err != nil {
			if err == os.EOF {
				logger.Log("connection closed by peer")
			} else {
				logger.Log(err)
			}
			return
		}

		rlogger := util.NewLogger("%v - req [%d]", c.RemoteAddr(), rid)
		rlogger.Logf("received <%v>", parts)

		if len(parts) == 0 {
			rlogger.Log("zero parts supplied")
			pc.SendError(rid, proto.InvalidCommand + ": no command")
			continue
		}

		switch parts[0] {
		default:
			rlogger.Logf("unknown command <%s>", parts[0])
			pc.SendError(rid, proto.InvalidCommand + " " + parts[0])
		case "set":
			if len(parts) != 4 {
				rlogger.Logf("invalid set command: %#v", parts)
				pc.SendError(rid, "wrong number of parts")
				break
			}

			leader := c.s.leader()
			if c.s.Self != leader {
				rlogger.Logf("redirect to %s", leader)
				pc.SendRedirect(rid, leader)
				break
			}

			rlogger.Logf("set %q=%q (cas %q)", parts[1], parts[2], parts[3])
			seqn, err := c.s.Set(parts[1], parts[2], parts[3])
			if err != nil {
				rlogger.Logf("bad: %s", err)
				pc.SendError(rid, err.String())
			} else {
				rlogger.Logf("good")
				pc.SendResponse(rid, strconv.Uitoa64(seqn))
			}
		case "del":
			if len(parts) != 3 {
				rlogger.Logf("invalid del command: %v", parts)
				pc.SendError(rid, "wrong number of parts")
				break
			}
			rlogger.Logf("del %q (cas %q)", parts[1], parts[2])
			_, err := c.s.Del(parts[1], parts[2])
			if err != nil {
				rlogger.Logf("bad: %s", err)
				pc.SendError(rid, err.String())
			} else {
				rlogger.Logf("good")
				pc.SendResponse(rid, "true")
			}
		case "wait-for-path-set": // TODO this is for demo purposes only
			if len(parts) != 2 {
				rlogger.Logf("invalid wait-for-path-set command: %v", parts)
				pc.SendError(rid, "wrong number of parts")
				break
			}
			rlogger.Logf("wait-for-path-set %q", parts[1])
			body, err := c.s.WaitForPathSet(parts[1])
			if err != nil {
				rlogger.Logf("bad: %s", err)
				pc.SendError(rid, err.String())
			} else {
				rlogger.Logf("good %q", body)
				pc.SendResponse(rid, body)
			}
		case "join":
			// join abc123 1.2.3.4:999
			if len(parts) != 3 {
				rlogger.Logf("invalid join command: %v", parts)
				pc.SendError(rid, "wrong number of parts")
				break
			}

			who, addr := parts[1], parts[2]
			rlogger.Logf("membership requested for %s at %s", who, addr)

			key := "/j/junta/members/" + who

			seqn, err := c.s.Set(key, addr, store.Missing)
			if err != nil {
				rlogger.Logf("bad: %s", err)
				pc.SendError(rid, err.String())
			} else {
				rlogger.Logf("good")
				done := make(chan int)
				go c.s.AdvanceUntil(done)
				c.s.St.Sync(seqn + uint64(c.s.Mg.Alpha()))
				close(done)
				seqn, snap := c.s.St.Snapshot()
				pc.SendResponse(rid, strconv.Uitoa64(seqn), snap)
			}
		}
	}
}
<|endoftext|>


<commit_before>package sharing

import (
	"os"
	"sort"
	"strings"
	"time"

	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/crypto"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/vfs"
)

// MakeXorKey generates a key for transforming the file identifiers
func MakeXorKey() []byte {
	random := crypto.GenerateRandomBytes(8)
	result := make([]byte, 2*len(random))
	for i, val := range random {
		result[2*i] = val & 0xf
		result[2*i+1] = val >> 4
	}
	return result
}

// XorID transforms the identifier of a file to a new identifier, in a
// reversible way: it makes a XOR on the hexadecimal characters
func XorID(id string, key []byte) string {
	l := len(key)
	buf := []byte(id)
	for i, c := range buf {
		switch {
		case '0' <= c && c <= '9':
			c = (c - '0') ^ key[i%l]
		case 'a' <= c && c <= 'f':
			c = (c - 'a' + 10) ^ key[i%l]
		case 'A' <= c && c <= 'F':
			c = (c - 'A' + 10) ^ key[i%l]
		default:
			continue
		}
		if c < 10 {
			buf[i] = c + '0'
		} else {
			buf[i] = (c - 10) + 'a'
		}
	}
	return string(buf)
}

// SortFilesToSent sorts the files slice that will be sent in bulk_docs:
// - directories must come before files (if a file is created in a new
//   directory, we must create directory before the file)
// - directories are sorted by increasing depth (if a sub-folder is created
//   in a new directory, we must create the parent before the child)
// TODO trashed / deleted files and folders
func (s *Sharing) SortFilesToSent(files []map[string]interface{}) {
	sort.SliceStable(files, func(i, j int) bool {
		a, b := files[i], files[j]
		if a["type"] == "file" {
			return false
		}
		if b["type"] == "file" {
			return true
		}
		p, ok := a["path"].(string)
		if !ok {
			return true
		}
		q, ok := b["path"].(string)
		if !ok {
			return false
		}
		return strings.Count(p, "/") < strings.Count(q, "/")
	})
}

// TransformFileToSent transforms an io.cozy.files document before sending it
// to another cozy instance:
// - its identifier is XORed
// - its dir_id is XORed or removed
// - the path is removed (directory only)
//
// ruleIndexes is a map of "doctype-docid" -> rule index
// TODO keep referenced_by that are not relevant to this sharing
// TODO the file/folder has been moved outside the shared directory
func (s *Sharing) TransformFileToSent(doc map[string]interface{}, xorKey []byte, ruleIndexes map[string]int) map[string]interface{} {
	if doc["type"] == "directory" {
		delete(doc, "path")
	}
	id, ok := doc["_id"].(string)
	if !ok {
		return doc
	}
	doc["_id"] = XorID(id, xorKey)
	dir, ok := doc["dir_id"].(string)
	if !ok {
		return doc
	}
	delete(doc, "referenced_by")
	rule := s.Rules[ruleIndexes[id]]
	noDirID := rule.Selector == "referenced_by"
	if !noDirID {
		for _, v := range rule.Values {
			if v == dir {
				noDirID = true
				break
			}
		}
	}
	if noDirID {
		delete(doc, "dir_id")
	} else {
		doc["dir_id"] = XorID(dir, xorKey)
	}
	return doc
}

// EnsureSharedWithMeDir returns the shared-with-me directory, and create it if
// it doesn't exist
func EnsureSharedWithMeDir(inst *instance.Instance) (*vfs.DirDoc, error) {
	fs := inst.VFS()
	dir, _, err := fs.DirOrFileByID(consts.SharedWithMeDirID)
	if err != nil && !couchdb.IsNotFoundError(err) {
		return nil, err
	}

	if dir == nil {
		name := inst.Translate("Tree Shared with me")
		dir, err = vfs.NewDirDocWithPath(name, consts.RootDirID, "/", nil)
		dir.DocID = consts.SharedWithMeDirID
		if err != nil {
			return nil, err
		}
		if err = fs.CreateDir(dir); err != nil {
			return nil, err
		}
		return dir, nil
	}

	if dir.RestorePath != "" {
		_, err = vfs.RestoreDir(fs, dir)
		if err != nil {
			return nil, err
		}
		children, err := fs.DirBatch(dir, &couchdb.SkipCursor{})
		if err != nil {
			return nil, err
		}
		for _, child := range children {
			d, f := child.Refine()
			if d != nil {
				_, err = vfs.TrashDir(fs, d)
			} else {
				_, err = vfs.TrashFile(fs, f)
			}
			if err != nil {
				return nil, err
			}
		}
	}

	return dir, nil
}

// CreateDirForSharing creates the directory where files for this sharing will
// be put. This directory will be initially inside the Shared with me folder.
func (s *Sharing) CreateDirForSharing(inst *instance.Instance, rule *Rule) error {
	parent, err := EnsureSharedWithMeDir(inst)
	if err != nil {
		return err
	}
	fs := inst.VFS()
	dir, err := vfs.NewDirDocWithParent(rule.Title, parent, []string{"from-sharing-" + s.SID})
	if err != nil {
		return err
	}
	dir.AddReferencedBy(couchdb.DocReference{
		ID:   s.SID,
		Type: consts.Sharings,
	})
	return fs.CreateDir(dir)
}

// GetSharingDir returns the directory used by this sharing for putting files
// and folders that have no dir_id.
func (s *Sharing) GetSharingDir(inst *instance.Instance) (*vfs.DirDoc, error) {
	key := []string{consts.Sharings, s.SID}
	end := []string{key[0], key[1], couchdb.MaxString}
	req := &couchdb.ViewRequest{
		StartKey:    key,
		EndKey:      end,
		IncludeDocs: true,
	}
	var res couchdb.ViewResponse
	err := couchdb.ExecView(inst, consts.FilesReferencedByView, req, &res)
	if err != nil || len(res.Rows) == 0 {
		inst.Logger().WithField("nspace", "sharing").Warnf("Sharing dir not found: %v (%s)", err, s.SID)
		return nil, ErrInternalServerError
	}
	return inst.VFS().DirByID(res.Rows[0].ID)
}

// ApplyBulkFiles takes a list of documents for the io.cozy.files doctype and
// will apply changes to the VFS according to those documents.
func (s *Sharing) ApplyBulkFiles(inst *instance.Instance, docs DocsList) error {
	fs := inst.VFS()
	for _, target := range docs {
		id, ok := target["_id"].(string)
		if !ok {
			return ErrMissingID
		}
		var ref *SharedRef
		err := couchdb.GetDoc(inst, consts.Shared, consts.Files+"/"+id, ref)
		if err != nil && !couchdb.IsNotFoundError(err) {
			inst.Logger().WithField("nspace", "replicator").Debugf("Error on finding doc of bulk files: %s", err)
			return err
		}
		// TODO it's only for directory currently, code needs to be adapted for files
		doc, err := fs.DirByID(id) // TODO DirOrFileByID
		if err != nil && err != os.ErrNotExist {
			inst.Logger().WithField("nspace", "replicator").Debugf("Error on finding ref of bulk files: %s", err)
			return err
		}
		if ref == nil && doc == nil {
			err = s.CreateDir(inst, target)
			if err != nil {
				return err
			}
			// TODO update the io.cozy.shared reference?
		} else if ref == nil {
			// TODO be safe => return an error
			continue
		} else if doc == nil {
			// TODO manage the conflict: doc was deleted/moved outside the
			// sharing on this cozy and updated on the other cozy
			continue
		} else {
			// TODO update the directory
			err = s.UpdateDir(inst, target, doc)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func copyTagsAndDatesToDir(target map[string]interface{}, dir *vfs.DirDoc) {
	if tags, ok := target["tags"].([]interface{}); ok {
		dir.Tags = make([]string, 0, len(tags))
		for _, tag := range tags {
			if t, ok := tag.(string); ok {
				dir.Tags = append(dir.Tags, t)
			}
		}
	}
	if created, ok := target["created_at"].(string); ok {
		if at, err := time.Parse(time.RFC3339Nano, created); err == nil {
			dir.CreatedAt = at
		}
	}
	if updated, ok := target["updated_at"].(string); ok {
		if at, err := time.Parse(time.RFC3339Nano, updated); err == nil {
			dir.UpdatedAt = at
		}
	}
}

// CreateDir creates a directory on this cozy to reflect a change on another
// cozy instance of this sharing.
func (s *Sharing) CreateDir(inst *instance.Instance, target map[string]interface{}) error {
	name, ok := target["name"].(string)
	if !ok {
		inst.Logger().WithField("nspace", "replicator").Debugf("Missing name for creating dir: %#v", target)
		return ErrInternalServerError
	}
	rev, ok := target["_rev"].(string)
	if !ok {
		inst.Logger().WithField("nspace", "replicator").Debugf("Missing _rev for creating dir: %#v", target)
		return ErrInternalServerError
	}
	revisions, ok := target["_revisions"].(map[string]interface{})
	if !ok {
		inst.Logger().WithField("nspace", "replicator").Debugf("Missing _revisions for creating dir: %#v", target)
		return ErrInternalServerError
	}
	indexer := NewSharingIndexer(inst, &bulkRevs{
		Rev:       rev,
		Revisions: revisions,
	})
	fs := inst.VFS().UseSharingIndexer(indexer)

	var parent *vfs.DirDoc
	var err error
	if dirID, ok := target["dir_id"].(string); ok {
		parent, err = fs.DirByID(dirID)
		// TODO better handling of this conflict
		if err != nil {
			inst.Logger().WithField("nspace", "replicator").Debugf("Conflict for parent on creating dir: %s", err)
			return err
		}
	} else {
		parent, err = s.GetSharingDir(inst)
		if err != nil {
			return err
		}
	}

	dir, err := vfs.NewDirDocWithParent(name, parent, nil)
	if err != nil {
		inst.Logger().WithField("nspace", "replicator").Warnf("Cannot initialize dir doc: %s", err)
		return err
	}
	dir.SetID(target["_id"].(string))
	copyTagsAndDatesToDir(target, dir)
	// TODO referenced_by
	// TODO manage conflicts
	if err := fs.CreateDir(dir); err != nil {
		inst.Logger().WithField("nspace", "replicator").Debugf("Cannot create dir: %s", err)
		return err
	}
	return nil
}

// UpdateDir updates a directory on this cozy to reflect a change on another
// cozy instance of this sharing.
func (s *Sharing) UpdateDir(inst *instance.Instance, target map[string]interface{}, dir *vfs.DirDoc) error {
	rev, ok := target["_rev"].(string)
	if !ok {
		// TODO add logs or better error
		return ErrInternalServerError
	}
	revisions, ok := target["_revisions"].(map[string]interface{})
	if !ok {
		return ErrInternalServerError
	}
	indexer := NewSharingIndexer(inst, &bulkRevs{
		Rev:       rev,
		Revisions: revisions,
	})
	copyTagsAndDatesToDir(target, dir)
	// TODO what if name or dir_id has changed
	// TODO referenced_by
	// TODO trash
	// TODO manage conflicts
	return indexer.UpdateDirDoc(nil, dir) // TODO oldDoc
}

// TODO referenced_by
func dirToJSONDoc(dir *vfs.DirDoc) couchdb.JSONDoc {
	doc := couchdb.JSONDoc{
		Type: consts.Files,
		M: map[string]interface{}{
			"type":       dir.Type,
			"_id":        dir.DocID,
			"_rev":       dir.DocRev,
			"name":       dir.DocName,
			"created_at": dir.CreatedAt,
			"updated_at": dir.UpdatedAt,
			"tags":       dir.Tags,
			"path":       dir.Fullpath,
		},
	}
	if dir.DirID != "" {
		doc.M["dir_id"] = dir.DirID
	}
	if dir.RestorePath != "" {
		doc.M["restore_path"] = dir.RestorePath
	}
	return doc
}

// TODO referenced_by
func fileToJSONDoc(file *vfs.FileDoc) couchdb.JSONDoc {
	doc := couchdb.JSONDoc{
		Type: consts.Files,
		M: map[string]interface{}{
			"type":       file.Type,
			"_id":        file.DocID,
			"_rev":       file.DocRev,
			"name":       file.DocName,
			"created_at": file.CreatedAt,
			"updated_at": file.UpdatedAt,
			"size":       file.ByteSize,
			"md5Sum":     file.MD5Sum,
			"mime":       file.Mime,
			"class":      file.Class,
			"executable": file.Executable,
			"trashed":    file.Trashed,
			"tags":       file.Tags,
		},
	}
	if file.DirID != "" {
		doc.M["dir_id"] = file.DirID
	}
	if file.RestorePath != "" {
		doc.M["restore_path"] = file.RestorePath
	}
	if len(file.Metadata) > 0 {
		doc.M["metadata"] = file.Metadata
	}
	return doc
}
<commit_msg>Do not stop on first error while replicating files/folders<commit_after>package sharing

import (
	"os"
	"sort"
	"strings"
	"time"

	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/crypto"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/vfs"
	multierror "github.com/hashicorp/go-multierror"
)

// MakeXorKey generates a key for transforming the file identifiers
func MakeXorKey() []byte {
	random := crypto.GenerateRandomBytes(8)
	result := make([]byte, 2*len(random))
	for i, val := range random {
		result[2*i] = val & 0xf
		result[2*i+1] = val >> 4
	}
	return result
}

// XorID transforms the identifier of a file to a new identifier, in a
// reversible way: it makes a XOR on the hexadecimal characters
func XorID(id string, key []byte) string {
	l := len(key)
	buf := []byte(id)
	for i, c := range buf {
		switch {
		case '0' <= c && c <= '9':
			c = (c - '0') ^ key[i%l]
		case 'a' <= c && c <= 'f':
			c = (c - 'a' + 10) ^ key[i%l]
		case 'A' <= c && c <= 'F':
			c = (c - 'A' + 10) ^ key[i%l]
		default:
			continue
		}
		if c < 10 {
			buf[i] = c + '0'
		} else {
			buf[i] = (c - 10) + 'a'
		}
	}
	return string(buf)
}

// SortFilesToSent sorts the files slice that will be sent in bulk_docs:
// - directories must come before files (if a file is created in a new
//   directory, we must create directory before the file)
// - directories are sorted by increasing depth (if a sub-folder is created
//   in a new directory, we must create the parent before the child)
// TODO trashed / deleted files and folders
func (s *Sharing) SortFilesToSent(files []map[string]interface{}) {
	sort.SliceStable(files, func(i, j int) bool {
		a, b := files[i], files[j]
		if a["type"] == "file" {
			return false
		}
		if b["type"] == "file" {
			return true
		}
		p, ok := a["path"].(string)
		if !ok {
			return true
		}
		q, ok := b["path"].(string)
		if !ok {
			return false
		}
		return strings.Count(p, "/") < strings.Count(q, "/")
	})
}

// TransformFileToSent transforms an io.cozy.files document before sending it
// to another cozy instance:
// - its identifier is XORed
// - its dir_id is XORed or removed
// - the path is removed (directory only)
//
// ruleIndexes is a map of "doctype-docid" -> rule index
// TODO keep referenced_by that are relevant to this sharing
// TODO the file/folder has been moved outside the shared directory
func (s *Sharing) TransformFileToSent(doc map[string]interface{}, xorKey []byte, ruleIndexes map[string]int) map[string]interface{} {
	if doc["type"] == "directory" {
		delete(doc, "path")
	}
	id, ok := doc["_id"].(string)
	if !ok {
		return doc
	}
	doc["_id"] = XorID(id, xorKey)
	dir, ok := doc["dir_id"].(string)
	if !ok {
		return doc
	}
	delete(doc, "referenced_by")
	rule := s.Rules[ruleIndexes[id]]
	noDirID := rule.Selector == "referenced_by"
	if !noDirID {
		for _, v := range rule.Values {
			if v == dir {
				noDirID = true
				break
			}
		}
	}
	if noDirID {
		delete(doc, "dir_id")
	} else {
		doc["dir_id"] = XorID(dir, xorKey)
	}
	return doc
}

// EnsureSharedWithMeDir returns the shared-with-me directory, and create it if
// it doesn't exist
func EnsureSharedWithMeDir(inst *instance.Instance) (*vfs.DirDoc, error) {
	fs := inst.VFS()
	dir, _, err := fs.DirOrFileByID(consts.SharedWithMeDirID)
	if err != nil && !couchdb.IsNotFoundError(err) {
		return nil, err
	}

	if dir == nil {
		name := inst.Translate("Tree Shared with me")
		dir, err = vfs.NewDirDocWithPath(name, consts.RootDirID, "/", nil)
		dir.DocID = consts.SharedWithMeDirID
		if err != nil {
			return nil, err
		}
		if err = fs.CreateDir(dir); err != nil {
			return nil, err
		}
		return dir, nil
	}

	if dir.RestorePath != "" {
		_, err = vfs.RestoreDir(fs, dir)
		if err != nil {
			return nil, err
		}
		children, err := fs.DirBatch(dir, &couchdb.SkipCursor{})
		if err != nil {
			return nil, err
		}
		for _, child := range children {
			d, f := child.Refine()
			if d != nil {
				_, err = vfs.TrashDir(fs, d)
			} else {
				_, err = vfs.TrashFile(fs, f)
			}
			if err != nil {
				return nil, err
			}
		}
	}

	return dir, nil
}

// CreateDirForSharing creates the directory where files for this sharing will
// be put. This directory will be initially inside the Shared with me folder.
func (s *Sharing) CreateDirForSharing(inst *instance.Instance, rule *Rule) error {
	parent, err := EnsureSharedWithMeDir(inst)
	if err != nil {
		return err
	}
	fs := inst.VFS()
	dir, err := vfs.NewDirDocWithParent(rule.Title, parent, []string{"from-sharing-" + s.SID})
	if err != nil {
		return err
	}
	dir.AddReferencedBy(couchdb.DocReference{
		ID:   s.SID,
		Type: consts.Sharings,
	})
	return fs.CreateDir(dir)
}

// GetSharingDir returns the directory used by this sharing for putting files
// and folders that have no dir_id.
func (s *Sharing) GetSharingDir(inst *instance.Instance) (*vfs.DirDoc, error) {
	key := []string{consts.Sharings, s.SID}
	end := []string{key[0], key[1], couchdb.MaxString}
	req := &couchdb.ViewRequest{
		StartKey:    key,
		EndKey:      end,
		IncludeDocs: true,
	}
	var res couchdb.ViewResponse
	err := couchdb.ExecView(inst, consts.FilesReferencedByView, req, &res)
	if err != nil || len(res.Rows) == 0 {
		inst.Logger().WithField("nspace", "sharing").Warnf("Sharing dir not found: %v (%s)", err, s.SID)
		return nil, ErrInternalServerError
	}
	return inst.VFS().DirByID(res.Rows[0].ID)
}

// ApplyBulkFiles takes a list of documents for the io.cozy.files doctype and
// will apply changes to the VFS according to those documents.
func (s *Sharing) ApplyBulkFiles(inst *instance.Instance, docs DocsList) error {
	var errm error
	fs := inst.VFS()

	for _, target := range docs {
		id, ok := target["_id"].(string)
		if !ok {
			errm = multierror.Append(errm, ErrMissingID)
			continue
		}
		var ref *SharedRef
		err := couchdb.GetDoc(inst, consts.Shared, consts.Files+"/"+id, ref)
		if err != nil && !couchdb.IsNotFoundError(err) {
			inst.Logger().WithField("nspace", "replicator").Debugf("Error on finding doc of bulk files: %s", err)
			errm = multierror.Append(errm, err)
			continue
		}
		// TODO it's only for directory currently, code needs to be adapted for files
		doc, err := fs.DirByID(id) // TODO DirOrFileByID
		if err != nil && err != os.ErrNotExist {
			inst.Logger().WithField("nspace", "replicator").Debugf("Error on finding ref of bulk files: %s", err)
			errm = multierror.Append(errm, err)
			continue
		}
		if ref == nil && doc == nil {
			err = s.CreateDir(inst, target)
			if err != nil {
				errm = multierror.Append(errm, err)
			}
			// TODO update the io.cozy.shared reference?
		} else if ref == nil {
			// TODO be safe => return an error
			continue
		} else if doc == nil {
			// TODO manage the conflict: doc was deleted/moved outside the
			// sharing on this cozy and updated on the other cozy
			continue
		} else {
			err = s.UpdateDir(inst, target, doc)
			if err != nil {
				errm = multierror.Append(errm, err)
			}
		}
	}
	return nil
}

func copyTagsAndDatesToDir(target map[string]interface{}, dir *vfs.DirDoc) {
	if tags, ok := target["tags"].([]interface{}); ok {
		dir.Tags = make([]string, 0, len(tags))
		for _, tag := range tags {
			if t, ok := tag.(string); ok {
				dir.Tags = append(dir.Tags, t)
			}
		}
	}
	if created, ok := target["created_at"].(string); ok {
		if at, err := time.Parse(time.RFC3339Nano, created); err == nil {
			dir.CreatedAt = at
		}
	}
	if updated, ok := target["updated_at"].(string); ok {
		if at, err := time.Parse(time.RFC3339Nano, updated); err == nil {
			dir.UpdatedAt = at
		}
	}
}

// CreateDir creates a directory on this cozy to reflect a change on another
// cozy instance of this sharing.
func (s *Sharing) CreateDir(inst *instance.Instance, target map[string]interface{}) error {
	name, ok := target["name"].(string)
	if !ok {
		inst.Logger().WithField("nspace", "replicator").
			Debugf("Missing name for creating dir: %#v", target)
		return ErrInternalServerError
	}
	rev, ok := target["_rev"].(string)
	if !ok {
		inst.Logger().WithField("nspace", "replicator").
			Debugf("Missing _rev for creating dir: %#v", target)
		return ErrInternalServerError
	}
	revisions, ok := target["_revisions"].(map[string]interface{})
	if !ok {
		inst.Logger().WithField("nspace", "replicator").
			Debugf("Missing _revisions for creating dir: %#v", target)
		return ErrInternalServerError
	}
	indexer := NewSharingIndexer(inst, &bulkRevs{
		Rev:       rev,
		Revisions: revisions,
	})
	fs := inst.VFS().UseSharingIndexer(indexer)

	var parent *vfs.DirDoc
	var err error
	if dirID, ok := target["dir_id"].(string); ok {
		parent, err = fs.DirByID(dirID)
		// TODO better handling of this conflict
		if err != nil {
			inst.Logger().WithField("nspace", "replicator").
				Debugf("Conflict for parent on creating dir: %s", err)
			return err
		}
	} else {
		parent, err = s.GetSharingDir(inst)
		if err != nil {
			return err
		}
	}

	dir, err := vfs.NewDirDocWithParent(name, parent, nil)
	if err != nil {
		inst.Logger().WithField("nspace", "replicator").
			Warnf("Cannot initialize dir doc: %s", err)
		return err
	}
	dir.SetID(target["_id"].(string))
	copyTagsAndDatesToDir(target, dir)
	// TODO referenced_by
	// TODO manage conflicts
	if err := fs.CreateDir(dir); err != nil {
		inst.Logger().WithField("nspace", "replicator").
			Debugf("Cannot create dir: %s", err)
		return err
	}
	return nil
}

// UpdateDir updates a directory on this cozy to reflect a change on another
// cozy instance of this sharing.
func (s *Sharing) UpdateDir(inst *instance.Instance, target map[string]interface{}, dir *vfs.DirDoc) error {
	rev, ok := target["_rev"].(string)
	if !ok {
		// TODO add logs or better error
		return ErrInternalServerError
	}
	revisions, ok := target["_revisions"].(map[string]interface{})
	if !ok {
		return ErrInternalServerError
	}
	indexer := NewSharingIndexer(inst, &bulkRevs{
		Rev:       rev,
		Revisions: revisions,
	})
	copyTagsAndDatesToDir(target, dir)
	// TODO what if name or dir_id has changed
	// TODO referenced_by
	// TODO trash
	// TODO manage conflicts
	return indexer.UpdateDirDoc(nil, dir) // TODO oldDoc
}

// TODO referenced_by
func dirToJSONDoc(dir *vfs.DirDoc) couchdb.JSONDoc {
	doc := couchdb.JSONDoc{
		Type: consts.Files,
		M: map[string]interface{}{
			"type":       dir.Type,
			"_id":        dir.DocID,
			"_rev":       dir.DocRev,
			"name":       dir.DocName,
			"created_at": dir.CreatedAt,
			"updated_at": dir.UpdatedAt,
			"tags":       dir.Tags,
			"path":       dir.Fullpath,
		},
	}
	if dir.DirID != "" {
		doc.M["dir_id"] = dir.DirID
	}
	if dir.RestorePath != "" {
		doc.M["restore_path"] = dir.RestorePath
	}
	return doc
}

// TODO referenced_by
func fileToJSONDoc(file *vfs.FileDoc) couchdb.JSONDoc {
	doc := couchdb.JSONDoc{
		Type: consts.Files,
		M: map[string]interface{}{
			"type":       file.Type,
			"_id":        file.DocID,
			"_rev":       file.DocRev,
			"name":       file.DocName,
			"created_at": file.CreatedAt,
			"updated_at": file.UpdatedAt,
			"size":       file.ByteSize,
			"md5Sum":     file.MD5Sum,
			"mime":       file.Mime,
			"class":      file.Class,
			"executable": file.Executable,
			"trashed":    file.Trashed,
			"tags":       file.Tags,
		},
	}
	if file.DirID != "" {
		doc.M["dir_id"] = file.DirID
	}
	if file.RestorePath != "" {
		doc.M["restore_path"] = file.RestorePath
	}
	if len(file.Metadata) > 0 {
		doc.M["metadata"] = file.Metadata
	}
	return doc
}
<|endoftext|>