<commit_before>/*
Copyright 2017 The Nuclio Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package app

import (
	"time"

	"github.com/nuclio/nuclio/pkg/errors"
	"github.com/nuclio/nuclio/pkg/loggersink"
	nuclioio_client "github.com/nuclio/nuclio/pkg/platform/kube/client/clientset/versioned"
	"github.com/nuclio/nuclio/pkg/platform/kube/controller"
	"github.com/nuclio/nuclio/pkg/platform/kube/functionres"
	"github.com/nuclio/nuclio/pkg/platformconfig"
	// load all sinks
	_ "github.com/nuclio/nuclio/pkg/sinks"

	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
)

func Run(kubeconfigPath string,
	namespace string,
	imagePullSecrets string,
	platformConfigurationPath string) error {

	newController, err := createController(kubeconfigPath, namespace, imagePullSecrets, platformConfigurationPath)
	if err != nil {
		return errors.Wrap(err, "Failed to create controller")
	}

	// start the controller
	if err := newController.Start(); err != nil {
		return errors.Wrap(err, "Failed to start controller")
	}

	// TODO: stop
	select {}
}

func createController(kubeconfigPath string,
	namespace string,
	imagePullSecrets string,
	platformConfigurationPath string) (*controller.Controller, error) {

	// read platform configuration
	platformConfiguration, err := readPlatformConfiguration(platformConfigurationPath)
	if err != nil {
		return nil, errors.Wrap(err, "Failed to read platform configuration")
	}

	// create a root logger
	rootLogger, err := loggersink.CreateSystemLogger("controller", platformConfiguration)
	if err != nil {
		return nil, errors.Wrap(err, "Failed to create logger")
	}

	restConfig, err := getClientConfig(kubeconfigPath)
	if err != nil {
		return nil, errors.Wrap(err, "Failed to get client configuration")
	}

	kubeClientSet, err := kubernetes.NewForConfig(restConfig)
	if err != nil {
		return nil, errors.Wrap(err, "Failed to create k8s client set")
	}

	nuclioClientSet, err := nuclioio_client.NewForConfig(restConfig)
	if err != nil {
		return nil, errors.Wrap(err, "Failed to create nuclio client set")
	}

	// create a client for function deployments
	functionresClient, err := functionres.NewLazyClient(rootLogger, kubeClientSet, nuclioClientSet)
	if err != nil {
		return nil, errors.Wrap(err, "Failed to create function deployment client")
	}

	newController, err := controller.NewController(rootLogger,
		namespace,
		imagePullSecrets,
		kubeClientSet,
		nuclioClientSet,
		functionresClient,
		5*time.Minute,
		platformConfiguration)

	if err != nil {
		return nil, err
	}

	return newController, nil
}

func getClientConfig(kubeconfigPath string) (*rest.Config, error) {
	if kubeconfigPath != "" {
		return clientcmd.BuildConfigFromFlags("", kubeconfigPath)
	}

	return rest.InClusterConfig()
}

func readPlatformConfiguration(configurationPath string) (*platformconfig.Config, error) {
	platformConfigurationReader, err := platformconfig.NewReader()
	if err != nil {
		return nil, errors.Wrap(err, "Failed to create platform configuration reader")
	}

	return platformConfigurationReader.ReadFileOrDefault(configurationPath)
}
<commit_msg>Upping controller's resync interval to 10min (#1479)<commit_after>/*
Copyright 2017 The Nuclio Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package app

import (
	"time"

	"github.com/nuclio/nuclio/pkg/errors"
	"github.com/nuclio/nuclio/pkg/loggersink"
	nuclioio_client "github.com/nuclio/nuclio/pkg/platform/kube/client/clientset/versioned"
	"github.com/nuclio/nuclio/pkg/platform/kube/controller"
	"github.com/nuclio/nuclio/pkg/platform/kube/functionres"
	"github.com/nuclio/nuclio/pkg/platformconfig"
	// load all sinks
	_ "github.com/nuclio/nuclio/pkg/sinks"

	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
)

func Run(kubeconfigPath string,
	namespace string,
	imagePullSecrets string,
	platformConfigurationPath string) error {

	newController, err := createController(kubeconfigPath, namespace, imagePullSecrets, platformConfigurationPath)
	if err != nil {
		return errors.Wrap(err, "Failed to create controller")
	}

	// start the controller
	if err := newController.Start(); err != nil {
		return errors.Wrap(err, "Failed to start controller")
	}

	// TODO: stop
	select {}
}

func createController(kubeconfigPath string,
	namespace string,
	imagePullSecrets string,
	platformConfigurationPath string) (*controller.Controller, error) {

	// read platform configuration
	platformConfiguration, err := readPlatformConfiguration(platformConfigurationPath)
	if err != nil {
		return nil, errors.Wrap(err, "Failed to read platform configuration")
	}

	// create a root logger
	rootLogger, err := loggersink.CreateSystemLogger("controller", platformConfiguration)
	if err != nil {
		return nil, errors.Wrap(err, "Failed to create logger")
	}

	restConfig, err := getClientConfig(kubeconfigPath)
	if err != nil {
		return nil, errors.Wrap(err, "Failed to get client configuration")
	}

	kubeClientSet, err := kubernetes.NewForConfig(restConfig)
	if err != nil {
		return nil, errors.Wrap(err, "Failed to create k8s client set")
	}

	nuclioClientSet, err := nuclioio_client.NewForConfig(restConfig)
	if err != nil {
		return nil, errors.Wrap(err, "Failed to create nuclio client set")
	}

	// create a client for function deployments
	functionresClient, err := functionres.NewLazyClient(rootLogger, kubeClientSet, nuclioClientSet)
	if err != nil {
		return nil, errors.Wrap(err, "Failed to create function deployment client")
	}

	newController, err := controller.NewController(rootLogger,
		namespace,
		imagePullSecrets,
		kubeClientSet,
		nuclioClientSet,
		functionresClient,
		10*time.Minute,
		platformConfiguration)

	if err != nil {
		return nil, err
	}

	return newController, nil
}

func getClientConfig(kubeconfigPath string) (*rest.Config, error) {
	if kubeconfigPath != "" {
		return clientcmd.BuildConfigFromFlags("", kubeconfigPath)
	}

	return rest.InClusterConfig()
}

func readPlatformConfiguration(configurationPath string) (*platformconfig.Config, error) {
	platformConfigurationReader, err := platformconfig.NewReader()
	if err != nil {
		return nil, errors.Wrap(err, "Failed to create platform configuration reader")
	}

	return platformConfigurationReader.ReadFileOrDefault(configurationPath)
}
<|endoftext|>


<commit_before>package dev

import (
	"encoding/binary"
	"fmt"
	"hash/fnv"
	"log"
	"net"
	"sort"
	"strings"
	"sync"
	"time"

	"golang.org/x/net/trace"
	"google.golang.org/grpc"
	"google.golang.org/grpc/backoff"
)

// Manager manages a pool of node configurations on which quorum remote
// procedure calls can be made.
type Manager struct {
	mu       sync.Mutex
	nodes    []*Node
	lookup   map[uint32]*Node
	configs  map[uint32]*Configuration
	eventLog trace.EventLog

	closeOnce sync.Once
	logger    *log.Logger
	opts      managerOptions

	*receiveQueue
}

// NewManager attempts to connect to the given set of node addresses and if
// successful returns a new Manager containing connections to those nodes.
func NewManager(opts ...ManagerOption) (*Manager, error) {

	m := &Manager{
		lookup:       make(map[uint32]*Node),
		configs:      make(map[uint32]*Configuration),
		receiveQueue: newReceiveQueue(),
		opts:         newManagerOptions(),
	}

	for _, opt := range opts {
		opt(&m.opts)
	}

	m.opts.grpcDialOpts = append(m.opts.grpcDialOpts, grpc.WithDefaultCallOptions(
		grpc.CallContentSubtype(gorumsContentType),
		grpc.ForceCodec(newGorumsCodec()),
	))

	if len(m.opts.addrsList) == 0 && len(m.opts.IDMapping) == 0 {
		return nil, fmt.Errorf("could not create manager: no nodes provided")
	}

	if m.opts.backoff != backoff.DefaultConfig {
		m.opts.grpcDialOpts = append(m.opts.grpcDialOpts, grpc.WithConnectParams(
			grpc.ConnectParams{Backoff: m.opts.backoff},
		))
	}

	var nodeAddrs []string
	if m.opts.IDMapping != nil {
		for naddr, id := range m.opts.IDMapping {

			nodeAddrs = append(nodeAddrs, naddr)
			node, err2 := m.createNode(naddr, id)

			if err2 != nil {
				return nil, ManagerCreationError(err2)
			}
			m.lookup[node.id] = node
			m.nodes = append(m.nodes, node)
		}

	} else if m.opts.addrsList != nil {

		nodeAddrs = m.opts.addrsList

		for _, naddr := range m.opts.addrsList {

			node, err2 := m.createNode(naddr, 0)

			if err2 != nil {
				return nil, ManagerCreationError(err2)
			}
			m.lookup[node.id] = node
			m.nodes = append(m.nodes, node)
		}

	}

	if m.opts.trace {
		title := strings.Join(nodeAddrs, ",")
		m.eventLog = trace.NewEventLog("gorums.Manager", title)
	}

	err := m.connectAll()
	if err != nil {
		return nil, ManagerCreationError(err)
	}

	if m.opts.logger != nil {
		m.logger = m.opts.logger
	}

	if m.eventLog != nil {
		m.eventLog.Printf("ready")
	}

	return m, nil
}

func (m *Manager) createNode(addr string, id uint32) (*Node, error) {
	m.mu.Lock()
	defer m.mu.Unlock()

	tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
	if err != nil {
		return nil, fmt.Errorf("create node %s error: %v", addr, err)
	}

	if id == 0 {
		h := fnv.New32a()
		_, _ = h.Write([]byte(tcpAddr.String()))
		id = h.Sum32()
	}

	if _, found := m.lookup[id]; found {
		return nil, fmt.Errorf("create node %s error: node already exists", addr)
	}

	node := &Node{
		id:      id,
		addr:    tcpAddr.String(),
		latency: -1 * time.Second,
	}
	node.createOrderedStream(m.receiveQueue, m.opts)

	return node, nil
}

func (m *Manager) connectAll() error {
	if m.opts.noConnect {
		return nil
	}

	if m.eventLog != nil {
		m.eventLog.Printf("connecting")
	}

	for _, node := range m.nodes {
		err := node.connect(m.opts)
		if err != nil {
			if m.eventLog != nil {
				m.eventLog.Errorf("connect failed, error connecting to node %s, error: %v", node.addr, err)
			}
			return fmt.Errorf("connect node %s error: %v", node.addr, err)
		}
	}
	return nil
}

func (m *Manager) closeNodeConns() {
	for _, node := range m.nodes {
		err := node.close()
		if err == nil {
			continue
		}
		if m.logger != nil {
			m.logger.Printf("node %d: error closing: %v", node.id, err)
		}
	}
}

// Close closes all node connections and any client streams.
func (m *Manager) Close() {
	m.closeOnce.Do(func() {
		if m.eventLog != nil {
			m.eventLog.Printf("closing")
		}
		m.closeNodeConns()
	})
}

// NodeIDs returns the identifier of each available node. IDs are returned in
// the same order as they were provided in the creation of the Manager.
func (m *Manager) NodeIDs() []uint32 {
	m.mu.Lock()
	defer m.mu.Unlock()
	ids := make([]uint32, 0, len(m.nodes))
	for _, node := range m.nodes {
		ids = append(ids, node.ID())
	}
	return ids
}

// Node returns the node with the given identifier if present.
func (m *Manager) Node(id uint32) (node *Node, found bool) {
	m.mu.Lock()
	defer m.mu.Unlock()
	node, found = m.lookup[id]
	return node, found
}

// Nodes returns a slice of each available node. IDs are returned in the same
// order as they were provided in the creation of the Manager.
func (m *Manager) Nodes() []*Node {
	m.mu.Lock()
	defer m.mu.Unlock()
	return m.nodes
}

// ConfigurationIDs returns the identifier of each available
// configuration.
func (m *Manager) ConfigurationIDs() []uint32 {
	m.mu.Lock()
	defer m.mu.Unlock()
	ids := make([]uint32, 0, len(m.configs))
	for id := range m.configs {
		ids = append(ids, id)
	}
	return ids
}

// Configuration returns the configuration with the given global
// identifier if present.
func (m *Manager) Configuration(id uint32) (config *Configuration, found bool) {
	m.mu.Lock()
	defer m.mu.Unlock()
	config, found = m.configs[id]
	return config, found
}

// Configurations returns a slice of each available configuration.
func (m *Manager) Configurations() []*Configuration {
	m.mu.Lock()
	defer m.mu.Unlock()
	configs := make([]*Configuration, 0, len(m.configs))
	for _, conf := range m.configs {
		configs = append(configs, conf)
	}
	return configs
}

// Size returns the number of nodes and configurations in the Manager.
func (m *Manager) Size() (nodes, configs int) {
	m.mu.Lock()
	defer m.mu.Unlock()
	return len(m.nodes), len(m.configs)
}

// AddNode attempts to dial to the provide node address. The node is
// added to the Manager's pool of nodes if a connection was established.
func (m *Manager) AddNode(addr string) error {
	panic("not implemented")
}

// NewConfiguration returns a new configuration given quorum specification and
// a timeout.
func (m *Manager) NewConfiguration(ids []uint32, qspec QuorumSpec) (*Configuration, error) {
	m.mu.Lock()
	defer m.mu.Unlock()

	if len(ids) == 0 {
		return nil, IllegalConfigError("need at least one node")
	}

	var nodes []*Node
	unique := make(map[uint32]struct{})
	var uniqueIDs []uint32
	for _, nid := range ids {
		// ensure that identical IDs are only counted once
		if _, duplicate := unique[nid]; duplicate {
			continue
		}
		unique[nid] = struct{}{}
		uniqueIDs = append(uniqueIDs, nid)

		node, found := m.lookup[nid]
		if !found {
			return nil, NodeNotFoundError(nid)
		}
		nodes = append(nodes, node)
	}

	// node IDs are sorted to ensure a globally consistent configuration ID
	sort.Sort(idSlice(uniqueIDs))

	h := fnv.New32a()
	for _, id := range uniqueIDs {
		_ = binary.Write(h, binary.LittleEndian, id)
	}
	cid := h.Sum32()

	conf, found := m.configs[cid]
	if found {
		return conf, nil
	}

	c := &Configuration{
		id:    cid,
		nodes: nodes,
		n:     len(nodes),
		mgr:   m,
		qspec: qspec,
	}
	m.configs[cid] = c

	return c, nil
}

type idSlice []uint32

func (p idSlice) Len() int           { return len(p) }
func (p idSlice) Less(i, j int) bool { return p[i] < p[j] }
func (p idSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
<commit_msg>nodes are now sorted when put in the manages nodes list if the id mapping option is chosen.<commit_after>package dev

import (
	"encoding/binary"
	"fmt"
	"hash/fnv"
	"log"
	"net"
	"sort"
	"strings"
	"sync"
	"time"

	"golang.org/x/net/trace"
	"google.golang.org/grpc"
	"google.golang.org/grpc/backoff"
)

// Manager manages a pool of node configurations on which quorum remote
// procedure calls can be made.
type Manager struct {
	mu       sync.Mutex
	nodes    []*Node
	lookup   map[uint32]*Node
	configs  map[uint32]*Configuration
	eventLog trace.EventLog

	closeOnce sync.Once
	logger    *log.Logger
	opts      managerOptions

	*receiveQueue
}

// NewManager attempts to connect to the given set of node addresses and if
// successful returns a new Manager containing connections to those nodes.
func NewManager(opts ...ManagerOption) (*Manager, error) {

	m := &Manager{
		lookup:       make(map[uint32]*Node),
		configs:      make(map[uint32]*Configuration),
		receiveQueue: newReceiveQueue(),
		opts:         newManagerOptions(),
	}

	for _, opt := range opts {
		opt(&m.opts)
	}

	m.opts.grpcDialOpts = append(m.opts.grpcDialOpts, grpc.WithDefaultCallOptions(
		grpc.CallContentSubtype(gorumsContentType),
		grpc.ForceCodec(newGorumsCodec()),
	))

	if len(m.opts.addrsList) == 0 && len(m.opts.IDMapping) == 0 {
		return nil, fmt.Errorf("could not create manager: no nodes provided")
	}

	if m.opts.backoff != backoff.DefaultConfig {
		m.opts.grpcDialOpts = append(m.opts.grpcDialOpts, grpc.WithConnectParams(
			grpc.ConnectParams{Backoff: m.opts.backoff},
		))
	}

	var nodeAddrs []string
	if m.opts.IDMapping != nil {
		for naddr, id := range m.opts.IDMapping {

			nodeAddrs = append(nodeAddrs, naddr)
			node, err2 := m.createNode(naddr, id)

			if err2 != nil {
				return nil, ManagerCreationError(err2)
			}
			m.lookup[node.id] = node

			// Sorting the nodes when puting them in the node list from lowest to highest id value. This has to be done in order for there to be some system when
			// returning node IDs. Looping over a map is none-deterministic and therefor the nodes can't simpely be put in a list.
			for i := range m.nodes {
				if id <= m.nodes[i].id {
					var temp []*Node
					if i == 0 {
						temp = append(temp, node)
						m.nodes = append(temp, m.nodes...)
						break
					}
					temp = m.nodes[i:]
					m.nodes = append(append(m.nodes[:i], node), temp...)
					break
				}
			}
			m.nodes = append(m.nodes, node)
		}

	} else if m.opts.addrsList != nil {

		nodeAddrs = m.opts.addrsList

		for _, naddr := range m.opts.addrsList {

			node, err2 := m.createNode(naddr, 0)

			if err2 != nil {
				return nil, ManagerCreationError(err2)
			}
			m.lookup[node.id] = node
			m.nodes = append(m.nodes, node)
		}

	}

	if m.opts.trace {
		title := strings.Join(nodeAddrs, ",")
		m.eventLog = trace.NewEventLog("gorums.Manager", title)
	}

	err := m.connectAll()
	if err != nil {
		return nil, ManagerCreationError(err)
	}

	if m.opts.logger != nil {
		m.logger = m.opts.logger
	}

	if m.eventLog != nil {
		m.eventLog.Printf("ready")
	}

	return m, nil
}

func (m *Manager) createNode(addr string, id uint32) (*Node, error) {
	m.mu.Lock()
	defer m.mu.Unlock()

	tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
	if err != nil {
		return nil, fmt.Errorf("create node %s error: %v", addr, err)
	}

	if id == 0 {
		h := fnv.New32a()
		_, _ = h.Write([]byte(tcpAddr.String()))
		id = h.Sum32()
	}

	if _, found := m.lookup[id]; found {
		return nil, fmt.Errorf("create node %s error: node already exists", addr)
	}

	node := &Node{
		id:      id,
		addr:    tcpAddr.String(),
		latency: -1 * time.Second,
	}
	node.createOrderedStream(m.receiveQueue, m.opts)

	return node, nil
}

func (m *Manager) connectAll() error {
	if m.opts.noConnect {
		return nil
	}

	if m.eventLog != nil {
		m.eventLog.Printf("connecting")
	}

	for _, node := range m.nodes {
		err := node.connect(m.opts)
		if err != nil {
			if m.eventLog != nil {
				m.eventLog.Errorf("connect failed, error connecting to node %s, error: %v", node.addr, err)
			}
			return fmt.Errorf("connect node %s error: %v", node.addr, err)
		}
	}
	return nil
}

func (m *Manager) closeNodeConns() {
	for _, node := range m.nodes {
		err := node.close()
		if err == nil {
			continue
		}
		if m.logger != nil {
			m.logger.Printf("node %d: error closing: %v", node.id, err)
		}
	}
}

// Close closes all node connections and any client streams.
func (m *Manager) Close() {
	m.closeOnce.Do(func() {
		if m.eventLog != nil {
			m.eventLog.Printf("closing")
		}
		m.closeNodeConns()
	})
}

// NodeIDs returns the identifier of each available node. IDs are returned in
// the same order as they were provided in the creation of the Manager.
func (m *Manager) NodeIDs() []uint32 {
	m.mu.Lock()
	defer m.mu.Unlock()
	ids := make([]uint32, 0, len(m.nodes))
	for _, node := range m.nodes {
		ids = append(ids, node.ID())
	}
	return ids
}

// Node returns the node with the given identifier if present.
func (m *Manager) Node(id uint32) (node *Node, found bool) {
	m.mu.Lock()
	defer m.mu.Unlock()
	node, found = m.lookup[id]
	return node, found
}

// Nodes returns a slice of each available node. IDs are returned in the same
// order as they were provided in the creation of the Manager.
func (m *Manager) Nodes() []*Node {
	m.mu.Lock()
	defer m.mu.Unlock()
	return m.nodes
}

// ConfigurationIDs returns the identifier of each available
// configuration.
func (m *Manager) ConfigurationIDs() []uint32 {
	m.mu.Lock()
	defer m.mu.Unlock()
	ids := make([]uint32, 0, len(m.configs))
	for id := range m.configs {
		ids = append(ids, id)
	}
	return ids
}

// Configuration returns the configuration with the given global
// identifier if present.
func (m *Manager) Configuration(id uint32) (config *Configuration, found bool) {
	m.mu.Lock()
	defer m.mu.Unlock()
	config, found = m.configs[id]
	return config, found
}

// Configurations returns a slice of each available configuration.
func (m *Manager) Configurations() []*Configuration {
	m.mu.Lock()
	defer m.mu.Unlock()
	configs := make([]*Configuration, 0, len(m.configs))
	for _, conf := range m.configs {
		configs = append(configs, conf)
	}
	return configs
}

// Size returns the number of nodes and configurations in the Manager.
func (m *Manager) Size() (nodes, configs int) {
	m.mu.Lock()
	defer m.mu.Unlock()
	return len(m.nodes), len(m.configs)
}

// AddNode attempts to dial to the provide node address. The node is
// added to the Manager's pool of nodes if a connection was established.
func (m *Manager) AddNode(addr string) error {
	panic("not implemented")
}

// NewConfiguration returns a new configuration given quorum specification and
// a timeout.
func (m *Manager) NewConfiguration(ids []uint32, qspec QuorumSpec) (*Configuration, error) {
	m.mu.Lock()
	defer m.mu.Unlock()

	if len(ids) == 0 {
		return nil, IllegalConfigError("need at least one node")
	}

	var nodes []*Node
	unique := make(map[uint32]struct{})
	var uniqueIDs []uint32
	for _, nid := range ids {
		// ensure that identical IDs are only counted once
		if _, duplicate := unique[nid]; duplicate {
			continue
		}
		unique[nid] = struct{}{}
		uniqueIDs = append(uniqueIDs, nid)

		node, found := m.lookup[nid]
		if !found {
			return nil, NodeNotFoundError(nid)
		}
		nodes = append(nodes, node)
	}

	// node IDs are sorted to ensure a globally consistent configuration ID
	sort.Sort(idSlice(uniqueIDs))

	h := fnv.New32a()
	for _, id := range uniqueIDs {
		_ = binary.Write(h, binary.LittleEndian, id)
	}
	cid := h.Sum32()

	conf, found := m.configs[cid]
	if found {
		return conf, nil
	}

	c := &Configuration{
		id:    cid,
		nodes: nodes,
		n:     len(nodes),
		mgr:   m,
		qspec: qspec,
	}
	m.configs[cid] = c

	return c, nil
}

type idSlice []uint32

func (p idSlice) Len() int           { return len(p) }
func (p idSlice) Less(i, j int) bool { return p[i] < p[j] }
func (p idSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"context"
	"fmt"
	"log"

	"github.com/apigee/registry/cmd/registry/core"
	"github.com/apigee/registry/connection"
	"github.com/apigee/registry/rpc"
	"github.com/apigee/registry/server/names"
	metrics "github.com/googleapis/gnostic/metrics"
	"github.com/spf13/cobra"
	"google.golang.org/protobuf/proto"
)

func init() {
	exportCmd.AddCommand(exportSheetCmd)
}

// exportSheetCmd represents the export sheet command
var exportSheetCmd = &cobra.Command{
	Use:   "sheet",
	Short: "Export a specified property to a Google sheet.",
	Long:  "Export a specified property to a Google sheet.",
	Args:  cobra.MinimumNArgs(1),
	Run: func(cmd *cobra.Command, args []string) {
		var err error
		ctx := context.TODO()
		client, err := connection.NewClient(ctx)
		if err != nil {
			log.Fatalf("%s", err.Error())
		}
		_, inputs := collectInputProperties(ctx, client, args, exportFilter)
		if len(inputs) == 0 {
			return
		}
		typeURL := messageTypeURL(inputs[0])
		if typeURL == "gnostic.metrics.Vocabulary" {
			if len(inputs) == 1 {
				vocabulary, err := getVocabulary(inputs[0])
				if err != nil {
					log.Fatalf("%s", err.Error())
				}
				err = core.ExportVocabularyToSheet(inputs[0].Name, vocabulary)
			} else {
				log.Fatalf("%d properties matched. Please specify exactly one for export.", len(inputs))
			}
		}
	},
}

func collectInputProperties(ctx context.Context, client connection.Client, args []string, filter string) ([]string, []*rpc.Property) {
	inputNames := make([]string, 0)
	inputs := make([]*rpc.Property, 0)
	for _, name := range args {
		if m := names.PropertyRegexp().FindStringSubmatch(name); m != nil {
			err := core.ListProperties(ctx, client, m, filter, func(property *rpc.Property) {
				inputNames = append(inputNames, property.Name)
				inputs = append(inputs, property)
			})
			if err != nil {
				log.Fatalf("%s", err.Error())
			}
		}
	}
	return inputNames, inputs
}

func messageTypeURL(property *rpc.Property) string {
	switch v := property.GetValue().(type) {
	case *rpc.Property_MessageValue:
		return v.MessageValue.TypeUrl
	default:
		return ""
	}
}

func getVocabulary(property *rpc.Property) (*metrics.Vocabulary, error) {
	switch v := property.GetValue().(type) {
	case *rpc.Property_MessageValue:
		if v.MessageValue.TypeUrl == "gnostic.metrics.Vocabulary" {
			vocab := &metrics.Vocabulary{}
			err := proto.Unmarshal(v.MessageValue.Value, vocab)
			if err != nil {
				return nil, err
			} else {
				return vocab, nil
			}
		} else {
			return nil, fmt.Errorf("not a vocabulary: %s\n", property.Name)
		}
	default:
		return nil, fmt.Errorf("not a vocabulary: %s\n", property.Name)
	}
}
<commit_msg>complexity export<commit_after>// Copyright 2020 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"context"
	"fmt"
	"log"
	"path/filepath"
	"strings"

	"github.com/apigee/registry/cmd/registry/core"
	"github.com/apigee/registry/connection"
	"github.com/apigee/registry/rpc"
	"github.com/apigee/registry/server/names"
	metrics "github.com/googleapis/gnostic/metrics"
	"github.com/spf13/cobra"
	"google.golang.org/protobuf/proto"
)

func init() {
	exportCmd.AddCommand(exportSheetCmd)
}

// exportSheetCmd represents the export sheet command
var exportSheetCmd = &cobra.Command{
	Use:   "sheet",
	Short: "Export a specified property to a Google sheet.",
	Long:  "Export a specified property to a Google sheet.",
	Args:  cobra.MinimumNArgs(1),
	Run: func(cmd *cobra.Command, args []string) {
		var err error
		ctx := context.TODO()
		client, err := connection.NewClient(ctx)
		if err != nil {
			log.Fatalf("%s", err.Error())
		}
		_, inputs := collectInputProperties(ctx, client, args, exportFilter)
		if len(inputs) == 0 {
			return
		}
		typeURL := messageTypeURL(inputs[0])
		if typeURL == "gnostic.metrics.Vocabulary" {
			if len(inputs) == 1 {
				vocabulary, err := getVocabulary(inputs[0])
				if err != nil {
					log.Fatalf("%s", err.Error())
				}
				err = core.ExportVocabularyToSheet(inputs[0].Name, vocabulary)
			} else {
				log.Fatalf("%d properties matched. Please specify exactly one for export.", len(inputs))
			}
		} else if typeURL == "gnostic.metrics.Complexity" {
			name := "Complexity"
			sheetsClient, err := core.NewSheetsClient("")
			if err != nil {
				log.Fatalf("%s", err.Error())
			}
			sheet, err := sheetsClient.CreateSheet(name, []string{"Complexity"})
			if err != nil {
				log.Fatalf("%s", err.Error())
			}
			rows := make([][]interface{}, 0)
			rows = append(rows, rowForLabeledComplexity("api", "version", nil))
			for _, input := range inputs {
				complexity, err := getComplexity(input)
				if err != nil {
					log.Fatalf("%s", err.Error())
				}
				parts := strings.Split(input.Name, "/") // use to get api_id [3] and version_id [5]
				rows = append(rows, rowForLabeledComplexity(parts[3], parts[5], complexity))
			}
			_, err = sheetsClient.Update(fmt.Sprintf("Complexity"), rows)
			if err != nil {
				log.Fatalf("%s", err.Error())
			}
			log.Printf("exported to %+v\n", sheet.SpreadsheetUrl)
		} else {
			log.Fatalf("Unknown message type: %s", typeURL)
		}
	},
}

func versionNameOfPropertyName(propertyName string) string {
	n := propertyName
	for i := 0; i < 4; i++ {
		n = filepath.Dir(n)
	}
	return n
}

func collectInputProperties(ctx context.Context, client connection.Client, args []string, filter string) ([]string, []*rpc.Property) {
	inputNames := make([]string, 0)
	inputs := make([]*rpc.Property, 0)
	for _, name := range args {
		if m := names.PropertyRegexp().FindStringSubmatch(name); m != nil {
			err := core.ListProperties(ctx, client, m, filter, func(property *rpc.Property) {
				inputNames = append(inputNames, property.Name)
				inputs = append(inputs, property)
			})
			if err != nil {
				log.Fatalf("%s", err.Error())
			}
		}
	}
	return inputNames, inputs
}

func messageTypeURL(property *rpc.Property) string {
	switch v := property.GetValue().(type) {
	case *rpc.Property_MessageValue:
		return v.MessageValue.TypeUrl
	default:
		return ""
	}
}

func getVocabulary(property *rpc.Property) (*metrics.Vocabulary, error) {
	switch v := property.GetValue().(type) {
	case *rpc.Property_MessageValue:
		if v.MessageValue.TypeUrl == "gnostic.metrics.Vocabulary" {
			vocab := &metrics.Vocabulary{}
			err := proto.Unmarshal(v.MessageValue.Value, vocab)
			if err != nil {
				return nil, err
			} else {
				return vocab, nil
			}
		} else {
			return nil, fmt.Errorf("not a vocabulary: %s", property.Name)
		}
	default:
		return nil, fmt.Errorf("not a vocabulary: %s", property.Name)
	}
}

func getComplexity(property *rpc.Property) (*metrics.Complexity, error) {
	switch v := property.GetValue().(type) {
	case *rpc.Property_MessageValue:
		if v.MessageValue.TypeUrl == "gnostic.metrics.Complexity" {
			value := &metrics.Complexity{}
			err := proto.Unmarshal(v.MessageValue.Value, value)
			if err != nil {
				return nil, err
			} else {
				return value, nil
			}
		} else {
			return nil, fmt.Errorf("not a complexity: %s", property.Name)
		}
	default:
		return nil, fmt.Errorf("not a complexity: %s", property.Name)
	}
}

func rowForLabeledComplexity(api, version string, c *metrics.Complexity) []interface{} {
	if c == nil {
		return []interface{}{
			"api",
			"version",
			"schemas",
			"schema properties",
			"paths",
			"gets",
			"posts",
			"puts",
			"deletes",
		}
	}
	return []interface{}{api, version, c.SchemaCount, c.SchemaPropertyCount, c.PathCount, c.GetCount, c.PostCount, c.PutCount, c.DeleteCount}
}
<|endoftext|>


<commit_before>package couchbase

import (
	"bytes"
	"fmt"
	"errors"
	cb "github.com/couchbaselabs/go-couchbase"
	"github.com/couchbaselabs/tuqtng/ast"
	"github.com/couchbaselabs/tuqtng/catalog"
	"hash/crc32"
	"strconv"
	"strings"
)

type ddocJSON struct {
	cb.DDocJSON
	IndexOn       []string `json:"indexOn"`
	IndexChecksum int      `json:"indexChecksum"`
}

func newViewIndex(name string, on catalog.IndexKey, bkt *bucket) (*viewIndex, error) {

	doc, err := newDesignDoc(name, on)
	if err != nil {
		return nil, err
	}

	inst := viewIndex{
		name:   name,
		using:  catalog.VIEW,
		on:     on,
		ddoc:   doc,
		bucket: bkt,
	}

	err = inst.putDesignDoc()
	if err != nil {
		return nil, err
	}

	return &inst, nil
}

func newDesignDoc(idxname string, on catalog.IndexKey) (*designdoc, error) {
	var doc designdoc

	doc.name = "ddl_" + idxname
	doc.viewname = idxname

	err := generateMap(on, &doc)
	if err != nil {
		return nil, err
	}

	err = generateReduce(on, &doc)
	if err != nil {
		return nil, err
	}

	return &doc, nil
}

func loadViewIndexes(b *bucket) ([]*viewIndex, error) {

	rows, err := b.cbbucket.GetDDocs()
	if err != nil {
		return nil, err
	}

	inames := make([]string, 0, len(rows.Rows))
	for _, row := range rows.Rows {
		cdoc := row.DDoc
		id := cdoc.Meta["id"].(string)
		if !strings.HasPrefix(id, "_design/ddl_") {
			continue
		}
		iname := strings.TrimPrefix(id, "_design/ddl_")
		inames = append(inames, iname)
	}

	indexes := make([]*viewIndex, 0, len(inames))
	for _, iname := range inames {
		ddname := "ddl_" + iname
		jdoc, err := getDesignDoc(b, ddname)
		if err != nil {
			fmt.Println("Warning - cannot fetch index: ", iname)
		}
		jview, ok := jdoc.Views[iname]
		if !ok {
			fmt.Println("Warning - missing view for index: ", iname)
		}
		ddoc := designdoc {
			name:     ddname,
			viewname: iname,
			mapfn:    jview.Map,
			reducefn: jview.Reduce,
		}
		if ddoc.checksum() != jdoc.IndexChecksum {
			fmt.Println("Warning - checksum failed on index: ", iname)
		}
		vidx := viewIndex {
			name:   iname,
			bucket: b,
			using:  catalog.VIEW,
			ddoc:   &ddoc,
		}
		indexes = append(indexes, &vidx)
	}
	return indexes, nil
}

func newPrimaryIndex(b *bucket, ddname string, view string) (*primaryIndex) {
	meta := ast.NewFunctionCall("meta", ast.FunctionArgExpressionList{})
	mdid := ast.NewDotMemberOperator(meta, ast.NewProperty("id"))
	name := "#primary"
	ddoc := designdoc{name: ddname, viewname: view}
	idx := primaryIndex{
		viewIndex{
			name:   name,
			using:  catalog.UNSPECIFIED,
			on:     catalog.IndexKey{mdid},
			ddoc:   &ddoc,
			bucket: b,
		},
	}
	return &idx
}

func generateMap(on catalog.IndexKey, doc *designdoc) error {

	buf := new(bytes.Buffer)

	fmt.Fprintln(buf, templStart)
	line := strings.Replace(templFunctions, "$null", strconv.Itoa(TYPE_NULL), -1)
	line = strings.Replace(line, "$boolean", strconv.Itoa(TYPE_BOOLEAN), -1)
	line = strings.Replace(line, "$number", strconv.Itoa(TYPE_NUMBER), -1)
	line = strings.Replace(line, "$string", strconv.Itoa(TYPE_STRING), -1)
	line = strings.Replace(line, "$array", strconv.Itoa(TYPE_ARRAY), -1)
	line = strings.Replace(line, "$object", strconv.Itoa(TYPE_OBJECT), -1)
	fmt.Fprintln(buf, line)

	keylist := new(bytes.Buffer)
	for idx, expr := range on {

		walker := NewWalker()
		_, err := walker.Visit(expr)
		if err != nil {
			return err
		}

		jvar := fmt.Sprintf("key%v", idx+1)
		line := strings.Replace(templExpr, "$var", jvar, -1)
		line = strings.Replace(line, "$path", walker.JS(), -1)
		fmt.Fprint(buf, line)

		if idx > 0 {
			fmt.Fprint(keylist, ", ")
		}
		fmt.Fprint(keylist, jvar)
	}

	line = strings.Replace(templKey, "$keylist", keylist.String(), -1)

	fmt.Fprint(buf, line)
	fmt.Fprint(buf, templEmit)
	fmt.Fprint(buf, templEnd)
	doc.mapfn = buf.String()

	return nil
}

func generateReduce(on catalog.IndexKey, doc *designdoc) error {
	doc.reducefn = ""
	return nil
}

func (idx *viewIndex) putDesignDoc() error {
	var view cb.ViewDefinition
	view.Map = idx.ddoc.mapfn

	var put ddocJSON
	put.Views = make(map[string]cb.ViewDefinition)
	put.Views[idx.name] = view
	put.IndexChecksum = idx.ddoc.checksum()

	put.IndexOn = make([]string, len(idx.on))
	for idx, expr := range idx.on {
		put.IndexOn[idx] = expr.String()
	}

	if err := idx.bucket.cbbucket.PutDDoc(idx.DDocName(), &put); err != nil {
		return err
	}

	saved, err := getDesignDoc(idx.bucket, idx.DDocName())
	if err != nil || saved.IndexChecksum != idx.ddoc.checksum() {
		return errors.New("Index creation failed, checksum mismatch: " + idx.name)
	}

	return nil
}

func (ddoc *designdoc) checksum() int {
	mapSum := crc32.ChecksumIEEE([]byte(ddoc.mapfn))
	reduceSum := crc32.ChecksumIEEE([]byte(ddoc.reducefn))
	return int(mapSum + reduceSum)
}

func getDesignDoc(b *bucket, ddocname string) (*ddocJSON, error) {
	var ddoc ddocJSON
	err := b.cbbucket.GetDDoc(ddocname, &ddoc)
	if err != nil {
		return nil, err
	}
	return &ddoc, nil
}

func (idx *viewIndex) DropViewIndex() error {
	if err := idx.bucket.cbbucket.DeleteDDoc(idx.DDocName()); err != nil {
		return err
	}
	return nil
}

// AST to JS conversion
type JsStatement struct {
	js bytes.Buffer
}

func NewWalker() *JsStatement {
	var js JsStatement
	return &js
}

func (this *JsStatement) JS() string {
	return this.js.String()
}

// inorder traversal of the AST to get JS expression out of it
func (this *JsStatement) Visit(e ast.Expression) (ast.Expression, error) {
	switch expr := e.(type) {

	case *ast.DotMemberOperator:
		if this.js.Len() == 0 {
			this.js.WriteString("doc.")
		}
		_, err := expr.Left.Accept(this)
		if err != nil {
			return nil, err
		}
		this.js.WriteString(".")
		_, err = expr.Right.Accept(this)
		if err != nil {
			return nil, err
		}

	case *ast.BracketMemberOperator:
		if this.js.Len() == 0 {
			this.js.WriteString("doc.")
		}
		_, err := expr.Left.Accept(this)
		if err != nil {
			return nil, err
		}
		this.js.WriteString("[")
		_, err = expr.Right.Accept(this)
		if err != nil {
			return nil, err
		}
		this.js.WriteString("]")

	case *ast.Property:
		if this.js.Len() == 0 {
			this.js.WriteString("doc.")
		}
		this.js.WriteString(expr.Path)

	case *ast.LiteralNumber:
		this.js.WriteString(fmt.Sprintf("%v", expr.Val))

	case *ast.LiteralString:
		this.js.WriteString(expr.Val)

	default:
		return e, errors.New("Expression is not supported by indexing currently: " + e.String())

	}
	return e, nil
}
<commit_msg>Properly save index expression<commit_after>package couchbase

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"hash/crc32"
	"strconv"
	"strings"

	cb "github.com/couchbaselabs/go-couchbase"
	"github.com/couchbaselabs/tuqtng/ast"
	"github.com/couchbaselabs/tuqtng/catalog"
)

type ddocJSON struct {
	cb.DDocJSON
	IndexOn       []string `json:"indexOn"`
	IndexChecksum int      `json:"indexChecksum"`
}

func newViewIndex(name string, on catalog.IndexKey, bkt *bucket) (*viewIndex, error) {

	doc, err := newDesignDoc(name, on)
	if err != nil {
		return nil, err
	}

	inst := viewIndex{
		name:   name,
		using:  catalog.VIEW,
		on:     on,
		ddoc:   doc,
		bucket: bkt,
	}

	err = inst.putDesignDoc()
	if err != nil {
		return nil, err
	}

	return &inst, nil
}

func (vi *viewIndex) String() string {
	var buf bytes.Buffer
	buf.WriteString(fmt.Sprintf("name: %v ", vi.name))
	buf.WriteString(fmt.Sprintf("on: %v ", vi.on))
	buf.WriteString(fmt.Sprintf("using: %v ", vi.using))
	buf.WriteString(fmt.Sprintf("ddoc: %v ", *vi.ddoc))
	buf.WriteString(fmt.Sprintf("bucket: %v ", *vi.bucket))
	return buf.String()
}

func newDesignDoc(idxname string, on catalog.IndexKey) (*designdoc, error) {
	var doc designdoc

	doc.name = "ddl_" + idxname
	doc.viewname = idxname

	err := generateMap(on, &doc)
	if err != nil {
		return nil, err
	}

	err = generateReduce(on, &doc)
	if err != nil {
		return nil, err
	}

	return &doc, nil
}

func loadViewIndexes(b *bucket) ([]*viewIndex, error) {

	rows, err := b.cbbucket.GetDDocs()
	if err != nil {
		return nil, err
	}

	inames := make([]string, 0, len(rows.Rows))
	for _, row := range rows.Rows {
		cdoc := row.DDoc
		id := cdoc.Meta["id"].(string)
		if !strings.HasPrefix(id, "_design/ddl_") {
			continue
		}
		iname := strings.TrimPrefix(id, "_design/ddl_")
		inames = append(inames, iname)
	}

	indexes := make([]*viewIndex, 0, len(inames))
	for _, iname := range inames {
		ddname := "ddl_" + iname
		jdoc, err := getDesignDoc(b, ddname)
		if err != nil {
			return nil, err
		}
		jview, ok := jdoc.Views[iname]
		if !ok {
			return nil, errors.New("Missing view for index " + iname)
		}

		exprlist := make([]ast.Expression, 0, len(jdoc.IndexOn))
		for _, ser := range jdoc.IndexOn {
			expr, err := ast.UnmarshalExpression([]byte(ser))
			if err != nil {
				return nil, errors.New("Cannot unmarshal expression for index " + iname)
			}
			exprlist = append(exprlist, expr)
		}
		if len(exprlist) != len(jdoc.IndexOn) {
			continue
		}

		ddoc := designdoc{
			name:     ddname,
			viewname: iname,
			mapfn:    jview.Map,
			reducefn: jview.Reduce,
		}
		if ddoc.checksum() != jdoc.IndexChecksum {
			return nil, errors.New("Warning - checksum failed on index " + iname)
		}

		vidx := viewIndex{
			name:   iname,
			bucket: b,
			using:  catalog.VIEW,
			ddoc:   &ddoc,
			on:     exprlist,
		}
		indexes = append(indexes, &vidx)
	}

	return indexes, nil
}

func newPrimaryIndex(b *bucket, ddname string, view string) *primaryIndex {
	meta := ast.NewFunctionCall("meta", ast.FunctionArgExpressionList{})
	mdid := ast.NewDotMemberOperator(meta, ast.NewProperty("id"))
	name := "#primary"
	ddoc := designdoc{name: ddname, viewname: view}
	idx := primaryIndex{
		viewIndex{
			name:   name,
			using:  catalog.VIEW,
			on:     catalog.IndexKey{mdid},
			ddoc:   &ddoc,
			bucket: b,
		},
	}
	return &idx
}

func generateMap(on catalog.IndexKey, doc *designdoc) error {

	buf := new(bytes.Buffer)

	fmt.Fprintln(buf, templStart)
	line := strings.Replace(templFunctions, "$null", strconv.Itoa(TYPE_NULL), -1)
	line = strings.Replace(line, "$boolean", strconv.Itoa(TYPE_BOOLEAN), -1)
	line = strings.Replace(line, "$number", strconv.Itoa(TYPE_NUMBER), -1)
	line = strings.Replace(line, "$string", strconv.Itoa(TYPE_STRING), -1)
	line = strings.Replace(line, "$array", strconv.Itoa(TYPE_ARRAY), -1)
	line = strings.Replace(line, "$object", strconv.Itoa(TYPE_OBJECT), -1)
	fmt.Fprintln(buf, line)

	keylist := new(bytes.Buffer)
	for idx, expr := range on {

		walker := NewWalker()
		_, err := walker.Visit(expr)
		if err != nil {
			return err
		}

		jvar := fmt.Sprintf("key%v", idx+1)
		line := strings.Replace(templExpr, "$var", jvar, -1)
		line = strings.Replace(line, "$path", walker.JS(), -1)
		fmt.Fprint(buf, line)

		if idx > 0 {
			fmt.Fprint(keylist, ", ")
		}
		fmt.Fprint(keylist, jvar)
	}

	line = strings.Replace(templKey, "$keylist", keylist.String(), -1)

	fmt.Fprint(buf, line)
	fmt.Fprint(buf, templEmit)
	fmt.Fprint(buf, templEnd)
	doc.mapfn = buf.String()

	return nil
}

func generateReduce(on catalog.IndexKey, doc *designdoc) error {
	doc.reducefn = ""
	return nil
}

func (idx *viewIndex) putDesignDoc() error {
	var view cb.ViewDefinition
	view.Map = idx.ddoc.mapfn

	var put ddocJSON
	put.Views = make(map[string]cb.ViewDefinition)
	put.Views[idx.name] = view
	put.IndexChecksum = idx.ddoc.checksum()

	put.IndexOn = make([]string, len(idx.on))
	for idx, expr := range idx.on {
		ser, err := json.Marshal(expr)
		if err != nil {
			return err
		}
		put.IndexOn[idx] = string(ser)
	}

	if err := idx.bucket.cbbucket.PutDDoc(idx.DDocName(), &put); err != nil {
		return err
	}

	saved, err := getDesignDoc(idx.bucket, idx.DDocName())
	if err != nil || saved.IndexChecksum != idx.ddoc.checksum() {
		return errors.New("Index creation failed, checksum mismatch: " + idx.name)
	}

	return nil
}

func (ddoc *designdoc) checksum() int {
	mapSum := crc32.ChecksumIEEE([]byte(ddoc.mapfn))
	reduceSum := crc32.ChecksumIEEE([]byte(ddoc.reducefn))
	return int(mapSum + reduceSum)
}

func getDesignDoc(b *bucket, ddocname string) (*ddocJSON, error) {
	var ddoc ddocJSON
	err := b.cbbucket.GetDDoc(ddocname, &ddoc)
	if err != nil {
		return nil, err
	}
	return &ddoc, nil
}

func (idx *viewIndex) DropViewIndex() error {
	if err := idx.bucket.cbbucket.DeleteDDoc(idx.DDocName()); err != nil {
		return err
	}
	return nil
}

// AST to JS conversion
type JsStatement struct {
	js bytes.Buffer
}

func NewWalker() *JsStatement {
	var js JsStatement
	return &js
}

func (this *JsStatement) JS() string {
	return this.js.String()
}

// inorder traversal of the AST to get JS expression out of it
func (this *JsStatement) Visit(e ast.Expression) (ast.Expression, error) {
	switch expr := e.(type) {

	case *ast.DotMemberOperator:
		if this.js.Len() == 0 {
			this.js.WriteString("doc.")
		}
		_, err := expr.Left.Accept(this)
		if err != nil {
			return nil, err
		}
		this.js.WriteString(".")
		_, err = expr.Right.Accept(this)
		if err != nil {
			return nil, err
		}

	case *ast.BracketMemberOperator:
		if this.js.Len() == 0 {
			this.js.WriteString("doc.")
		}
		_, err := expr.Left.Accept(this)
		if err != nil {
			return nil, err
		}
		this.js.WriteString("[")
		_, err = expr.Right.Accept(this)
		if err != nil {
			return nil, err
		}
		this.js.WriteString("]")

	case *ast.Property:
		if this.js.Len() == 0 {
			this.js.WriteString("doc.")
		}
		this.js.WriteString(expr.Path)

	case *ast.LiteralNumber:
		this.js.WriteString(fmt.Sprintf("%v", expr.Val))

	case *ast.LiteralString:
		this.js.WriteString(expr.Val)

	default:
		return e, errors.New("Expression is not supported by indexing currently: " + e.String())

	}
	return e, nil
}
<|endoftext|>


<commit_before>package formats

import (
	"archive/zip"
	"io"
	"strings"

	"github.com/anaminus/rbxmk"
	"github.com/anaminus/rbxmk/dump"
	"github.com/anaminus/rbxmk/reflect"
	"github.com/anaminus/rbxmk/rtypes"
	"github.com/robloxapi/types"
)

const F_Zip = "zip"

func init() { register(Zip) }
func Zip() rbxmk.Format {
	return rbxmk.Format{
		MediaTypes: []string{"application/zip"},
		Name:       F_Zip,
		CanDecode: func(g rtypes.Global, f rbxmk.FormatOptions, typeName string) bool {
			return typeName == rtypes.T_Table
		},
		Decode: func(g rtypes.Global, f rbxmk.FormatOptions, r io.Reader) (v types.Value, err error) {
			var rat io.ReaderAt
			var size int64
			if srat, ok := r.(interface {
				io.Seeker
				io.ReaderAt
			}); ok {
				if size, err = srat.Seek(0, io.SeekEnd); err == nil {
					if _, err := srat.Seek(0, io.SeekStart); err == nil {
						rat = srat
					}
				}
			}
			if rat == nil {
				b, err := io.ReadAll(r)
				if err != nil {
					return nil, err
				}
				rat = strings.NewReader(string(b))
				size = int64(len(b))
			}
			z, err := zip.NewReader(rat, size)
			if err != nil {
				return nil, err
			}

			zip := make(rtypes.Dictionary, len(z.File))
			for _, file := range z.File {
				if strings.HasSuffix(file.Name, "/") {
					// Skip directory
					continue
				}
				f, err := file.Open()
				if err != nil {
					continue
				}
				b, err := io.ReadAll(f)
				f.Close()
				if err != nil {
					continue
				}
				zip[file.Name] = types.BinaryString(b)
			}
			return zip, nil
		},
		Encode: func(g rtypes.Global, f rbxmk.FormatOptions, w io.Writer, v types.Value) error {
			z := v.(rtypes.Dictionary)
			zw := zip.NewWriter(w)
			for path, value := range z {
				content, ok := value.(types.BinaryString)
				if !ok {
					continue
				}
				zf, err := zw.CreateHeader(&zip.FileHeader{
					Name:   path,
					Method: zip.Deflate,
				})
				if err != nil {
					return err
				}
				if _, err := zf.Write([]byte(content)); err != nil {
					return err
				}
			}
			return zw.Close()
		},
		Dump: func() dump.Format {
			return dump.Format{
				Summary:     "Formats/zip:Summary",
				Description: "Formats/zip:Description",
			}
		},
		Types: []func() rbxmk.Reflector{
			reflect.Dictionary,
		},
	}
}
<commit_msg>Mark zip format as hidden.<commit_after>package formats

import (
	"archive/zip"
	"io"
	"strings"

	"github.com/anaminus/rbxmk"
	"github.com/anaminus/rbxmk/dump"
	"github.com/anaminus/rbxmk/reflect"
	"github.com/anaminus/rbxmk/rtypes"
	"github.com/robloxapi/types"
)

const F_Zip = "zip"

func init() { register(Zip) }
func Zip() rbxmk.Format {
	return rbxmk.Format{
		MediaTypes: []string{"application/zip"},
		Name:       F_Zip,
		CanDecode: func(g rtypes.Global, f rbxmk.FormatOptions, typeName string) bool {
			return typeName == rtypes.T_Table
		},
		Decode: func(g rtypes.Global, f rbxmk.FormatOptions, r io.Reader) (v types.Value, err error) {
			var rat io.ReaderAt
			var size int64
			if srat, ok := r.(interface {
				io.Seeker
				io.ReaderAt
			}); ok {
				if size, err = srat.Seek(0, io.SeekEnd); err == nil {
					if _, err := srat.Seek(0, io.SeekStart); err == nil {
						rat = srat
					}
				}
			}
			if rat == nil {
				b, err := io.ReadAll(r)
				if err != nil {
					return nil, err
				}
				rat = strings.NewReader(string(b))
				size = int64(len(b))
			}
			z, err := zip.NewReader(rat, size)
			if err != nil {
				return nil, err
			}

			zip := make(rtypes.Dictionary, len(z.File))
			for _, file := range z.File {
				if strings.HasSuffix(file.Name, "/") {
					// Skip directory
					continue
				}
				f, err := file.Open()
				if err != nil {
					continue
				}
				b, err := io.ReadAll(f)
				f.Close()
				if err != nil {
					continue
				}
				zip[file.Name] = types.BinaryString(b)
			}
			return zip, nil
		},
		Encode: func(g rtypes.Global, f rbxmk.FormatOptions, w io.Writer, v types.Value) error {
			z := v.(rtypes.Dictionary)
			zw := zip.NewWriter(w)
			for path, value := range z {
				content, ok := value.(types.BinaryString)
				if !ok {
					continue
				}
				zf, err := zw.CreateHeader(&zip.FileHeader{
					Name:   path,
					Method: zip.Deflate,
				})
				if err != nil {
					return err
				}
				if _, err := zf.Write([]byte(content)); err != nil {
					return err
				}
			}
			return zw.Close()
		},
		Dump: func() dump.Format {
			return dump.Format{
				Hidden:      true,
				Summary:     "Formats/zip:Summary",
				Description: "Formats/zip:Description",
			}
		},
		Types: []func() rbxmk.Reflector{
			reflect.Dictionary,
		},
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/binary"
	"log"
	"os"
	"sync"
	"time"

	"github.com/creiht/formic/flother"
	pb "github.com/creiht/formic/proto"
	"github.com/garyburd/redigo/redis"
	"github.com/spaolacci/murmur3"
	"golang.org/x/net/context"
)

type apiServer struct {
	sync.RWMutex
	ds        DirService
	fs        FileService
	fl        *flother.Flother
	blocksize int64
}

func NewApiServer(ds DirService, fs FileService) *apiServer {
	s := new(apiServer)
	s.ds = ds
	s.fs = fs
	// TODO: Get epoch and node id from some config
	s.fl = flother.NewFlother(time.Time{}, 1)
	s.blocksize = int64(1024 * 64) // Default Block Size (64K)
	return s
}

func (s *apiServer) GetID(custID, shareID, inode, block uint64) []byte {
	// TODO: Figure out what arrangement we want to use for the hash
	h := murmur3.New128()
	binary.Write(h, binary.BigEndian, custID)
	binary.Write(h, binary.BigEndian, shareID)
	binary.Write(h, binary.BigEndian, inode)
	binary.Write(h, binary.BigEndian, block)
	s1, s2 := h.Sum128()
	b := bytes.NewBuffer([]byte(""))
	binary.Write(b, binary.BigEndian, s1)
	binary.Write(b, binary.BigEndian, s2)
	return b.Bytes()
}

func (s *apiServer) GetAttr(ctx context.Context, r *pb.Node) (*pb.Attr, error) {
	return s.ds.GetAttr(r.Inode)
}

func (s *apiServer) SetAttr(ctx context.Context, r *pb.SetAttrRequest) (*pb.Attr, error) {
	return s.ds.SetAttr(r.Inode, r)
}

func (s *apiServer) Create(ctx context.Context, r *pb.DirEnt) (*pb.DirEnt, error) {
	ts := time.Now().Unix()
	inode := s.fl.GetID()
	attr := &pb.Attr{
		Inode:  inode,
		Atime:  ts,
		Mtime:  ts,
		Ctime:  ts,
		Crtime: ts,
		Mode:   r.Attr.Mode,
		Uid:    r.Attr.Uid,
		Gid:    r.Attr.Gid,
	}
	return s.ds.Create(r.Parent, inode, r.Name, attr, false)
}

func (s *apiServer) MkDir(ctx context.Context, r *pb.DirEnt) (*pb.DirEnt, error) {
	ts := time.Now().Unix()
	inode := s.fl.GetID()
	attr := &pb.Attr{
		Inode:  inode,
		Atime:  ts,
		Mtime:  ts,
		Ctime:  ts,
		Crtime: ts,
		Mode:   uint32(os.ModeDir) | r.Attr.Mode,
		Uid:    r.Attr.Uid,
		Gid:    r.Attr.Gid,
	}
	return s.ds.Create(r.Parent, inode, r.Name, attr, true)
}

func (s *apiServer) Read(ctx context.Context, r *pb.ReadRequest) (*pb.FileChunk, error) {
	log.Printf("READ: Inode: %d Offset: %d Size: %d", r.Inode, r.Offset, r.Size)
	block := uint64(r.Offset / s.blocksize)
	data := make([]byte, r.Size)
	firstOffset := int64(0)
	if r.Offset%s.blocksize != 0 {
		// Handle non-aligned offset
		firstOffset = r.Offset - int64(block)*s.blocksize
		log.Printf("Offset: %d, firstOffset: %d", r.Offset, firstOffset)
	}
	cur := int64(0)
	for cur < r.Size {
		id := s.GetID(1, 1, r.Inode, block)
		log.Printf("Reading Inode: %d, Block: %d ID: %d", r.Inode, block, id)
		chunk, err := s.fs.GetChunk(id)
		log.Printf("LEN: %d", len(chunk))
		if err != nil {
			if err == redis.ErrNil {
				// TODO: Handle failures to get block better
				log.Printf("Err: Failed to read block")
				return &pb.FileChunk{}, nil
			}
			log.Print("Err: Failed to read block: ", err)
			return &pb.FileChunk{}, err
		}
		if len(chunk) == 0 {
			log.Printf("Err: Read 0 Bytes")
			break
		}
		count := copy(data[cur:], chunk[firstOffset:])
		firstOffset = 0
		block += 1
		cur += int64(len(chunk))
		log.Printf("Read %d bytes", count)
		if int64(len(chunk)) < s.blocksize {
			break
		}
	}
	f := &pb.FileChunk{Inode: r.Inode, Payload: data}
	return f, nil
}

func min(a, b int64) int64 {
	if a < b {
		return a
	} else {
		return b
	}
}

func (s *apiServer) Write(ctx context.Context, r *pb.FileChunk) (*pb.WriteResponse, error) {
	block := uint64(r.Offset / s.blocksize)
	// TODO: Handle unaligned offsets
	firstOffset := int64(0)
	if r.Offset%s.blocksize != 0 {
		// Handle non-aligned offset
		firstOffset = r.Offset - int64(block)*s.blocksize
		log.Printf("Offset: %d, firstOffset: %d", r.Offset, firstOffset)
	}
	cur := int64(0)
	for cur < int64(len(r.Payload)) {
		sendSize := min(s.blocksize, int64(len(r.Payload))-cur)
		payload := r.Payload[cur : cur+sendSize]
		id := s.GetID(1, 1, r.Inode, block)
		if firstOffset > 0 || sendSize < s.blocksize {
			// need to get the block and update
			chunk := make([]byte, firstOffset+int64(len(payload)))
			data, err := s.fs.GetChunk(id)
			if firstOffset > 0 && (err != nil || len(data) == 0) {
				// TODO: Need better error handling for when there is a block but it can't retreive it
				log.Printf("ERR: couldn't get block id %d", id)
			} else {
				if len(data) > len(chunk) {
					chunk = data
				} else {
					copy(chunk, data)
				}
			}
			log.Printf("DATA LEN: %d CHUNK LEN: %d", len(data), len(chunk))
			copy(chunk[firstOffset:], payload)
			payload = chunk
			firstOffset = 0
		}
		log.Printf("Writing Inode: %d Block: %d ID: %d Len: %d", r.Inode, block, id, sendSize)
		err := s.fs.WriteChunk(id, payload)
		// TODO: Need better error handling for failing with multiple chunks
		if err != nil {
			return &pb.WriteResponse{Status: 1}, err
		}
		s.ds.Update(r.Inode, block, uint64(s.blocksize), uint64(len(payload)), time.Now().Unix())
		cur += sendSize
		block += 1
	}
	return &pb.WriteResponse{Status: 0}, nil
}

func (s *apiServer) Lookup(ctx context.Context, r *pb.LookupRequest) (*pb.DirEnt, error) {
	return s.ds.Lookup(r.Parent, r.Name)
}

func (s *apiServer) ReadDirAll(ctx context.Context, n *pb.Node) (*pb.DirEntries, error) {
	return s.ds.ReadDirAll(n.Inode)
}

func (s *apiServer) Remove(ctx context.Context, r *pb.DirEnt) (*pb.WriteResponse, error) {
	// TODO: Add calls to remove from backing store
	return s.ds.Remove(r.Parent, r.Name)
}

func (s *apiServer) Symlink(ctx context.Context, r *pb.SymlinkRequest) (*pb.DirEnt, error) {
	ts := time.Now().Unix()
	inode := s.fl.GetID()
	attr := &pb.Attr{
		Inode:  inode,
		Atime:  ts,
		Mtime:  ts,
		Ctime:  ts,
		Crtime: ts,
		Mode:   uint32(os.ModeSymlink | 0777),
		Size:   uint64(len(r.Target)),
	}
	return s.ds.Symlink(r.Parent, r.Name, r.Target, attr, inode)
}

func (s *apiServer) Readlink(ctx context.Context, n *pb.Node) (*pb.ReadlinkResponse, error) {
	return s.ds.Readlink(n.Inode)
}

func (s *apiServer) Getxattr(ctx context.Context, r *pb.GetxattrRequest) (*pb.GetxattrResponse, error) {
	return s.ds.Getxattr(r)
}

func (s *apiServer) Setxattr(ctx context.Context, r *pb.SetxattrRequest) (*pb.SetxattrResponse, error) {
	return s.ds.Setxattr(r)
}

func (s *apiServer) Listxattr(ctx context.Context, r *pb.ListxattrRequest) (*pb.ListxattrResponse, error) {
	return s.ds.Listxattr(r)
}

func (s *apiServer) Removexattr(ctx context.Context, r *pb.RemovexattrRequest) (*pb.RemovexattrResponse, error) {
	return s.ds.Removexattr(r)
}

func (s *apiServer) Rename(ctx context.Context, r *pb.RenameRequest) (*pb.RenameResponse, error) {
	return s.ds.Rename(r)
}

func (s *apiServer) Statfs(ctx context.Context, r *pb.StatfsRequest) (*pb.StatfsResponse, error) {
	resp := &pb.StatfsResponse{
		Blocks:  281474976710656, // 1 exabyte (asuming 4K block size)
		Bfree:   281474976710656,
		Bavail:  281474976710656,
		Files:   1000000000000, // 1 trillion inodes
		Ffree:   1000000000000,
		Bsize:   4096, // it looked like ext4 used 4KB blocks
		Namelen: 256,
		Frsize:  4096, // this should probably match Bsize so we don't allow fragmented blocks
	}
	return resp, nil
}
<commit_msg>Fix for offset write crossing a chunk boundary<commit_after>package main

import (
	"bytes"
	"encoding/binary"
	"log"
	"os"
	"sync"
	"time"

	"github.com/creiht/formic/flother"
	pb "github.com/creiht/formic/proto"
	"github.com/garyburd/redigo/redis"
	"github.com/spaolacci/murmur3"
	"golang.org/x/net/context"
)

type apiServer struct {
	sync.RWMutex
	ds        DirService
	fs        FileService
	fl        *flother.Flother
	blocksize int64
}

func NewApiServer(ds DirService, fs FileService) *apiServer {
	s := new(apiServer)
	s.ds = ds
	s.fs = fs
	// TODO: Get epoch and node id from some config
	s.fl = flother.NewFlother(time.Time{}, 1)
	s.blocksize = int64(1024 * 64) // Default Block Size (64K)
	return s
}

func (s *apiServer) GetID(custID, shareID, inode, block uint64) []byte {
	// TODO: Figure out what arrangement we want to use for the hash
	h := murmur3.New128()
	binary.Write(h, binary.BigEndian, custID)
	binary.Write(h, binary.BigEndian, shareID)
	binary.Write(h, binary.BigEndian, inode)
	binary.Write(h, binary.BigEndian, block)
	s1, s2 := h.Sum128()
	b := bytes.NewBuffer([]byte(""))
	binary.Write(b, binary.BigEndian, s1)
	binary.Write(b, binary.BigEndian, s2)
	return b.Bytes()
}

func (s *apiServer) GetAttr(ctx context.Context, r *pb.Node) (*pb.Attr, error) {
	return s.ds.GetAttr(r.Inode)
}

func (s *apiServer) SetAttr(ctx context.Context, r *pb.SetAttrRequest) (*pb.Attr, error) {
	return s.ds.SetAttr(r.Inode, r)
}

func (s *apiServer) Create(ctx context.Context, r *pb.DirEnt) (*pb.DirEnt, error) {
	ts := time.Now().Unix()
	inode := s.fl.GetID()
	attr := &pb.Attr{
		Inode:  inode,
		Atime:  ts,
		Mtime:  ts,
		Ctime:  ts,
		Crtime: ts,
		Mode:   r.Attr.Mode,
		Uid:    r.Attr.Uid,
		Gid:    r.Attr.Gid,
	}
	return s.ds.Create(r.Parent, inode, r.Name, attr, false)
}

func (s *apiServer) MkDir(ctx context.Context, r *pb.DirEnt) (*pb.DirEnt, error) {
	ts := time.Now().Unix()
	inode := s.fl.GetID()
	attr := &pb.Attr{
		Inode:  inode,
		Atime:  ts,
		Mtime:  ts,
		Ctime:  ts,
		Crtime: ts,
		Mode:   uint32(os.ModeDir) | r.Attr.Mode,
		Uid:    r.Attr.Uid,
		Gid:    r.Attr.Gid,
	}
	return s.ds.Create(r.Parent, inode, r.Name, attr, true)
}

func (s *apiServer) Read(ctx context.Context, r *pb.ReadRequest) (*pb.FileChunk, error) {
	log.Printf("READ: Inode: %d Offset: %d Size: %d", r.Inode, r.Offset, r.Size)
	block := uint64(r.Offset / s.blocksize)
	data := make([]byte, r.Size)
	firstOffset := int64(0)
	if r.Offset%s.blocksize != 0 {
		// Handle non-aligned offset
		firstOffset = r.Offset - int64(block)*s.blocksize
		log.Printf("Offset: %d, firstOffset: %d", r.Offset, firstOffset)
	}
	cur := int64(0)
	for cur < r.Size {
		id := s.GetID(1, 1, r.Inode, block)
		log.Printf("Reading Inode: %d, Block: %d ID: %d", r.Inode, block, id)
		chunk, err := s.fs.GetChunk(id)
		log.Printf("LEN: %d", len(chunk))
		if err != nil {
			if err == redis.ErrNil {
				// TODO: Handle failures to get block better
				log.Printf("Err: Failed to read block")
				return &pb.FileChunk{}, nil
			}
			log.Print("Err: Failed to read block: ", err)
			return &pb.FileChunk{}, err
		}
		if len(chunk) == 0 {
			log.Printf("Err: Read 0 Bytes")
			break
		}
		count := copy(data[cur:], chunk[firstOffset:])
		firstOffset = 0
		block += 1
		cur += int64(len(chunk))
		log.Printf("Read %d bytes", count)
		if int64(len(chunk)) < s.blocksize {
			break
		}
	}
	f := &pb.FileChunk{Inode: r.Inode, Payload: data}
	return f, nil
}

func min(a, b int64) int64 {
	if a < b {
		return a
	} else {
		return b
	}
}

func (s *apiServer) Write(ctx context.Context, r *pb.FileChunk) (*pb.WriteResponse, error) {
	block := uint64(r.Offset / s.blocksize)
	// TODO: Handle unaligned offsets
	firstOffset := int64(0)
	if r.Offset%s.blocksize != 0 {
		// Handle non-aligned offset
		firstOffset = r.Offset - int64(block)*s.blocksize
		log.Printf("Offset: %d, firstOffset: %d", r.Offset, firstOffset)
	}
	cur := int64(0)
	for cur < int64(len(r.Payload)) {
		sendSize := min(s.blocksize, int64(len(r.Payload))-cur)
		if sendSize+firstOffset > s.blocksize {
			sendSize = s.blocksize - firstOffset
		}
		payload := r.Payload[cur : cur+sendSize]
		id := s.GetID(1, 1, r.Inode, block)
		if firstOffset > 0 || sendSize < s.blocksize {
			// need to get the block and update
			chunk := make([]byte, firstOffset+int64(len(payload)))
			data, err := s.fs.GetChunk(id)
			if firstOffset > 0 && (err != nil || len(data) == 0) {
				// TODO: Need better error handling for when there is a block but it can't retreive it
				log.Printf("ERR: couldn't get block id %d", id)
			} else {
				if len(data) > len(chunk) {
					chunk = data
				} else {
					copy(chunk, data)
				}
			}
			log.Printf("DATA LEN: %d CHUNK LEN: %d", len(data), len(chunk))
			copy(chunk[firstOffset:], payload)
			payload = chunk
			firstOffset = 0
		}
		log.Printf("Writing Inode: %d Block: %d ID: %d Len: %d", r.Inode, block, id, sendSize)
		err := s.fs.WriteChunk(id, payload)
		// TODO: Need better error handling for failing with multiple chunks
		if err != nil {
			return &pb.WriteResponse{Status: 1}, err
		}
		s.ds.Update(r.Inode, block, uint64(s.blocksize), uint64(len(payload)), time.Now().Unix())
		cur += sendSize
		block += 1
	}
	return &pb.WriteResponse{Status: 0}, nil
}

func (s *apiServer) Lookup(ctx context.Context, r *pb.LookupRequest) (*pb.DirEnt, error) {
	return s.ds.Lookup(r.Parent, r.Name)
}

func (s *apiServer) ReadDirAll(ctx context.Context, n *pb.Node) (*pb.DirEntries, error) {
	return s.ds.ReadDirAll(n.Inode)
}

func (s *apiServer) Remove(ctx context.Context, r *pb.DirEnt) (*pb.WriteResponse, error) {
	// TODO: Add calls to remove from backing store
	return s.ds.Remove(r.Parent, r.Name)
}

func (s *apiServer) Symlink(ctx context.Context, r *pb.SymlinkRequest) (*pb.DirEnt, error) {
	ts := time.Now().Unix()
	inode := s.fl.GetID()
	attr := &pb.Attr{
		Inode:  inode,
		Atime:  ts,
		Mtime:  ts,
		Ctime:  ts,
		Crtime: ts,
		Mode:   uint32(os.ModeSymlink | 0777),
		Size:   uint64(len(r.Target)),
	}
	return s.ds.Symlink(r.Parent, r.Name, r.Target, attr, inode)
}

func (s *apiServer) Readlink(ctx context.Context, n *pb.Node) (*pb.ReadlinkResponse, error) {
	return s.ds.Readlink(n.Inode)
}

func (s *apiServer) Getxattr(ctx context.Context, r *pb.GetxattrRequest) (*pb.GetxattrResponse, error) {
	return s.ds.Getxattr(r)
}

func (s *apiServer) Setxattr(ctx context.Context, r *pb.SetxattrRequest) (*pb.SetxattrResponse, error) {
	return s.ds.Setxattr(r)
}

func (s *apiServer) Listxattr(ctx context.Context, r *pb.ListxattrRequest) (*pb.ListxattrResponse, error) {
	return s.ds.Listxattr(r)
}

func (s *apiServer) Removexattr(ctx context.Context, r *pb.RemovexattrRequest) (*pb.RemovexattrResponse, error) {
	return s.ds.Removexattr(r)
}

func (s *apiServer) Rename(ctx context.Context, r *pb.RenameRequest) (*pb.RenameResponse, error) {
	return s.ds.Rename(r)
}

func (s *apiServer) Statfs(ctx context.Context, r *pb.StatfsRequest) (*pb.StatfsResponse, error) {
	resp := &pb.StatfsResponse{
		Blocks:  281474976710656, // 1 exabyte (asuming 4K block size)
		Bfree:   281474976710656,
		Bavail:  281474976710656,
		Files:   1000000000000, // 1 trillion inodes
		Ffree:   1000000000000,
		Bsize:   4096, // it looked like ext4 used 4KB blocks
		Namelen: 256,
		Frsize:  4096, // this should probably match Bsize so we don't allow fragmented blocks
	}
	return resp, nil
}
<|endoftext|>


<commit_before>package eval

import (
	"errors"
	"fmt"

	"github.com/elves/elvish/glob"
)

// GlobPattern is en ephemeral Value generated when evaluating tilde and
// wildcards.
type GlobPattern glob.Pattern

var (
	_ Value   = GlobPattern{}
	_ Indexer = GlobPattern{}
)

var (
	ErrMustFollowWildcard   = errors.New("must follow wildcard")
	ErrModifierMustBeString = errors.New("modifier must be string")
)

func (GlobPattern) Kind() string {
	return "glob-pattern"
}

func (gp GlobPattern) Repr(int) string {
	return fmt.Sprintf("<GlobPattern%v>", gp)
}

func (gp GlobPattern) Index(modifiers []Value) []Value {
	for _, value := range modifiers {
		modifier, ok := value.(String)
		if !ok {
			throw(ErrModifierMustBeString)
		}
		switch string(modifier) {
		case "a", "all":
			if len(gp.Segments) == 0 {
				throw(ErrBadGlobPattern)
			}
			if !glob.IsWild(gp.Segments[len(gp.Segments)-1]) {
				throw(ErrMustFollowWildcard)
			}
			gp.Segments[len(gp.Segments)-1] = glob.Wild{
				gp.Segments[len(gp.Segments)-1].(glob.Wild).Type, true,
			}
		default:
			throw(fmt.Errorf("unknown modifier %s", modifier.Repr(NoPretty)))
		}
	}
	return []Value{gp}
}

func (gp *GlobPattern) append(segs ...glob.Segment) {
	gp.Segments = append(gp.Segments, segs...)
}

func wildcardToSegment(s string) glob.Segment {
	switch s {
	case "*":
		return glob.Wild{glob.Star, false}
	case "**":
		return glob.Wild{glob.StarStar, false}
	case "?":
		return glob.Wild{glob.Question, false}
	default:
		throw(fmt.Errorf("bad wildcard: %q", s))
		panic("unreachable")
	}
}

func stringToSegments(s string) []glob.Segment {
	segs := []glob.Segment{}
	for i := 0; i < len(s); {
		j := i
		for ; j < len(s) && s[j] != '/'; j++ {
		}
		if j > i {
			segs = append(segs, glob.Literal{s[i:j]})
		}
		if j < len(s) {
			for ; j < len(s) && s[j] == '/'; j++ {
			}
			segs = append(segs, glob.Slash{})
			i = j
		} else {
			break
		}
	}
	return segs
}

func doGlob(gp GlobPattern, abort <-chan struct{}) []Value {
	names := []string{}
	if !glob.Pattern(gp).Glob(func(name string) bool {
		select {
		case <-abort:
			Logger.Println("glob aborted")
			return false
		default:
		}
		names = append(names, name)
		return true
	}) {
		throw(ErrInterrupted)
	}
	vs := make([]Value, len(names))
	for i, name := range names {
		vs[i] = String(name)
	}
	return vs
}
<commit_msg>Make doGlob less stupid.<commit_after>package eval

import (
	"errors"
	"fmt"

	"github.com/elves/elvish/glob"
)

// GlobPattern is en ephemeral Value generated when evaluating tilde and
// wildcards.
type GlobPattern glob.Pattern

var (
	_ Value   = GlobPattern{}
	_ Indexer = GlobPattern{}
)

var (
	ErrMustFollowWildcard   = errors.New("must follow wildcard")
	ErrModifierMustBeString = errors.New("modifier must be string")
)

func (GlobPattern) Kind() string {
	return "glob-pattern"
}

func (gp GlobPattern) Repr(int) string {
	return fmt.Sprintf("<GlobPattern%v>", gp)
}

func (gp GlobPattern) Index(modifiers []Value) []Value {
	for _, value := range modifiers {
		modifier, ok := value.(String)
		if !ok {
			throw(ErrModifierMustBeString)
		}
		switch string(modifier) {
		case "a", "all":
			if len(gp.Segments) == 0 {
				throw(ErrBadGlobPattern)
			}
			if !glob.IsWild(gp.Segments[len(gp.Segments)-1]) {
				throw(ErrMustFollowWildcard)
			}
			gp.Segments[len(gp.Segments)-1] = glob.Wild{
				gp.Segments[len(gp.Segments)-1].(glob.Wild).Type, true,
			}
		default:
			throw(fmt.Errorf("unknown modifier %s", modifier.Repr(NoPretty)))
		}
	}
	return []Value{gp}
}

func (gp *GlobPattern) append(segs ...glob.Segment) {
	gp.Segments = append(gp.Segments, segs...)
}

func wildcardToSegment(s string) glob.Segment {
	switch s {
	case "*":
		return glob.Wild{glob.Star, false}
	case "**":
		return glob.Wild{glob.StarStar, false}
	case "?":
		return glob.Wild{glob.Question, false}
	default:
		throw(fmt.Errorf("bad wildcard: %q", s))
		panic("unreachable")
	}
}

func stringToSegments(s string) []glob.Segment {
	segs := []glob.Segment{}
	for i := 0; i < len(s); {
		j := i
		for ; j < len(s) && s[j] != '/'; j++ {
		}
		if j > i {
			segs = append(segs, glob.Literal{s[i:j]})
		}
		if j < len(s) {
			for ; j < len(s) && s[j] == '/'; j++ {
			}
			segs = append(segs, glob.Slash{})
			i = j
		} else {
			break
		}
	}
	return segs
}

func doGlob(gp GlobPattern, abort <-chan struct{}) []Value {
	vs := make([]Value)
	if !glob.Pattern(gp).Glob(func(name string) bool {
		select {
		case <-abort:
			Logger.Println("glob aborted")
			return false
		default:
		}
		vs = append(vs, String(name))
		return true
	}) {
		throw(ErrInterrupted)
	}
	return vs
}
<|endoftext|>


<commit_before>package makeupdater

// Note this package is exclusively compatible with Drupal 7 make files.

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
	"sync"
)

// replaceTextInFile will replace a string of test in a file.
func replaceTextInFile(fullPath string, oldString string, newString string) {
	read, err := ioutil.ReadFile(fullPath)
	if err != nil {
		panic(err)
	}
	newContents := strings.Replace(string(read), oldString, newString, -1)
	err = ioutil.WriteFile(fullPath, []byte(newContents), 0)
	if err != nil {
		panic(err)
	}
}

// removeChar will remove particular characters from a string.
func removeChar(input string, chars ...string) string {
	for _, value := range chars {
		input = strings.Replace(input, value, "", -1)
	}
	return input
}

// inArray will return the quanity of specific input values in the input slice.
func inArray(input []string, subject string) int {
	counter := 0
	for _, value := range input {
		if value == subject {
			counter++
		}
	}
	return counter
}

// UpdateMake will update the version numbers in a specified make file
func UpdateMake(fullpath string) {
	_, err := os.Stat(fullpath)
	if err != nil {
		panic(err)
	}
	projects := GetProjectsFromMake(fullpath)
	count := 0
	wg := sync.WaitGroup{}
	for _, project := range projects {
		go func(project string) {
			wg.Add(1)
			if project != "" {
				catCmd := "cat " + fullpath + " | grep \"projects\\[" + project + "\\]\" | grep version | cut -d '=' -f2"
				z, _ := exec.Command("sh", "-c", catCmd).Output()
				for _, stream := range strings.Split(string(z), "\n") {
					stream = strings.Replace(stream, " ", "", -1)
					stream = strings.Replace(stream, "\"", "", -1)
					if stream != "" {
						x, _ := exec.Command("sh", "-c", "drush pm-releases --pipe "+project+" | grep Recommended | cut -d',' -f2").Output()
						versionNew := removeChar(string(x), " ", "7.x-", "\"", "\n", "[", "]")
						if !strings.Contains(stream, versionNew) {
							fmt.Printf("Replacing %v v%v with v%v\n", project, stream, versionNew)
							replaceTextInFile(fullpath, fmt.Sprintf("projects[%v][version] = \"%v\"\n", project, stream), fmt.Sprintf("projects[%v][version] = \"%v\"\n", project, versionNew))
							count++
						}
					}
				}
			}
			wg.Done()
		}(project)
	}
	wg.Wait()
	if count == 0 {
		fmt.Printf("%v is already up to date.\n", fullpath)
	} else {
		fmt.Printf("Updated %v projects in %v.\n", count, fullpath)
	}
}

// FindDuplicatesInMake will find and report Duplicate projects in Drupal make files.
// It will not return a value.
func FindDuplicatesInMake(makefile string) {
	projects := GetProjectsFromMake(makefile)
	// Run a short report containing information on all duplicates.
	for _, project := range projects {
		projectCounter := 0
		if project != "" {
			catCmd := "cat " + makefile + " | grep \"projects\\[" + project + "\\]\" | grep version | cut -d '=' -f2"
			z, _ := exec.Command("sh", "-c", catCmd).Output()
			for _, stream := range strings.Split(string(z), "\n") {
				if stream != "" {
					projectCounter ++
				}
			}
			if projectCounter > 1 {
				fmt.Printf("Found %v instances of project %v\n", projectCounter, project)
			}
		}
	}
}

// GetProjectsFromMake returns a list of projects from a given make file
func GetProjectsFromMake(fullpath string) []string {
	Projects := []string{}
	catCmd := "cat " + fullpath + " | grep projects | cut -d'[' -f2 | cut -d']' -f1 | uniq | sort"
	y, _ := exec.Command("sh", "-c", catCmd).Output()
	rawProjects := strings.Split(string(y), "\n")
	for _, project := range rawProjects {
		project = strings.Replace(project, " ", "", -1)
		if project != "" && project != "projects" {
			if inArray(Projects, project) == 0 {
				Projects = append(Projects, project)
			}
		}
	}
	return Projects
}

// GenerateMake takes a []string of projects and writes out a make file
// Modules are added with the latest recommended version.
func GenerateMake(Projects []string, File string) {
	headerLines := []string{}
	headerLines = append(headerLines, "; Generated by make-updater")
	headerLines = append(headerLines, "; Script created by Fubarhouse")
	headerLines = append(headerLines, "; Toolkit available at github.com/fubarhouse/golang-drush/...")
	headerLines = append(headerLines, "core = 7.x")
	headerLines = append(headerLines, "api = 2")
	headerLines = append(headerLines, "")

	// Rewrite core, if core is in the original Projects list.

	for _, Project := range Projects {
		coreAppended := 0
		if Project == "drupal" {
			if coreAppended == 0 {
				headerLines = append(headerLines, "; core")
				x, _ := exec.Command("sh", "-c", "drush pm-releases --pipe drupal | grep Recommended | cut -d',' -f2").Output()
				ProjectVersion := removeChar(string(x), " ", "7.x-", "\"", "\n", "[", "]")
				headerLines = append(headerLines, "projects[drupal][type] = \"core\"")
				headerLines = append(headerLines, fmt.Sprintf("projects[drupal][version] = \"%v\"", ProjectVersion))
				headerLines = append(headerLines, "projects[drupal][download][type] = \"get\"")
				headerLines = append(headerLines, fmt.Sprintf("projects[drupal][download][url] = \"https://ftp.drupal.org/files/projects/drupal-%v.tar.gz\"", ProjectVersion))
				headerLines = append(headerLines, "")
				coreAppended++
			}
		}
	}

	// Rewrite contrib
	headerLines = append(headerLines, "; modules")
	headerLines = append(headerLines, "defaults[projects][subdir] = contrib")
	headerLines = append(headerLines, "")

	for _, Project := range Projects {

		if Project != "drupal" {
			x, y := exec.Command("sh", "-c", "drush pm-releases --pipe "+Project+" | grep Recommended | cut -d',' -f2").Output()
			if y == nil {
				ProjectVersion := removeChar(string(x), " ", "7.x-", "\"", "\n", "[", "]")
				ProjectType := "contrib"
				if ProjectVersion == "" {
					ProjectType = "custom"
				}
				headerLines = append(headerLines, fmt.Sprintf("projects[%v][version] = \"%v\"", Project, ProjectVersion))
				headerLines = append(headerLines, fmt.Sprintf("projects[%v][type] = \"module\"", Project))
				headerLines = append(headerLines, fmt.Sprintf("projects[%v][subdir] = \"%v\"", Project, ProjectType))
				headerLines = append(headerLines, fmt.Sprint(""))
			}
		}
	}

	// Print to path File

	newFile, _ := os.Create(File)
	for _, line := range headerLines {
		fmt.Fprintln(newFile, line)
	}
	newFile.Sync()
	defer newFile.Close()

}
<commit_msg>Small bugfixes in the make updater; better comparrison for version checking.<commit_after>package makeupdater

// Note this package is exclusively compatible with Drupal 7 make files.

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
	"sync"
)

// replaceTextInFile will replace a string of test in a file.
func replaceTextInFile(fullPath string, oldString string, newString string) {
	read, err := ioutil.ReadFile(fullPath)
	if err != nil {
		panic(err)
	}
	newContents := strings.Replace(string(read), oldString, newString, -1)
	err = ioutil.WriteFile(fullPath, []byte(newContents), 0)
	if err != nil {
		panic(err)
	}
}

// removeChar will remove particular characters from a string.
func removeChar(input string, chars ...string) string {
	for _, value := range chars {
		input = strings.Replace(input, value, "", -1)
	}
	return input
}

// inArray will return the quanity of specific input values in the input slice.
func inArray(input []string, subject string) int {
	counter := 0
	for _, value := range input {
		if value == subject {
			counter++
		}
	}
	return counter
}

// UpdateMake will update the version numbers in a specified make file
func UpdateMake(fullpath string) {
	_, err := os.Stat(fullpath)
	if err != nil {
		panic(err)
	}
	projects := GetProjectsFromMake(fullpath)
	count := 0
	wg := sync.WaitGroup{}
	for _, project := range projects {
		go func(project string) {
			if project != "" {
				wg.Add(1)
				catCmd := "cat " + fullpath + " | grep \"projects\\[" + project + "\\]\" | grep version | cut -d '=' -f2"
				z, _ := exec.Command("sh", "-c", catCmd).Output()
				for _, stream := range strings.Split(string(z), "\n") {
					stream = strings.Replace(stream, "\"", "", -1)
					stream = strings.Replace(stream, " ", "", -1)
					if stream != "" {
						x, _ := exec.Command("sh", "-c", "drush pm-releases --pipe "+project+" | grep Recommended | cut -d',' -f2").Output()
						versionNew := removeChar(string(x), " ", "7.x-", "\"", "\n", "[", "]")
						if stream != versionNew {
							fmt.Printf("Replacing %v v%v with v%v\n", project, stream, versionNew)
							replaceTextInFile(fullpath, fmt.Sprintf("projects[%v][version] = \"%v\"\n", project, stream), fmt.Sprintf("projects[%v][version] = \"%v\"\n", project, versionNew))
							count++
						}
					}
				}
			}
			wg.Done()
		}(project)
	}
	wg.Wait()
	if count == 0 {
		fmt.Printf("%v is already up to date.\n", fullpath)
	} else {
		fmt.Printf("Updated %v projects in %v.\n", count, fullpath)
	}
}

// FindDuplicatesInMake will find and report Duplicate projects in Drupal make files.
// It will not return a value.
func FindDuplicatesInMake(makefile string) {
	projects := GetProjectsFromMake(makefile)
	// Run a short report containing information on all duplicates.
	for _, project := range projects {
		projectCounter := 0
		if project != "" {
			catCmd := "cat " + makefile + " | grep \"projects\\[" + project + "\\]\" | grep version | cut -d '=' -f2"
			z, _ := exec.Command("sh", "-c", catCmd).Output()
			for _, stream := range strings.Split(string(z), "\n") {
				if stream != "" {
					projectCounter ++
				}
			}
			if projectCounter > 1 {
				fmt.Printf("Found %v instances of project %v\n", projectCounter, project)
			}
		}
	}
}

// GetProjectsFromMake returns a list of projects from a given make file
func GetProjectsFromMake(fullpath string) []string {
	Projects := []string{}
	catCmd := "cat " + fullpath + " | grep projects | cut -d'[' -f2 | cut -d']' -f1 | uniq | sort"
	y, _ := exec.Command("sh", "-c", catCmd).Output()
	rawProjects := strings.Split(string(y), "\n")
	for _, project := range rawProjects {
		project = strings.Replace(project, " ", "", -1)
		if project != "" && project != "projects" {
			if inArray(Projects, project) == 0 {
				Projects = append(Projects, project)
			}
		}
	}
	return Projects
}

// GenerateMake takes a []string of projects and writes out a make file
// Modules are added with the latest recommended version.
func GenerateMake(Projects []string, File string) {
	headerLines := []string{}
	headerLines = append(headerLines, "; Generated by make-updater")
	headerLines = append(headerLines, "; Script created by Fubarhouse")
	headerLines = append(headerLines, "; Toolkit available at github.com/fubarhouse/golang-drush/...")
	headerLines = append(headerLines, "core = 7.x")
	headerLines = append(headerLines, "api = 2")
	headerLines = append(headerLines, "")

	// Rewrite core, if core is in the original Projects list.

	for _, Project := range Projects {
		coreAppended := 0
		if Project == "drupal" {
			if coreAppended == 0 {
				headerLines = append(headerLines, "; core")
				x, _ := exec.Command("sh", "-c", "drush pm-releases --pipe drupal | grep Recommended | cut -d',' -f2").Output()
				ProjectVersion := removeChar(string(x), " ", "7.x-", "\"", "\n", "[", "]")
				headerLines = append(headerLines, "projects[drupal][type] = \"core\"")
				headerLines = append(headerLines, fmt.Sprintf("projects[drupal][version] = \"%v\"", ProjectVersion))
				headerLines = append(headerLines, "projects[drupal][download][type] = \"get\"")
				headerLines = append(headerLines, fmt.Sprintf("projects[drupal][download][url] = \"https://ftp.drupal.org/files/projects/drupal-%v.tar.gz\"", ProjectVersion))
				headerLines = append(headerLines, "")
				coreAppended++
			}
		}
	}

	// Rewrite contrib
	headerLines = append(headerLines, "; modules")
	headerLines = append(headerLines, "defaults[projects][subdir] = contrib")
	headerLines = append(headerLines, "")

	for _, Project := range Projects {

		if Project != "drupal" {
			x, y := exec.Command("sh", "-c", "drush pm-releases --pipe "+Project+" | grep Recommended | cut -d',' -f2").Output()
			if y == nil {
				ProjectVersion := removeChar(string(x), " ", "7.x-", "\"", "\n", "[", "]")
				ProjectType := "contrib"
				if ProjectVersion == "" {
					ProjectType = "custom"
				}
				headerLines = append(headerLines, fmt.Sprintf("projects[%v][version] = \"%v\"", Project, ProjectVersion))
				headerLines = append(headerLines, fmt.Sprintf("projects[%v][type] = \"module\"", Project))
				headerLines = append(headerLines, fmt.Sprintf("projects[%v][subdir] = \"%v\"", Project, ProjectType))
				headerLines = append(headerLines, fmt.Sprint(""))
			}
		}
	}

	// Print to path File

	newFile, _ := os.Create(File)
	for _, line := range headerLines {
		fmt.Fprintln(newFile, line)
	}
	newFile.Sync()
	defer newFile.Close()

}
<|endoftext|>