<commit_before>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"knative.dev/serving/pkg/reconciler/clusteringress"

	// This defines the shared main for injected controllers.
	"knative.dev/pkg/injection/sharedmain"
)

func main() {
	sharedmain.Main("istiocontroller",
		clusteringress.NewController)
}
<commit_msg>Add namespaced ingress implementation which runs along side with cluster ingress (#4750)<commit_after>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"knative.dev/serving/pkg/reconciler/clusteringress"
	"knative.dev/serving/pkg/reconciler/ingress"

	// This defines the shared main for injected controllers.
	"knative.dev/pkg/injection/sharedmain"
)

func main() {
	sharedmain.Main("istiocontroller",
		clusteringress.NewController, ingress.NewController)
}
<|endoftext|>


<commit_before>// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"path"
	"regexp"
	"sync"
	"time"

	"github.com/golang/glog"
	"golang.org/x/net/context"
	"google.golang.org/grpc"

	pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1alpha"
)

const (
	// All NVIDIA GPUs cards should be mounted with nvidiactl and nvidia-uvm
	// If the driver installed correctly, these two devices will be there.
	nvidiaCtlDevice = "/dev/nvidiactl"
	nvidiaUVMDevice = "/dev/nvidia-uvm"
	// Optional device.
	nvidiaUVMToolsDevice = "/dev/nvidia-uvm-tools"
	devDirectory         = "/dev"
	nvidiaDeviceRE       = `^nvidia[0-9]*$`

	// Device plugin settings.
	pluginMountPath      = "/device-plugin"
	kubeletEndpoint      = "kubelet.sock"
	pluginEndpointPrefix = "nvidiaGPU"
	resourceName         = "nvidia.com/gpu"
)

var (
	hostPathPrefix      = flag.String("host-path", "/home/kubernetes/bin/nvidia", "Path on the host that contains nvidia libraries. This will be mounted inside the container as '-container-path'")
	containerPathPrefix = flag.String("container-path", "/usr/local/nvidia", "Path on the container that mounts '-host-path'")
)

// nvidiaGPUManager manages nvidia gpu devices.
type nvidiaGPUManager struct {
	defaultDevices []string
	devices        map[string]pluginapi.Device
	grpcServer     *grpc.Server
}

func NewNvidiaGPUManager() *nvidiaGPUManager {
	return &nvidiaGPUManager{
		devices: make(map[string]pluginapi.Device),
	}
}

// Discovers all NVIDIA GPU devices available on the local node by walking `/dev` directory.
func (ngm *nvidiaGPUManager) discoverGPUs() error {
	reg := regexp.MustCompile(nvidiaDeviceRE)
	files, err := ioutil.ReadDir(devDirectory)
	if err != nil {
		return err
	}
	for _, f := range files {
		if f.IsDir() {
			continue
		}
		if reg.MatchString(f.Name()) {
			glog.Infof("Found Nvidia GPU %q\n", f.Name())
			ngm.devices[f.Name()] = pluginapi.Device{f.Name(), pluginapi.Healthy}
		}
	}
	return nil
}

func (ngm *nvidiaGPUManager) GetDeviceState(DeviceName string) string {
	// TODO: calling Nvidia tools to figure out actual device state
	return pluginapi.Healthy
}

// Discovers Nvidia GPU devices and sets up device access environment.
func (ngm *nvidiaGPUManager) Start() error {
	if _, err := os.Stat(nvidiaCtlDevice); err != nil {
		return err
	}

	if _, err := os.Stat(nvidiaUVMDevice); err != nil {
		return err
	}

	ngm.defaultDevices = []string{nvidiaCtlDevice, nvidiaUVMDevice}

	if _, err := os.Stat(nvidiaUVMToolsDevice); err != nil {
		ngm.defaultDevices = append(ngm.defaultDevices, nvidiaUVMToolsDevice)
	}

	if err := ngm.discoverGPUs(); err != nil {
		return err
	}

	return nil
}

// Act as a grpc client and register with the kubelet.
func Register(kubeletEndpoint, pluginEndpoint, resourceName string) error {
	conn, err := grpc.Dial(kubeletEndpoint, grpc.WithInsecure(),
		grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) {
			return net.DialTimeout("unix", addr, timeout)
		}))
	if err != nil {
		return fmt.Errorf("device-plugin: cannot connect to kubelet service: %v", err)
	}
	defer conn.Close()
	client := pluginapi.NewRegistrationClient(conn)

	request := &pluginapi.RegisterRequest{
		Version:      pluginapi.Version,
		Endpoint:     pluginEndpoint,
		ResourceName: resourceName,
	}

	if _, err = client.Register(context.Background(), request); err != nil {
		return fmt.Errorf("device-plugin: cannot register to kubelet service: %v", err)
	}
	return nil
}

// Implements DevicePlugin service functions
func (ngm *nvidiaGPUManager) ListAndWatch(emtpy *pluginapi.Empty, stream pluginapi.DevicePlugin_ListAndWatchServer) error {
	glog.Infoln("device-plugin: ListAndWatch start")
	changed := true
	for {
		for id, dev := range ngm.devices {
			state := ngm.GetDeviceState(id)
			if dev.Health != state {
				changed = true
				dev.Health = state
				ngm.devices[id] = dev
			}
		}
		if changed {
			resp := new(pluginapi.ListAndWatchResponse)
			for _, dev := range ngm.devices {
				resp.Devices = append(resp.Devices, &pluginapi.Device{dev.ID, dev.Health})
			}
			glog.Infof("ListAndWatch: send devices %v\n", resp)
			if err := stream.Send(resp); err != nil {
				glog.Errorf("device-plugin: cannot update device states: %v\n", err)
				ngm.grpcServer.Stop()
				return err
			}
		}
		changed = false
		time.Sleep(5 * time.Second)
	}
}

func (ngm *nvidiaGPUManager) Allocate(ctx context.Context, rqt *pluginapi.AllocateRequest) (*pluginapi.AllocateResponse, error) {
	resp := new(pluginapi.AllocateResponse)
	// Add all requested devices to Allocate Response
	for _, id := range rqt.DevicesIDs {
		dev, ok := ngm.devices[id]
		if !ok {
			return nil, fmt.Errorf("invalid allocation request with non-existing device %s", id)
		}
		if dev.Health != pluginapi.Healthy {
			return nil, fmt.Errorf("invalid allocation request with unhealthy device %s", id)
		}
		resp.Devices = append(resp.Devices, &pluginapi.DeviceSpec{
			HostPath:      "/dev/" + id,
			ContainerPath: "/dev/" + id,
			Permissions:   "mrw",
		})
	}
	// Add all default devices to Allocate Response
	for _, d := range ngm.defaultDevices {
		resp.Devices = append(resp.Devices, &pluginapi.DeviceSpec{
			HostPath:      d,
			ContainerPath: d,
			Permissions:   "mrw",
		})
	}

	resp.Mounts = append(resp.Mounts, &pluginapi.Mount{
		ContainerPath: *containerPathPrefix,
		HostPath:      *hostPathPrefix,
		ReadOnly:      true,
	})
	return resp, nil
}

func (ngm *nvidiaGPUManager) Serve(pMountPath, kEndpoint, pEndpointPrefix string) {
	for {
		pluginEndpoint := fmt.Sprintf("%s-%d.sock", pEndpointPrefix, time.Now().Unix())
		pluginEndpointPath := path.Join(pMountPath, pluginEndpoint)
		glog.Infof("starting device-plugin server at: %s\n", pluginEndpointPath)
		lis, err := net.Listen("unix", pluginEndpointPath)
		if err != nil {
			glog.Fatalf("starting device-plugin server failed: %v", err)
		}
		ngm.grpcServer = grpc.NewServer()
		pluginapi.RegisterDevicePluginServer(ngm.grpcServer, ngm)

		var wg sync.WaitGroup
		wg.Add(1)
		// Starts device plugin service.
		go func() {
			defer wg.Done()
			// Blocking call to accept incoming connections.
			err := ngm.grpcServer.Serve(lis)
			glog.Errorf("device-plugin server stopped serving: %v", err)
		}()

		// Wait till the grpcServer is ready to serve services.
		for len(ngm.grpcServer.GetServiceInfo()) <= 0 {
			time.Sleep(1 * time.Second)
		}
		glog.Infoln("device-plugin server started serving")

		// Registers with Kubelet.
		err = Register(path.Join(pMountPath, kEndpoint), pluginEndpoint, resourceName)
		if err != nil {
			ngm.grpcServer.Stop()
			wg.Wait()
			glog.Fatal(err)
		}
		glog.Infoln("device-plugin registered with the kubelet")

		// This is checking if the plugin socket was deleted. If so,
		// stop the grpc server and start the whole thing again.
		for {
			if _, err := os.Lstat(pluginEndpointPath); err != nil {
				glog.Errorln(err)
				ngm.grpcServer.Stop()
				break
			}
			time.Sleep(1 * time.Second)
		}
		wg.Wait()
	}
}

func main() {
	flag.Parse()
	glog.Infoln("device-plugin started")
	ngm := NewNvidiaGPUManager()
	// Keep on trying until success. This is required
	// because Nvidia drivers may not be installed initially.
	for {
		err := ngm.Start()
		if err == nil {
			break
		}
		// Use non-default level to avoid log spam.
		glog.V(3).Infof("nvidiaGPUManager.Start() failed: %v", err)
		time.Sleep(5 * time.Second)
	}
	ngm.Serve(pluginMountPath, kubeletEndpoint, pluginEndpointPrefix)
}
<commit_msg>Fix logic inversion error.<commit_after>// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"path"
	"regexp"
	"sync"
	"time"

	"github.com/golang/glog"
	"golang.org/x/net/context"
	"google.golang.org/grpc"

	pluginapi "k8s.io/kubernetes/pkg/kubelet/apis/deviceplugin/v1alpha"
)

const (
	// All NVIDIA GPUs cards should be mounted with nvidiactl and nvidia-uvm
	// If the driver installed correctly, these two devices will be there.
	nvidiaCtlDevice = "/dev/nvidiactl"
	nvidiaUVMDevice = "/dev/nvidia-uvm"
	// Optional device.
	nvidiaUVMToolsDevice = "/dev/nvidia-uvm-tools"
	devDirectory         = "/dev"
	nvidiaDeviceRE       = `^nvidia[0-9]*$`

	// Device plugin settings.
	pluginMountPath      = "/device-plugin"
	kubeletEndpoint      = "kubelet.sock"
	pluginEndpointPrefix = "nvidiaGPU"
	resourceName         = "nvidia.com/gpu"
)

var (
	hostPathPrefix      = flag.String("host-path", "/home/kubernetes/bin/nvidia", "Path on the host that contains nvidia libraries. This will be mounted inside the container as '-container-path'")
	containerPathPrefix = flag.String("container-path", "/usr/local/nvidia", "Path on the container that mounts '-host-path'")
)

// nvidiaGPUManager manages nvidia gpu devices.
type nvidiaGPUManager struct {
	defaultDevices []string
	devices        map[string]pluginapi.Device
	grpcServer     *grpc.Server
}

func NewNvidiaGPUManager() *nvidiaGPUManager {
	return &nvidiaGPUManager{
		devices: make(map[string]pluginapi.Device),
	}
}

// Discovers all NVIDIA GPU devices available on the local node by walking `/dev` directory.
func (ngm *nvidiaGPUManager) discoverGPUs() error {
	reg := regexp.MustCompile(nvidiaDeviceRE)
	files, err := ioutil.ReadDir(devDirectory)
	if err != nil {
		return err
	}
	for _, f := range files {
		if f.IsDir() {
			continue
		}
		if reg.MatchString(f.Name()) {
			glog.Infof("Found Nvidia GPU %q\n", f.Name())
			ngm.devices[f.Name()] = pluginapi.Device{f.Name(), pluginapi.Healthy}
		}
	}
	return nil
}

func (ngm *nvidiaGPUManager) GetDeviceState(DeviceName string) string {
	// TODO: calling Nvidia tools to figure out actual device state
	return pluginapi.Healthy
}

// Discovers Nvidia GPU devices and sets up device access environment.
func (ngm *nvidiaGPUManager) Start() error {
	if _, err := os.Stat(nvidiaCtlDevice); err != nil {
		return err
	}

	if _, err := os.Stat(nvidiaUVMDevice); err != nil {
		return err
	}

	ngm.defaultDevices = []string{nvidiaCtlDevice, nvidiaUVMDevice}

	if _, err := os.Stat(nvidiaUVMToolsDevice); err == nil {
		ngm.defaultDevices = append(ngm.defaultDevices, nvidiaUVMToolsDevice)
	}

	if err := ngm.discoverGPUs(); err != nil {
		return err
	}

	return nil
}

// Act as a grpc client and register with the kubelet.
func Register(kubeletEndpoint, pluginEndpoint, resourceName string) error {
	conn, err := grpc.Dial(kubeletEndpoint, grpc.WithInsecure(),
		grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) {
			return net.DialTimeout("unix", addr, timeout)
		}))
	if err != nil {
		return fmt.Errorf("device-plugin: cannot connect to kubelet service: %v", err)
	}
	defer conn.Close()
	client := pluginapi.NewRegistrationClient(conn)

	request := &pluginapi.RegisterRequest{
		Version:      pluginapi.Version,
		Endpoint:     pluginEndpoint,
		ResourceName: resourceName,
	}

	if _, err = client.Register(context.Background(), request); err != nil {
		return fmt.Errorf("device-plugin: cannot register to kubelet service: %v", err)
	}
	return nil
}

// Implements DevicePlugin service functions
func (ngm *nvidiaGPUManager) ListAndWatch(emtpy *pluginapi.Empty, stream pluginapi.DevicePlugin_ListAndWatchServer) error {
	glog.Infoln("device-plugin: ListAndWatch start")
	changed := true
	for {
		for id, dev := range ngm.devices {
			state := ngm.GetDeviceState(id)
			if dev.Health != state {
				changed = true
				dev.Health = state
				ngm.devices[id] = dev
			}
		}
		if changed {
			resp := new(pluginapi.ListAndWatchResponse)
			for _, dev := range ngm.devices {
				resp.Devices = append(resp.Devices, &pluginapi.Device{dev.ID, dev.Health})
			}
			glog.Infof("ListAndWatch: send devices %v\n", resp)
			if err := stream.Send(resp); err != nil {
				glog.Errorf("device-plugin: cannot update device states: %v\n", err)
				ngm.grpcServer.Stop()
				return err
			}
		}
		changed = false
		time.Sleep(5 * time.Second)
	}
}

func (ngm *nvidiaGPUManager) Allocate(ctx context.Context, rqt *pluginapi.AllocateRequest) (*pluginapi.AllocateResponse, error) {
	resp := new(pluginapi.AllocateResponse)
	// Add all requested devices to Allocate Response
	for _, id := range rqt.DevicesIDs {
		dev, ok := ngm.devices[id]
		if !ok {
			return nil, fmt.Errorf("invalid allocation request with non-existing device %s", id)
		}
		if dev.Health != pluginapi.Healthy {
			return nil, fmt.Errorf("invalid allocation request with unhealthy device %s", id)
		}
		resp.Devices = append(resp.Devices, &pluginapi.DeviceSpec{
			HostPath:      "/dev/" + id,
			ContainerPath: "/dev/" + id,
			Permissions:   "mrw",
		})
	}
	// Add all default devices to Allocate Response
	for _, d := range ngm.defaultDevices {
		resp.Devices = append(resp.Devices, &pluginapi.DeviceSpec{
			HostPath:      d,
			ContainerPath: d,
			Permissions:   "mrw",
		})
	}

	resp.Mounts = append(resp.Mounts, &pluginapi.Mount{
		ContainerPath: *containerPathPrefix,
		HostPath:      *hostPathPrefix,
		ReadOnly:      true,
	})
	return resp, nil
}

func (ngm *nvidiaGPUManager) Serve(pMountPath, kEndpoint, pEndpointPrefix string) {
	for {
		pluginEndpoint := fmt.Sprintf("%s-%d.sock", pEndpointPrefix, time.Now().Unix())
		pluginEndpointPath := path.Join(pMountPath, pluginEndpoint)
		glog.Infof("starting device-plugin server at: %s\n", pluginEndpointPath)
		lis, err := net.Listen("unix", pluginEndpointPath)
		if err != nil {
			glog.Fatalf("starting device-plugin server failed: %v", err)
		}
		ngm.grpcServer = grpc.NewServer()
		pluginapi.RegisterDevicePluginServer(ngm.grpcServer, ngm)

		var wg sync.WaitGroup
		wg.Add(1)
		// Starts device plugin service.
		go func() {
			defer wg.Done()
			// Blocking call to accept incoming connections.
			err := ngm.grpcServer.Serve(lis)
			glog.Errorf("device-plugin server stopped serving: %v", err)
		}()

		// Wait till the grpcServer is ready to serve services.
		for len(ngm.grpcServer.GetServiceInfo()) <= 0 {
			time.Sleep(1 * time.Second)
		}
		glog.Infoln("device-plugin server started serving")

		// Registers with Kubelet.
		err = Register(path.Join(pMountPath, kEndpoint), pluginEndpoint, resourceName)
		if err != nil {
			ngm.grpcServer.Stop()
			wg.Wait()
			glog.Fatal(err)
		}
		glog.Infoln("device-plugin registered with the kubelet")

		// This is checking if the plugin socket was deleted. If so,
		// stop the grpc server and start the whole thing again.
		for {
			if _, err := os.Lstat(pluginEndpointPath); err != nil {
				glog.Errorln(err)
				ngm.grpcServer.Stop()
				break
			}
			time.Sleep(1 * time.Second)
		}
		wg.Wait()
	}
}

func main() {
	flag.Parse()
	glog.Infoln("device-plugin started")
	ngm := NewNvidiaGPUManager()
	// Keep on trying until success. This is required
	// because Nvidia drivers may not be installed initially.
	for {
		err := ngm.Start()
		if err == nil {
			break
		}
		// Use non-default level to avoid log spam.
		glog.V(3).Infof("nvidiaGPUManager.Start() failed: %v", err)
		time.Sleep(5 * time.Second)
	}
	ngm.Serve(pluginMountPath, kubeletEndpoint, pluginEndpointPrefix)
}
<|endoftext|>


<commit_before>// web100_cli provides a simple CLI interface to web100 functions.
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"os"

	"cloud.google.com/go/bigquery"
	"github.com/m-lab/etl/web100"
)

var (
	filename = flag.String("filename", "", "Trace filename.")
	tcpKis   = flag.String("tcp-kis", "tcp-kis.txt", "tcp-kis.txt filename.")
)

func PrettyPrint(results map[string]bigquery.Value) {
	b, err := json.MarshalIndent(results, "", "  ")
	if err != nil {
		fmt.Println("error:", err)
	}
	fmt.Print(string(b))
}

func main() {
	flag.Parse()

	// Parse tcp-kis.txt variable definitions.
	k, err := os.Open(*tcpKis)
	if err != nil {
		panic(err)
	}
	legacyNames, err := web100.ParseWeb100Definitions(k)
	if err != nil {
		panic(err)
	}

	// Open web100 snapshot log.
	w, err := web100.Open(*filename, legacyNames)
	if err != nil {
		panic(err)
	}
	defer w.Close()

	// Find all last web100 snapshot.
	for {
		err = w.Next()
		if err != nil {
			break
		}
	}
	if err != io.EOF {
		panic(err)
	}
	results, err := w.Values()
	if err != nil {
		panic(err)
	}
	PrettyPrint(results)

	// Get results.
	// results, err = w.SnapValues(legacyNames)
	// if err != nil {
	// 	panic(err)
	// }
	// PrettyPrint(results)
	// fmt.Printf("%#v\n", w)
}
<commit_msg>Use the new web100 interfaces.<commit_after>// web100_cli provides a simple CLI interface to web100 functions.
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"os"

	"cloud.google.com/go/bigquery"
	"github.com/m-lab/etl/schema"
	"github.com/m-lab/etl/web100"
)

var (
	filename = flag.String("filename", "", "Trace filename.")
	tcpKis   = flag.String("tcp-kis", "tcp-kis.txt", "tcp-kis.txt filename.")
)

func prettyPrint(results map[string]bigquery.Value) {
	b, err := json.MarshalIndent(results, "", "  ")
	if err != nil {
		fmt.Println("error:", err)
	}
	fmt.Print(string(b))
}

func main() {
	flag.Parse()

	// Parse tcp-kis.txt variable definitions.
	k, err := os.Open(*tcpKis)
	if err != nil {
		panic(err)
	}
	legacyNames, err := web100.ParseWeb100Definitions(k)
	if err != nil {
		panic(err)
	}

	// Open web100 snapshot log.
	w, err := web100.Open(*filename, legacyNames)
	if err != nil {
		panic(err)
	}
	defer w.Close()

	// Find all last web100 snapshot.
	for {
		err = w.Next()
		if err != nil {
			break
		}
	}
	if err != io.EOF {
		panic(err)
	}

	snapValues := schema.Web100ValueMap{}
	err = w.SnapValues(snapValues)
	if err != nil {
		panic(err)
	}
	connSpec := schema.Web100ValueMap{}
	w.ConnectionSpec(connSpec)
	results := schema.NewWeb100FullRecord(
		w.LogVersion(), w.LogTime(),
		(map[string]bigquery.Value)(connSpec),
		(map[string]bigquery.Value)(snapValues))

	prettyPrint(results)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/fubarhouse/golang-drush/command"
	"github.com/fubarhouse/golang-drush/make"
	"os"
	"strings"
)

func main() {

	var Path = flag.String("path", "", "Path to site")
	var Site = flag.String("site", "", "Shortname of site")
	var Domain = flag.String("domain", "", "Domain of site")
	var Alias = flag.String("alias", "", "Alias of site")
	var Makes = flag.String("makes", "", "Comma-separated list of make files to use")
	var BuildID = flag.String("build", "", "optional timestamp of site")
	var VHostDir = flag.String("vhost-dir", "/etc/nginx/sites-enabled", "Directory containing virtual host file(s)")
	var WebserverName = flag.String("webserver-name", "nginx", "The name of the web service on the server.")

	// Usage:
	// -path="/path/to/site" \
	// -site="mysite" \
	// -domain="mysite.dev" \
	// -alias="mysite.dev" \
	// -makes="/path/to/make1.make, /path/to/make2.make" \

	flag.Parse()

	if *Site == "" || *Alias == "" || *Makes == "" || *Path == "" || *Domain == "" {
		flag.Usage()
		os.Exit(1)
	}

	x := make.NewSite(*Makes, *Site, *Path, *Alias, *WebserverName, *Domain, *VHostDir)
	y := make.NewmakeDB("127.0.0.1", "root", "root", 3306)
	x.DatabaseSet(y)
	if *BuildID == "" {
		x.TimeStampReset()
	} else {
		x.TimeStampSet(*BuildID)
	}

	MakefilesFormatted := strings.Replace(x.Make, " ", "", -1)
	MakeFiles := strings.Split(MakefilesFormatted, ",")

	x.ActionRebuildCodebase(MakeFiles)
	x.InstallSiteRef()
	x.ActionInstall()
	x.SymReinstall(x.TimeStampGet())
	x.InstallPrivateFileSystem()
	x.VhostInstall()
	x.AliasInstall()
	command.DrushUpdateDatabase(x.Alias)
	command.DrushRebuildRegistry(x.Alias)
	x.RestartWebServer()
}
<commit_msg>remove unused imports<commit_after>package main

import (
	"flag"
	"github.com/fubarhouse/golang-drush/command"
	"github.com/fubarhouse/golang-drush/make"
	"os"
	"strings"
)

func main() {

	var Path = flag.String("path", "", "Path to site")
	var Site = flag.String("site", "", "Shortname of site")
	var Domain = flag.String("domain", "", "Domain of site")
	var Alias = flag.String("alias", "", "Alias of site")
	var Makes = flag.String("makes", "", "Comma-separated list of make files to use")
	var BuildID = flag.String("build", "", "optional timestamp of site")
	var VHostDir = flag.String("vhost-dir", "/etc/nginx/sites-enabled", "Directory containing virtual host file(s)")
	var WebserverName = flag.String("webserver-name", "nginx", "The name of the web service on the server.")

	// Usage:
	// -path="/path/to/site" \
	// -site="mysite" \
	// -domain="mysite.dev" \
	// -alias="mysite.dev" \
	// -makes="/path/to/make1.make, /path/to/make2.make" \

	flag.Parse()

	if *Site == "" || *Alias == "" || *Makes == "" || *Path == "" || *Domain == "" {
		flag.Usage()
		os.Exit(1)
	}

	x := make.NewSite(*Makes, *Site, *Path, *Alias, *WebserverName, *Domain, *VHostDir)
	y := make.NewmakeDB("127.0.0.1", "root", "root", 3306)
	x.DatabaseSet(y)
	if *BuildID == "" {
		x.TimeStampReset()
	} else {
		x.TimeStampSet(*BuildID)
	}

	MakefilesFormatted := strings.Replace(x.Make, " ", "", -1)
	MakeFiles := strings.Split(MakefilesFormatted, ",")

	x.ActionRebuildCodebase(MakeFiles)
	x.InstallSiteRef()
	x.ActionInstall()
	x.SymReinstall(x.TimeStampGet())
	x.InstallPrivateFileSystem()
	x.VhostInstall()
	x.AliasInstall()
	command.DrushUpdateDatabase(x.Alias)
	command.DrushRebuildRegistry(x.Alias)
	x.RestartWebServer()
}
<|endoftext|>


<commit_before>package v3

import (
	"net/http"

	"code.cloudfoundry.org/cli/actor/sharedaction"
	"code.cloudfoundry.org/cli/actor/v3action"
	"code.cloudfoundry.org/cli/api/cloudcontroller/ccerror"
	"code.cloudfoundry.org/cli/api/cloudcontroller/ccversion"
	"code.cloudfoundry.org/cli/command"
	"code.cloudfoundry.org/cli/command/flag"
	"code.cloudfoundry.org/cli/command/translatableerror"
	"code.cloudfoundry.org/cli/command/v3/shared"
	"code.cloudfoundry.org/cli/util/clissh"
)

//go:generate counterfeiter . SSHActor

type SSHActor interface {
	ExecuteSecureShell(sshClient sharedaction.SecureShellClient, sshOptions sharedaction.SSHOptions) error
}

//go:generate counterfeiter . V3SSHActor

type V3SSHActor interface {
	CloudControllerAPIVersion() string
	GetSecureShellConfigurationByApplicationNameSpaceProcessTypeAndIndex(appName string, spaceGUID string, processType string, processIndex uint) (v3action.SSHAuthentication, v3action.Warnings, error)
}

type V3SSHCommand struct {
	RequiredArgs          flag.AppName             `positional-args:"yes"`
	ProcessIndex          uint                     `long:"app-instance-index" short:"i" description:"App process instance index (Default: 0)"`
	Commands              []string                 `long:"command" short:"c" description:"Command to run"`
	DisablePseudoTTY      bool                     `long:"disable-pseudo-tty" short:"T" description:"Disable pseudo-tty allocation"`
	ForcePseudoTTY        bool                     `long:"force-pseudo-tty" description:"Force pseudo-tty allocation"`
	LocalPortForwardSpecs []flag.SSHPortForwarding `short:"L" description:"Local port forward specification"`
	ProcessType           string                   `long:"process" description:"App process name (Default: web)"`
	RequestPseudoTTY      bool                     `long:"request-pseudo-tty" short:"t" description:"Request pseudo-tty allocation"`
	SkipHostValidation    bool                     `long:"skip-host-validation" short:"k" description:"Skip host key validation. Not recommended!"`
	SkipRemoteExecution   bool                     `long:"skip-remote-execution" short:"N" description:"Do not execute a remote command"`

	usage           interface{} `usage:"cf v3-ssh APP_NAME [--process PROCESS] [-i INDEX] [-c COMMAND]\n   [-L [BIND_ADDRESS:]LOCAL_PORT:REMOTE_HOST:REMOTE_PORT]... [--skip-remote-execution]\n   [--disable-pseudo-tty | --force-pseudo-tty | --request-pseudo-tty] [--skip-host-validation]\n"`
	relatedCommands interface{} `related_commands:"allow-space-ssh, enable-ssh, space-ssh-allowed, ssh-code, ssh-enabled"`

	UI          command.UI
	Config      command.Config
	SharedActor command.SharedActor
	Actor       V3SSHActor
	SSHActor    SSHActor
	SSHClient   *clissh.SecureShell
}

func (cmd *V3SSHCommand) Setup(config command.Config, ui command.UI) error {
	cmd.UI = ui
	cmd.Config = config
	sharedActor := sharedaction.NewActor(config)
	cmd.SharedActor = sharedActor
	cmd.SSHActor = sharedActor

	ccClient, uaaClient, err := shared.NewClients(config, ui, true)
	if err != nil {
		if v3Err, ok := err.(ccerror.V3UnexpectedResponseError); ok && v3Err.ResponseCode == http.StatusNotFound {
			return translatableerror.MinimumAPIVersionNotMetError{MinimumVersion: ccversion.MinVersionV3}
		}

		return err
	}

	cmd.Actor = v3action.NewActor(ccClient, config, sharedActor, uaaClient)

	cmd.SSHClient = clissh.NewDefaultSecureShell(ui.GetIn(), ui.GetOut(), ui.GetErr())

	return nil
}

func (cmd V3SSHCommand) Execute(args []string) error {
	cmd.UI.DisplayText(command.ExperimentalWarning)
	cmd.UI.DisplayNewline()

	err := command.MinimumAPIVersionCheck(cmd.Actor.CloudControllerAPIVersion(), ccversion.MinVersionV3)
	if err != nil {
		return err
	}

	err = cmd.SharedActor.CheckTarget(true, true)
	if err != nil {
		return err
	}

	ttyOption, err := cmd.EvaluateTTYOption()
	if err != nil {
		return err
	}

	if cmd.ProcessType == "" {
		cmd.ProcessType = "web"
	}

	var forwardSpecs []sharedaction.LocalPortForward
	for _, spec := range cmd.LocalPortForwardSpecs {
		forwardSpecs = append(forwardSpecs, sharedaction.LocalPortForward(spec))
	}

	sshAuth, warnings, err := cmd.Actor.GetSecureShellConfigurationByApplicationNameSpaceProcessTypeAndIndex(
		cmd.RequiredArgs.AppName,
		cmd.Config.TargetedSpace().GUID,
		cmd.ProcessType,
		cmd.ProcessIndex,
	)
	cmd.UI.DisplayWarnings(warnings)
	if err != nil {
		return err
	}

	err = cmd.SSHActor.ExecuteSecureShell(
		cmd.SSHClient,
		sharedaction.SSHOptions{
			Commands:              cmd.Commands,
			Endpoint:              sshAuth.Endpoint,
			HostKeyFingerprint:    sshAuth.HostKeyFingerprint,
			LocalPortForwardSpecs: forwardSpecs,
			Passcode:              sshAuth.Passcode,
			SkipHostValidation:    cmd.SkipHostValidation,
			SkipRemoteExecution:   cmd.SkipRemoteExecution,
			TTYOption:             ttyOption,
			Username:              sshAuth.Username,
		})
	if err != nil {
		return err
	}

	return nil
}

func (cmd V3SSHCommand) parseForwardSpecs() ([]sharedaction.LocalPortForward, error) {
	return nil, nil
}

// EvaluateTTYOption determines which TTY options are mutually exclusive and
// returns an error accordingly.
func (cmd V3SSHCommand) EvaluateTTYOption() (sharedaction.TTYOption, error) {
	var count int

	option := sharedaction.RequestTTYAuto
	if cmd.DisablePseudoTTY {
		option = sharedaction.RequestTTYNo
		count++
	}
	if cmd.ForcePseudoTTY {
		option = sharedaction.RequestTTYForce
		count++
	}
	if cmd.RequestPseudoTTY {
		option = sharedaction.RequestTTYYes
		count++
	}

	if count > 1 {
		return option, translatableerror.ArgumentCombinationError{Args: []string{
			"--disable-pseudo-tty", "-T", "--force-pseudo-tty", "--request-pseudo-tty", "-t",
		}}
	}

	return option, nil
}
<commit_msg>remove newline<commit_after>package v3

import (
	"net/http"

	"code.cloudfoundry.org/cli/actor/sharedaction"
	"code.cloudfoundry.org/cli/actor/v3action"
	"code.cloudfoundry.org/cli/api/cloudcontroller/ccerror"
	"code.cloudfoundry.org/cli/api/cloudcontroller/ccversion"
	"code.cloudfoundry.org/cli/command"
	"code.cloudfoundry.org/cli/command/flag"
	"code.cloudfoundry.org/cli/command/translatableerror"
	"code.cloudfoundry.org/cli/command/v3/shared"
	"code.cloudfoundry.org/cli/util/clissh"
)

//go:generate counterfeiter . SSHActor

type SSHActor interface {
	ExecuteSecureShell(sshClient sharedaction.SecureShellClient, sshOptions sharedaction.SSHOptions) error
}

//go:generate counterfeiter . V3SSHActor

type V3SSHActor interface {
	CloudControllerAPIVersion() string
	GetSecureShellConfigurationByApplicationNameSpaceProcessTypeAndIndex(appName string, spaceGUID string, processType string, processIndex uint) (v3action.SSHAuthentication, v3action.Warnings, error)
}

type V3SSHCommand struct {
	RequiredArgs          flag.AppName             `positional-args:"yes"`
	ProcessIndex          uint                     `long:"app-instance-index" short:"i" description:"App process instance index (Default: 0)"`
	Commands              []string                 `long:"command" short:"c" description:"Command to run"`
	DisablePseudoTTY      bool                     `long:"disable-pseudo-tty" short:"T" description:"Disable pseudo-tty allocation"`
	ForcePseudoTTY        bool                     `long:"force-pseudo-tty" description:"Force pseudo-tty allocation"`
	LocalPortForwardSpecs []flag.SSHPortForwarding `short:"L" description:"Local port forward specification"`
	ProcessType           string                   `long:"process" description:"App process name (Default: web)"`
	RequestPseudoTTY      bool                     `long:"request-pseudo-tty" short:"t" description:"Request pseudo-tty allocation"`
	SkipHostValidation    bool                     `long:"skip-host-validation" short:"k" description:"Skip host key validation. Not recommended!"`
	SkipRemoteExecution   bool                     `long:"skip-remote-execution" short:"N" description:"Do not execute a remote command"`

	usage           interface{} `usage:"cf v3-ssh APP_NAME [--process PROCESS] [-i INDEX] [-c COMMAND]\n   [-L [BIND_ADDRESS:]LOCAL_PORT:REMOTE_HOST:REMOTE_PORT]... [--skip-remote-execution]\n   [--disable-pseudo-tty | --force-pseudo-tty | --request-pseudo-tty] [--skip-host-validation]"`
	relatedCommands interface{} `related_commands:"allow-space-ssh, enable-ssh, space-ssh-allowed, ssh-code, ssh-enabled"`

	UI          command.UI
	Config      command.Config
	SharedActor command.SharedActor
	Actor       V3SSHActor
	SSHActor    SSHActor
	SSHClient   *clissh.SecureShell
}

func (cmd *V3SSHCommand) Setup(config command.Config, ui command.UI) error {
	cmd.UI = ui
	cmd.Config = config
	sharedActor := sharedaction.NewActor(config)
	cmd.SharedActor = sharedActor
	cmd.SSHActor = sharedActor

	ccClient, uaaClient, err := shared.NewClients(config, ui, true)
	if err != nil {
		if v3Err, ok := err.(ccerror.V3UnexpectedResponseError); ok && v3Err.ResponseCode == http.StatusNotFound {
			return translatableerror.MinimumAPIVersionNotMetError{MinimumVersion: ccversion.MinVersionV3}
		}

		return err
	}

	cmd.Actor = v3action.NewActor(ccClient, config, sharedActor, uaaClient)

	cmd.SSHClient = clissh.NewDefaultSecureShell(ui.GetIn(), ui.GetOut(), ui.GetErr())

	return nil
}

func (cmd V3SSHCommand) Execute(args []string) error {
	cmd.UI.DisplayText(command.ExperimentalWarning)
	cmd.UI.DisplayNewline()

	err := command.MinimumAPIVersionCheck(cmd.Actor.CloudControllerAPIVersion(), ccversion.MinVersionV3)
	if err != nil {
		return err
	}

	err = cmd.SharedActor.CheckTarget(true, true)
	if err != nil {
		return err
	}

	ttyOption, err := cmd.EvaluateTTYOption()
	if err != nil {
		return err
	}

	if cmd.ProcessType == "" {
		cmd.ProcessType = "web"
	}

	var forwardSpecs []sharedaction.LocalPortForward
	for _, spec := range cmd.LocalPortForwardSpecs {
		forwardSpecs = append(forwardSpecs, sharedaction.LocalPortForward(spec))
	}

	sshAuth, warnings, err := cmd.Actor.GetSecureShellConfigurationByApplicationNameSpaceProcessTypeAndIndex(
		cmd.RequiredArgs.AppName,
		cmd.Config.TargetedSpace().GUID,
		cmd.ProcessType,
		cmd.ProcessIndex,
	)
	cmd.UI.DisplayWarnings(warnings)
	if err != nil {
		return err
	}

	err = cmd.SSHActor.ExecuteSecureShell(
		cmd.SSHClient,
		sharedaction.SSHOptions{
			Commands:              cmd.Commands,
			Endpoint:              sshAuth.Endpoint,
			HostKeyFingerprint:    sshAuth.HostKeyFingerprint,
			LocalPortForwardSpecs: forwardSpecs,
			Passcode:              sshAuth.Passcode,
			SkipHostValidation:    cmd.SkipHostValidation,
			SkipRemoteExecution:   cmd.SkipRemoteExecution,
			TTYOption:             ttyOption,
			Username:              sshAuth.Username,
		})
	if err != nil {
		return err
	}

	return nil
}

func (cmd V3SSHCommand) parseForwardSpecs() ([]sharedaction.LocalPortForward, error) {
	return nil, nil
}

// EvaluateTTYOption determines which TTY options are mutually exclusive and
// returns an error accordingly.
func (cmd V3SSHCommand) EvaluateTTYOption() (sharedaction.TTYOption, error) {
	var count int

	option := sharedaction.RequestTTYAuto
	if cmd.DisablePseudoTTY {
		option = sharedaction.RequestTTYNo
		count++
	}
	if cmd.ForcePseudoTTY {
		option = sharedaction.RequestTTYForce
		count++
	}
	if cmd.RequestPseudoTTY {
		option = sharedaction.RequestTTYYes
		count++
	}

	if count > 1 {
		return option, translatableerror.ArgumentCombinationError{Args: []string{
			"--disable-pseudo-tty", "-T", "--force-pseudo-tty", "--request-pseudo-tty", "-t",
		}}
	}

	return option, nil
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"

	"github.com/git-lfs/git-lfs/filepathfilter"
	"github.com/git-lfs/git-lfs/git"
	"github.com/git-lfs/git-lfs/lfs"
	"github.com/git-lfs/git-lfs/progress"
	"github.com/spf13/cobra"
)

func checkoutCommand(cmd *cobra.Command, args []string) {
	requireInRepo()
	ref, err := git.CurrentRef()
	if err != nil {
		Panic(err, "Could not checkout")
	}

	singleCheckout := newSingleCheckout(cfg.Git, "")
	if singleCheckout.Skip() {
		fmt.Println("Cannot checkout LFS objects, Git LFS is not installed.")
		return
	}

	var totalBytes int64
	var pointers []*lfs.WrappedPointer
	meter := progress.NewMeter(progress.WithOSEnv(cfg.Os))
	chgitscanner := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) {
		if err != nil {
			LoggedError(err, "Scanner error: %s", err)
			return
		}

		totalBytes += p.Size
		meter.Add(p.Size)
		meter.StartTransfer(p.Name)
		pointers = append(pointers, p)
	})

	chgitscanner.Filter = filepathfilter.New(rootedPaths(args), nil)

	if err := chgitscanner.ScanTree(ref.Sha); err != nil {
		ExitWithError(err)
	}
	chgitscanner.Close()

	meter.Start()
	for _, p := range pointers {
		singleCheckout.Run(p)

		// not strictly correct (parallel) but we don't have a callback & it's just local
		// plus only 1 slot in channel so it'll block & be close
		meter.TransferBytes("checkout", p.Name, p.Size, totalBytes, int(p.Size))
		meter.FinishTransfer(p.Name)
	}

	meter.Finish()
	singleCheckout.Close()
}

// Parameters are filters
// firstly convert any pathspecs to the root of the repo, in case this is being
// executed in a sub-folder
func rootedPaths(args []string) []string {
	pathConverter, err := lfs.NewCurrentToRepoPathConverter(cfg)
	if err != nil {
		Panic(err, "Could not checkout")
	}

	rootedpaths := make([]string, 0, len(args))
	for _, arg := range args {
		rootedpaths = append(rootedpaths, pathConverter.Convert(arg))
	}
	return rootedpaths
}

func init() {
	RegisterCommand("checkout", checkoutCommand, nil)
}
<commit_msg>commands/checkout: log progress to os.Stdout<commit_after>package commands

import (
	"fmt"
	"os"

	"github.com/git-lfs/git-lfs/filepathfilter"
	"github.com/git-lfs/git-lfs/git"
	"github.com/git-lfs/git-lfs/git/githistory/log"
	"github.com/git-lfs/git-lfs/lfs"
	"github.com/git-lfs/git-lfs/progress"
	"github.com/spf13/cobra"
)

func checkoutCommand(cmd *cobra.Command, args []string) {
	requireInRepo()
	ref, err := git.CurrentRef()
	if err != nil {
		Panic(err, "Could not checkout")
	}

	singleCheckout := newSingleCheckout(cfg.Git, "")
	if singleCheckout.Skip() {
		fmt.Println("Cannot checkout LFS objects, Git LFS is not installed.")
		return
	}

	var totalBytes int64
	var pointers []*lfs.WrappedPointer
	logger := log.NewLogger(os.Stdout)
	meter := progress.NewMeter(progress.WithOSEnv(cfg.Os))
	logger.Enqueue(meter)
	chgitscanner := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) {
		if err != nil {
			LoggedError(err, "Scanner error: %s", err)
			return
		}

		totalBytes += p.Size
		meter.Add(p.Size)
		meter.StartTransfer(p.Name)
		pointers = append(pointers, p)
	})

	chgitscanner.Filter = filepathfilter.New(rootedPaths(args), nil)

	if err := chgitscanner.ScanTree(ref.Sha); err != nil {
		ExitWithError(err)
	}
	chgitscanner.Close()

	meter.Start()
	for _, p := range pointers {
		singleCheckout.Run(p)

		// not strictly correct (parallel) but we don't have a callback & it's just local
		// plus only 1 slot in channel so it'll block & be close
		meter.TransferBytes("checkout", p.Name, p.Size, totalBytes, int(p.Size))
		meter.FinishTransfer(p.Name)
	}

	meter.Finish()
	singleCheckout.Close()
}

// Parameters are filters
// firstly convert any pathspecs to the root of the repo, in case this is being
// executed in a sub-folder
func rootedPaths(args []string) []string {
	pathConverter, err := lfs.NewCurrentToRepoPathConverter(cfg)
	if err != nil {
		Panic(err, "Could not checkout")
	}

	rootedpaths := make([]string, 0, len(args))
	for _, arg := range args {
		rootedpaths = append(rootedpaths, pathConverter.Convert(arg))
	}
	return rootedpaths
}

func init() {
	RegisterCommand("checkout", checkoutCommand, nil)
}
<|endoftext|>


<commit_before>package commands

import (
	"github.com/github/git-lfs/git"
	"github.com/github/git-lfs/lfs"
	"github.com/github/git-lfs/vendor/_nuts/github.com/spf13/cobra"
	"os"
	"os/exec"
	"sync"
)

var (
	checkoutCmd = &cobra.Command{
		Use:   "checkout",
		Short: "Checks out LFS files into the working copy",
		Run:   checkoutCommand,
	}
)

func checkoutCommand(cmd *cobra.Command, args []string) {

	// Parameters are filters
	checkoutWithIncludeExclude(args, nil)
}

func init() {
	RootCmd.AddCommand(checkoutCmd)
}

func checkoutWithIncludeExclude(include []string, exclude []string) {
	ref, err := git.CurrentRef()
	if err != nil {
		Panic(err, "Could not checkout")
	}

	pointers, err := lfs.ScanTree(ref)
	if err != nil {
		Panic(err, "Could not scan for Git LFS files")
	}

	var wait sync.WaitGroup
	wait.Add(1)

	c := make(chan *lfs.WrappedPointer)

	checkoutWithChan(c, &wait)
	for _, pointer := range pointers {
		if lfs.FilenamePassesIncludeExcludeFilter(pointer.Name, include, exclude) {
			c <- pointer
		}

	}
	close(c)
	wait.Wait()

}

func checkoutAll() {
	checkoutWithIncludeExclude(nil, nil)
}

// Populate the working copy with the real content of objects where the file is
// either missing, or contains a matching pointer placeholder, from a list of pointers.
// If the file exists but has other content it is left alone
// returns immediately but a goroutine listens on the in channel for objects
// calls wait.Done() when the final item after the channel is closed is done
func checkoutWithChan(in <-chan *lfs.WrappedPointer, wait *sync.WaitGroup) {
	go func() {
		defer wait.Done()
		// Fire up the update-index command
		cmd := exec.Command("git", "update-index", "-q", "--refresh", "--stdin")
		updateIdxStdin, err := cmd.StdinPipe()
		if err != nil {
			Panic(err, "Could not update the index")
		}

		if err := cmd.Start(); err != nil {
			Panic(err, "Could not update the index")
		}

		// As files come in, write them to the wd and update the index
		for pointer := range in {

			// Check the content - either missing or still this pointer (not exist is ok)
			filepointer, err := lfs.DecodePointerFromFile(pointer.Name)
			if err != nil && !os.IsNotExist(err) {
				if err == lfs.NotAPointerError {
					// File has non-pointer content, leave it alone
					continue
				}
				Panic(err, "Problem accessing %v", pointer.Name)
			}
			if filepointer != nil && filepointer.Oid != pointer.Oid {
				// User has probably manually reset a file to another commit
				// while leaving it a pointer; don't mess with this
				continue
			}
			// OK now we can (over)write the file content
			file, err := os.Create(pointer.Name)
			if err != nil {
				Panic(err, "Could not create working directory file")
			}

			if err := lfs.PointerSmudge(file, pointer.Pointer, pointer.Name, nil); err != nil {
				Panic(err, "Could not write working directory file")
			}
			file.Close()

			updateIdxStdin.Write([]byte(pointer.Name + "\n"))
		}

		updateIdxStdin.Close()
		if err := cmd.Wait(); err != nil {
			Panic(err, "Error updating the git index")
		}
	}()

}
<commit_msg>ンンンンン ンンンン<commit_after>package commands

import (
	"github.com/github/git-lfs/git"
	"github.com/github/git-lfs/lfs"
	"github.com/github/git-lfs/vendor/_nuts/github.com/spf13/cobra"
	"os"
	"os/exec"
	"sync"
)

var (
	checkoutCmd = &cobra.Command{
		Use:   "checkout",
		Short: "Checks out LFS files into the working copy",
		Run:   checkoutCommand,
	}
)

func checkoutCommand(cmd *cobra.Command, args []string) {

	// Parameters are filters
	checkoutWithIncludeExclude(args, nil)
}

func init() {
	RootCmd.AddCommand(checkoutCmd)
}

func checkoutWithIncludeExclude(include []string, exclude []string) {
	ref, err := git.CurrentRef()
	if err != nil {
		Panic(err, "Could not checkout")
	}

	pointers, err := lfs.ScanTree(ref)
	if err != nil {
		Panic(err, "Could not scan for Git LFS files")
	}

	var wait sync.WaitGroup
	wait.Add(1)

	c := make(chan *lfs.WrappedPointer)

	go func() {
		checkoutWithChan(c)
		wait.Done()
	}()
	for _, pointer := range pointers {
		if lfs.FilenamePassesIncludeExcludeFilter(pointer.Name, include, exclude) {
			c <- pointer
		}

	}
	close(c)
	wait.Wait()

}

func checkoutAll() {
	checkoutWithIncludeExclude(nil, nil)
}

// Populate the working copy with the real content of objects where the file is
// either missing, or contains a matching pointer placeholder, from a list of pointers.
// If the file exists but has other content it is left alone
func checkoutWithChan(in <-chan *lfs.WrappedPointer) {
	// Fire up the update-index command
	cmd := exec.Command("git", "update-index", "-q", "--refresh", "--stdin")
	updateIdxStdin, err := cmd.StdinPipe()
	if err != nil {
		Panic(err, "Could not update the index")
	}

	if err := cmd.Start(); err != nil {
		Panic(err, "Could not update the index")
	}

	// As files come in, write them to the wd and update the index
	for pointer := range in {

		// Check the content - either missing or still this pointer (not exist is ok)
		filepointer, err := lfs.DecodePointerFromFile(pointer.Name)
		if err != nil && !os.IsNotExist(err) {
			if err == lfs.NotAPointerError {
				// File has non-pointer content, leave it alone
				continue
			}
			Panic(err, "Problem accessing %v", pointer.Name)
		}
		if filepointer != nil && filepointer.Oid != pointer.Oid {
			// User has probably manually reset a file to another commit
			// while leaving it a pointer; don't mess with this
			continue
		}
		// OK now we can (over)write the file content
		file, err := os.Create(pointer.Name)
		if err != nil {
			Panic(err, "Could not create working directory file")
		}

		if err := lfs.PointerSmudge(file, pointer.Pointer, pointer.Name, nil); err != nil {
			Panic(err, "Could not write working directory file")
		}
		file.Close()

		updateIdxStdin.Write([]byte(pointer.Name + "\n"))
	}

	updateIdxStdin.Close()
	if err := cmd.Wait(); err != nil {
		Panic(err, "Error updating the git index")
	}

}
<|endoftext|>


<commit_before>package commands

import (
	"bufio"
	"fmt"
	"os"
	"strings"

	"github.com/git-lfs/git-lfs/git"
	"github.com/git-lfs/git-lfs/lfs"
	"github.com/git-lfs/git-lfs/locking"
	"github.com/rubyist/tracerx"
	"github.com/spf13/cobra"
)

var (
	prePushDryRun       = false
	prePushDeleteBranch = strings.Repeat("0", 40)
)

// prePushCommand is run through Git's pre-push hook. The pre-push hook passes
// two arguments on the command line:
//
//   1. Name of the remote to which the push is being done
//   2. URL to which the push is being done
//
// The hook receives commit information on stdin in the form:
//   <local ref> <local sha1> <remote ref> <remote sha1>
//
// In the typical case, prePushCommand will get a list of git objects being
// pushed by using the following:
//
//    git rev-list --objects <local sha1> ^<remote sha1>
//
// If any of those git objects are associated with Git LFS objects, those
// objects will be pushed to the Git LFS API.
//
// In the case of pushing a new branch, the list of git objects will be all of
// the git objects in this branch.
//
// In the case of deleting a branch, no attempts to push Git LFS objects will be
// made.
func prePushCommand(cmd *cobra.Command, args []string) {
	if len(args) == 0 {
		Print("This should be run through Git's pre-push hook.  Run `git lfs update` to install it.")
		os.Exit(1)
	}

	requireGitVersion()

	// Remote is first arg
	if err := git.ValidateRemote(args[0]); err != nil {
		Exit("Invalid remote name %q", args[0])
	}

	cfg.CurrentRemote = args[0]
	ctx := newUploadContext(prePushDryRun)

	gitscanner := lfs.NewGitScanner(nil)
	if err := gitscanner.RemoteForPush(cfg.CurrentRemote); err != nil {
		ExitWithError(err)
	}

	defer gitscanner.Close()

	// We can be passed multiple lines of refs
	scanner := bufio.NewScanner(os.Stdin)

	name, email := cfg.CurrentCommitter()
	lc, err := locking.NewClient(cfg)
	if err != nil {
		Exit("Unable to create lock system: %v", err.Error())
	}
	defer lc.Close()

	locks, err := lc.SearchLocks(map[string]string{}, 0, false)
	if err != nil {
		Exit("error finding locks: %s", err)
	}
	lockSet := make(map[string]locking.Lock, len(locks))
	for _, l := range locks {
		lockSet[l.Name] = l
	}
	lockConflicts := make([]string, 0, len(locks))
	myLocks := make([]string, 0, len(locks))

	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())

		if len(line) == 0 {
			continue
		}

		tracerx.Printf("pre-push: %s", line)

		left, _ := decodeRefs(line)
		if left == prePushDeleteBranch {
			continue
		}

		pointers, err := scanLeftOrAll(gitscanner, left)
		if err != nil {
			Print("Error scanning for Git LFS files in %q", left)
			ExitWithError(err)
		}

		for _, p := range pointers {
			if l, ok := lockSet[p.Name]; ok {
				if l.Name == name && l.Email == email {
					myLocks = append(myLocks, l.Path)
				} else {
					lockConflicts = append(lockConflicts, p.Name)
				}
			}
		}

		if len(lockConflicts) > 0 {
			Error("Some files are locked in %s...%s", left, cfg.CurrentRemote)
			for _, file := range lockConflicts {
				Error("* %s", file)
			}
			os.Exit(1)
		}

		uploadPointers(ctx, pointers)
	}

	if len(myLocks) > 0 {
		Print("Pushing your locked files:")
		for _, file := range myLocks {
			Print("* %s", file)
		}
	}
}

func scanLeft(g *lfs.GitScanner, ref string) ([]*lfs.WrappedPointer, error) {
	var pointers []*lfs.WrappedPointer
	var multiErr error
	cb := func(p *lfs.WrappedPointer, err error) {
		if err != nil {
			if multiErr != nil {
				multiErr = fmt.Errorf("%v\n%v", multiErr, err)
			} else {
				multiErr = err
			}
			return
		}

		pointers = append(pointers, p)
	}

	if err := g.ScanLeftToRemote(ref, cb); err != nil {
		return pointers, err
	}

	return pointers, multiErr
}

// decodeRefs pulls the sha1s out of the line read from the pre-push
// hook's stdin.
func decodeRefs(input string) (string, string) {
	refs := strings.Split(strings.TrimSpace(input), " ")
	var left, right string

	if len(refs) > 1 {
		left = refs[1]
	}

	if len(refs) > 3 {
		right = "^" + refs[3]
	}

	return left, right
}

func init() {
	RegisterCommand("pre-push", prePushCommand, func(cmd *cobra.Command) {
		cmd.Flags().BoolVarP(&prePushDryRun, "dry-run", "d", false, "Do everything except actually send the updates")
	})
}
<commit_msg>commands/pre-push: use findLocks helper<commit_after>package commands

import (
	"bufio"
	"fmt"
	"os"
	"strings"

	"github.com/git-lfs/git-lfs/git"
	"github.com/git-lfs/git-lfs/lfs"
	"github.com/git-lfs/git-lfs/locking"
	"github.com/rubyist/tracerx"
	"github.com/spf13/cobra"
)

var (
	prePushDryRun       = false
	prePushDeleteBranch = strings.Repeat("0", 40)
)

// prePushCommand is run through Git's pre-push hook. The pre-push hook passes
// two arguments on the command line:
//
//   1. Name of the remote to which the push is being done
//   2. URL to which the push is being done
//
// The hook receives commit information on stdin in the form:
//   <local ref> <local sha1> <remote ref> <remote sha1>
//
// In the typical case, prePushCommand will get a list of git objects being
// pushed by using the following:
//
//    git rev-list --objects <local sha1> ^<remote sha1>
//
// If any of those git objects are associated with Git LFS objects, those
// objects will be pushed to the Git LFS API.
//
// In the case of pushing a new branch, the list of git objects will be all of
// the git objects in this branch.
//
// In the case of deleting a branch, no attempts to push Git LFS objects will be
// made.
func prePushCommand(cmd *cobra.Command, args []string) {
	if len(args) == 0 {
		Print("This should be run through Git's pre-push hook.  Run `git lfs update` to install it.")
		os.Exit(1)
	}

	requireGitVersion()

	// Remote is first arg
	if err := git.ValidateRemote(args[0]); err != nil {
		Exit("Invalid remote name %q", args[0])
	}

	cfg.CurrentRemote = args[0]
	ctx := newUploadContext(prePushDryRun)

	gitscanner := lfs.NewGitScanner(nil)
	if err := gitscanner.RemoteForPush(cfg.CurrentRemote); err != nil {
		ExitWithError(err)
	}

	defer gitscanner.Close()

	// We can be passed multiple lines of refs
	scanner := bufio.NewScanner(os.Stdin)

	name, email := cfg.CurrentCommitter()
	lc, err := locking.NewClient(cfg)
	if err != nil {
		Exit("Unable to create lock system: %v", err.Error())
	}
	defer lc.Close()

	lockSet, err := findLocks(lc, nil, 0, false)
	if err != nil {
		ExitWithError(err)
	}

	lockConflicts := make([]string, 0, len(lockSet))
	myLocks := make([]string, 0, len(lockSet))

	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())

		if len(line) == 0 {
			continue
		}

		tracerx.Printf("pre-push: %s", line)

		left, _ := decodeRefs(line)
		if left == prePushDeleteBranch {
			continue
		}

		pointers, err := scanLeftOrAll(gitscanner, left)
		if err != nil {
			Print("Error scanning for Git LFS files in %q", left)
			ExitWithError(err)
		}

		for _, p := range pointers {
			if l, ok := lockSet[p.Name]; ok {
				if l.Name == name && l.Email == email {
					myLocks = append(myLocks, l.Path)
				} else {
					lockConflicts = append(lockConflicts, p.Name)
				}
			}
		}

		if len(lockConflicts) > 0 {
			Error("Some files are locked in %s...%s", left, cfg.CurrentRemote)
			for _, file := range lockConflicts {
				Error("* %s", file)
			}
			os.Exit(1)
		}

		uploadPointers(ctx, pointers)
	}

	if len(myLocks) > 0 {
		Print("Pushing your locked files:")
		for _, file := range myLocks {
			Print("* %s", file)
		}
	}
}

func scanLeft(g *lfs.GitScanner, ref string) ([]*lfs.WrappedPointer, error) {
	var pointers []*lfs.WrappedPointer
	var multiErr error
	cb := func(p *lfs.WrappedPointer, err error) {
		if err != nil {
			if multiErr != nil {
				multiErr = fmt.Errorf("%v\n%v", multiErr, err)
			} else {
				multiErr = err
			}
			return
		}

		pointers = append(pointers, p)
	}

	if err := g.ScanLeftToRemote(ref, cb); err != nil {
		return pointers, err
	}

	return pointers, multiErr
}

// decodeRefs pulls the sha1s out of the line read from the pre-push
// hook's stdin.
func decodeRefs(input string) (string, string) {
	refs := strings.Split(strings.TrimSpace(input), " ")
	var left, right string

	if len(refs) > 1 {
		left = refs[1]
	}

	if len(refs) > 3 {
		right = "^" + refs[3]
	}

	return left, right
}

func init() {
	RegisterCommand("pre-push", prePushCommand, func(cmd *cobra.Command) {
		cmd.Flags().BoolVarP(&prePushDryRun, "dry-run", "d", false, "Do everything except actually send the updates")
	})
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
)

func main() {
	fmt.Println("程序开始")

}
	<commit_msg>提供了1.3.28的答案<commit_after>package main

import (
	"fmt"
	"math/rand"
	"time"
)

func main() {
	fmt.Println("程序开始")
	rand.Seed(time.Now().UnixNano())
	n := 10
	fmt.Println("n=", n)

	first := newNode(2)
	last := first

	for i := 1; i < n; i++ {
		newND := newNode(rand.Intn(n * 3))
		last.next = newND
		last = newND
	}

	temp := first
	for temp != nil {
		fmt.Printf("%v ", temp.item)
		temp = temp.next
	}
	fmt.Println()

	maxInt := max(first)

	fmt.Println("max=", maxInt)
	fmt.Println("空链条的情况")
	fmt.Println("max(nil)=", max(nil))
}

func max(first *node) int {
	if first == nil {
		return 0
	}
	fint := first.item.(int)

	return bigger(fint, max(first.next))
}

func bigger(a, b int) int {
	if a > b {
		return a
	}
	return b
}

type node struct {
	item interface{}
	next *node
}

func newNode(item interface{}) *node {
	return &node{
		item: item,
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"log"
	"sort"

	"github.com/couchbaselabs/go-couchbase"
)

var ddocs = map[string]string{
	"jenkins": `{
		"views": {
			"data_by_build": {
				"map": "function (doc, meta) {emit(doc.build, [doc.failCount, doc.totalCount, doc.os, doc.priority, doc.component]);}"
			}
		}
	}`,
}

type DataSource struct {
	CouchbaseAddress string
	Release          string
}

func (ds *DataSource) GetBucket(bucket string) *couchbase.Bucket {
	client, _ := couchbase.Connect(ds.CouchbaseAddress)
	pool, _ := client.GetPool("default")

	b, err := pool.GetBucket(bucket)
	if err != nil {
		log.Fatalf("Error reading bucket:  %v", err)
	}
	return b
}

func (ds *DataSource) QueryView(b *couchbase.Bucket, ddoc, view string,
	params map[string]interface{}) []couchbase.ViewRow {
	params["stale"] = "false"
	vr, err := b.View(ddoc, view, params)
	if err != nil {
		ds.installDDoc(ddoc)
	}
	return vr.Rows
}

func (ds *DataSource) installDDoc(ddoc string) {
	b := ds.GetBucket(ddoc) // bucket name == ddoc name
	err := b.PutDDoc(ddoc, ddocs[ddoc])
	if err != nil {
		log.Fatalf("%v", err)
	}
}

var TIMELINE_SIZE = 40

var VIEW = map[string]int{
	"failCount":  0,
	"totalCount": 1,
	"Platform":   2,
	"Priority":   3,
	"Category":   4,
}

type MapBuild struct {
	Passed   float64
	Failed   float64
	Category string
	Platform string
	Priority string
}

type Breakdown struct {
	Passed float64
	Failed float64
}

type ReduceBuild struct {
	Version    string
	AbsPassed  float64
	AbsFailed  float64
	RelPassed  float64
	RelFailed  float64
	ByCategory map[string]Breakdown
	ByPlatform map[string]Breakdown
	ByPriority map[string]Breakdown
}

type FullSet struct {
	ByPlatform map[string]Breakdown
	ByPriority map[string]Breakdown
}

func appendIfUnique(slice []string, s string) []string {
	for i := range slice {
		if slice[i] == s {
			return slice
		}
	}
	return append(slice, s)
}

func posInSlice(slice []string, s string) int {
	for i := range slice {
		if slice[i] == s {
			return i
		}
	}
	return -1
}

func (ds *DataSource) GetTimeline() []byte {
	b := ds.GetBucket("jenkins")
	params := map[string]interface{}{"startkey": ds.Release}
	rows := ds.QueryView(b, "jenkins", "data_by_build", params)

	/***************** MAP *****************/
	mapBuilds := map[string][]MapBuild{}
	for _, row := range rows {
		version := row.Key.(string)

		value := row.Value.([]interface{})
		failed, ok := value[VIEW["failCount"]].(float64)
		if !ok {
			continue
		}
		total, ok := value[VIEW["totalCount"]].(float64)
		if !ok {
			continue
		}
		category := value[VIEW["Category"]].(string)
		platform := value[VIEW["Platform"]].(string)
		priority := value[VIEW["Priority"]].(string)

		mapBuilds[version] = append(mapBuilds[version], MapBuild{
			total - failed,
			failed,
			category,
			platform,
			priority,
		})
	}

	/***************** REDUCE *****************/
	versions := []string{}
	for version, _ := range mapBuilds {
		versions = append(versions, version)
	}
	sort.Strings(versions)

	fullSet := map[string]FullSet{}
	allCategories := []string{}
	reduceBuilds := []ReduceBuild{}

	skip := len(versions) - TIMELINE_SIZE
	for _, version := range versions[skip:] {
		currCategories := []string{}

		reduce := ReduceBuild{}
		reduce.Version = version
		reduce.ByCategory = map[string]Breakdown{}
		reduce.ByPlatform = map[string]Breakdown{}
		reduce.ByPriority = map[string]Breakdown{}

		for _, build := range mapBuilds[version] {
			// Totals
			reduce.AbsPassed += build.Passed
			reduce.AbsFailed -= build.Failed

			// By Category
			passed := build.Passed
			failed := build.Failed
			if _, ok := reduce.ByCategory[build.Category]; ok {
				passed += reduce.ByCategory[build.Category].Passed
				failed += reduce.ByCategory[build.Category].Failed
			}
			reduce.ByCategory[build.Category] = Breakdown{passed, failed}

			// By Platform
			passed = build.Passed
			failed = build.Failed
			if _, ok := reduce.ByPlatform[build.Platform]; ok {
				passed += reduce.ByPlatform[build.Platform].Passed
				failed += reduce.ByPlatform[build.Platform].Failed
			}
			reduce.ByPlatform[build.Platform] = Breakdown{passed, failed}

			// By Priority
			passed = build.Passed
			failed = build.Failed
			if _, ok := reduce.ByPriority[build.Priority]; ok {
				passed += reduce.ByPriority[build.Priority].Passed
				failed += reduce.ByPriority[build.Priority].Failed
			}
			reduce.ByPriority[build.Priority] = Breakdown{passed, failed}

			// Full Set
			if posInSlice(currCategories, build.Category) == -1 {
				byPlatform := map[string]Breakdown{
					build.Platform: Breakdown{build.Passed, build.Failed},
				}
				byPriority := map[string]Breakdown{
					build.Priority: Breakdown{build.Passed, build.Failed},
				}
				fullSet[build.Category] = FullSet{byPlatform, byPriority}
			} else {
				// By Platform
				passed := build.Passed + fullSet[build.Category].ByPlatform[build.Platform].Passed
				failed := build.Failed + fullSet[build.Category].ByPlatform[build.Platform].Failed
				fullSet[build.Category].ByPlatform[build.Platform] = Breakdown{passed, failed}

				// By Priority
				passed = build.Passed + fullSet[build.Category].ByPriority[build.Priority].Passed
				failed = build.Failed + fullSet[build.Category].ByPriority[build.Priority].Failed
				fullSet[build.Category].ByPriority[build.Priority] = Breakdown{passed, failed}
			}

			// Categories
			currCategories = appendIfUnique(currCategories, build.Category)
			allCategories = appendIfUnique(allCategories, build.Category)
		}

		/***************** BACKFILL *****************/
		for _, category := range allCategories {
			totalPassed := float64(0)
			totalFailed := float64(0)
			if _, ok := reduce.ByCategory[category]; !ok {
				for platform, breakdown := range fullSet[category].ByPlatform {
					reduce.AbsPassed += breakdown.Passed
					reduce.AbsFailed += breakdown.Failed

					passed := breakdown.Passed
					failed := breakdown.Failed
					if _, ok := reduce.ByPlatform[platform]; ok {
						passed += reduce.ByPlatform[platform].Passed
						failed += reduce.ByPlatform[platform].Failed

					}
					reduce.ByPlatform[platform] = Breakdown{passed, failed}
				}

				for priority, breakdown := range fullSet[category].ByPriority {
					passed := breakdown.Passed
					failed := breakdown.Failed
					if _, ok := reduce.ByPriority[priority]; ok {
						passed += reduce.ByPriority[priority].Passed
						failed += reduce.ByPriority[priority].Failed
					}
					reduce.ByPriority[priority] = Breakdown{passed, failed}
				}
				reduce.ByCategory[category] = Breakdown{totalPassed, totalFailed}
			}
		}

		total := reduce.AbsPassed - reduce.AbsFailed
		reduce.RelPassed = 100.0 * reduce.AbsPassed / total
		reduce.RelFailed = -100.0 * reduce.AbsFailed / total
		reduceBuilds = append(reduceBuilds, reduce)
	}

	j, _ := json.Marshal(reduceBuilds)
	return j
}
<commit_msg>fix regression after refactoring<commit_after>package main

import (
	"encoding/json"
	"log"
	"sort"

	"github.com/couchbaselabs/go-couchbase"
)

var ddocs = map[string]string{
	"jenkins": `{
		"views": {
			"data_by_build": {
				"map": "function (doc, meta) {emit(doc.build, [doc.failCount, doc.totalCount, doc.os, doc.priority, doc.component]);}"
			}
		}
	}`,
}

type DataSource struct {
	CouchbaseAddress string
	Release          string
}

func (ds *DataSource) GetBucket(bucket string) *couchbase.Bucket {
	client, _ := couchbase.Connect(ds.CouchbaseAddress)
	pool, _ := client.GetPool("default")

	b, err := pool.GetBucket(bucket)
	if err != nil {
		log.Fatalf("Error reading bucket:  %v", err)
	}
	return b
}

func (ds *DataSource) QueryView(b *couchbase.Bucket, ddoc, view string,
	params map[string]interface{}) []couchbase.ViewRow {
	params["stale"] = "false"
	vr, err := b.View(ddoc, view, params)
	if err != nil {
		ds.installDDoc(ddoc)
	}
	return vr.Rows
}

func (ds *DataSource) installDDoc(ddoc string) {
	b := ds.GetBucket(ddoc) // bucket name == ddoc name
	err := b.PutDDoc(ddoc, ddocs[ddoc])
	if err != nil {
		log.Fatalf("%v", err)
	}
}

var TIMELINE_SIZE = 40

var VIEW = map[string]int{
	"failCount":  0,
	"totalCount": 1,
	"Platform":   2,
	"Priority":   3,
	"Category":   4,
}

type MapBuild struct {
	Passed   float64
	Failed   float64
	Category string
	Platform string
	Priority string
}

type Breakdown struct {
	Passed float64
	Failed float64
}

type ReduceBuild struct {
	Version    string
	AbsPassed  float64
	AbsFailed  float64
	RelPassed  float64
	RelFailed  float64
	ByCategory map[string]Breakdown
	ByPlatform map[string]Breakdown
	ByPriority map[string]Breakdown
}

type FullSet struct {
	ByPlatform map[string]Breakdown
	ByPriority map[string]Breakdown
}

func appendIfUnique(slice []string, s string) []string {
	for i := range slice {
		if slice[i] == s {
			return slice
		}
	}
	return append(slice, s)
}

func posInSlice(slice []string, s string) int {
	for i := range slice {
		if slice[i] == s {
			return i
		}
	}
	return -1
}

func (ds *DataSource) GetTimeline() []byte {
	b := ds.GetBucket("jenkins")
	params := map[string]interface{}{"startkey": ds.Release}
	rows := ds.QueryView(b, "jenkins", "data_by_build", params)

	/***************** MAP *****************/
	mapBuilds := map[string][]MapBuild{}
	for _, row := range rows {
		version := row.Key.(string)

		value := row.Value.([]interface{})
		failed, ok := value[VIEW["failCount"]].(float64)
		if !ok {
			continue
		}
		total, ok := value[VIEW["totalCount"]].(float64)
		if !ok {
			continue
		}
		category := value[VIEW["Category"]].(string)
		platform := value[VIEW["Platform"]].(string)
		priority := value[VIEW["Priority"]].(string)

		mapBuilds[version] = append(mapBuilds[version], MapBuild{
			total - failed,
			failed,
			category,
			platform,
			priority,
		})
	}

	/***************** REDUCE *****************/
	versions := []string{}
	for version, _ := range mapBuilds {
		versions = append(versions, version)
	}
	sort.Strings(versions)

	fullSet := map[string]FullSet{}
	allCategories := []string{}
	reduceBuilds := []ReduceBuild{}

	skip := len(versions) - TIMELINE_SIZE
	for _, version := range versions[skip:] {
		currCategories := []string{}

		reduce := ReduceBuild{}
		reduce.Version = version
		reduce.ByCategory = map[string]Breakdown{}
		reduce.ByPlatform = map[string]Breakdown{}
		reduce.ByPriority = map[string]Breakdown{}

		for _, build := range mapBuilds[version] {
			// Totals
			reduce.AbsPassed += build.Passed
			reduce.AbsFailed -= build.Failed

			// By Category
			passed := build.Passed
			failed := build.Failed
			if _, ok := reduce.ByCategory[build.Category]; ok {
				passed += reduce.ByCategory[build.Category].Passed
				failed += reduce.ByCategory[build.Category].Failed
			}
			reduce.ByCategory[build.Category] = Breakdown{passed, failed}

			// By Platform
			passed = build.Passed
			failed = build.Failed
			if _, ok := reduce.ByPlatform[build.Platform]; ok {
				passed += reduce.ByPlatform[build.Platform].Passed
				failed += reduce.ByPlatform[build.Platform].Failed
			}
			reduce.ByPlatform[build.Platform] = Breakdown{passed, failed}

			// By Priority
			passed = build.Passed
			failed = build.Failed
			if _, ok := reduce.ByPriority[build.Priority]; ok {
				passed += reduce.ByPriority[build.Priority].Passed
				failed += reduce.ByPriority[build.Priority].Failed
			}
			reduce.ByPriority[build.Priority] = Breakdown{passed, failed}

			// Full Set
			if posInSlice(currCategories, build.Category) == -1 {
				byPlatform := map[string]Breakdown{
					build.Platform: Breakdown{build.Passed, build.Failed},
				}
				byPriority := map[string]Breakdown{
					build.Priority: Breakdown{build.Passed, build.Failed},
				}
				fullSet[build.Category] = FullSet{byPlatform, byPriority}
			} else {
				// By Platform
				passed := build.Passed + fullSet[build.Category].ByPlatform[build.Platform].Passed
				failed := build.Failed + fullSet[build.Category].ByPlatform[build.Platform].Failed
				fullSet[build.Category].ByPlatform[build.Platform] = Breakdown{passed, failed}

				// By Priority
				passed = build.Passed + fullSet[build.Category].ByPriority[build.Priority].Passed
				failed = build.Failed + fullSet[build.Category].ByPriority[build.Priority].Failed
				fullSet[build.Category].ByPriority[build.Priority] = Breakdown{passed, failed}
			}

			// Categories
			currCategories = appendIfUnique(currCategories, build.Category)
			allCategories = appendIfUnique(allCategories, build.Category)
		}

		/***************** BACKFILL *****************/
		for _, category := range allCategories {
			totalPassed := float64(0)
			totalFailed := float64(0)
			if _, ok := reduce.ByCategory[category]; !ok {
				for platform, breakdown := range fullSet[category].ByPlatform {
					totalPassed += breakdown.Passed
					totalFailed += breakdown.Failed

					passed := breakdown.Passed
					failed := breakdown.Failed
					if _, ok := reduce.ByPlatform[platform]; ok {
						passed += reduce.ByPlatform[platform].Passed
						failed += reduce.ByPlatform[platform].Failed

					}
					reduce.ByPlatform[platform] = Breakdown{passed, failed}
				}

				for priority, breakdown := range fullSet[category].ByPriority {
					passed := breakdown.Passed
					failed := breakdown.Failed
					if _, ok := reduce.ByPriority[priority]; ok {
						passed += reduce.ByPriority[priority].Passed
						failed += reduce.ByPriority[priority].Failed
					}
					reduce.ByPriority[priority] = Breakdown{passed, failed}
				}

				reduce.AbsPassed += totalPassed
				reduce.AbsFailed -= totalFailed
				reduce.ByCategory[category] = Breakdown{totalPassed, totalFailed}
			}
		}

		total := reduce.AbsPassed - reduce.AbsFailed
		reduce.RelPassed = 100.0 * reduce.AbsPassed / total
		reduce.RelFailed = -100.0 * reduce.AbsFailed / total
		reduceBuilds = append(reduceBuilds, reduce)
	}

	j, _ := json.Marshal(reduceBuilds)
	return j
}
<|endoftext|>


<commit_before>package fileutil_test

import (
	"archive/tar"
	"github.com/APTrust/easy-store/constants"
	"github.com/APTrust/easy-store/util/fileutil"
	"github.com/APTrust/easy-store/util/testutil"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"testing"
)

var aptrustFile = pathToFile("aptrust_bagit_profile_2.1.json")
var dpnFile = pathToFile("dpn_bagit_profile_2.1.json")
var algorithms = []string{constants.MD5, constants.SHA256}

func TestNewTarWriter(t *testing.T) {
	dir, err := ioutil.TempDir("", "tarwriter_test")
	if err != nil {
		assert.FailNow(t, "Cannot create temp dir", err.Error())
	}
	tempFilePath := filepath.Join(dir, "test_file.tar")
	defer os.RemoveAll(dir)
	w := fileutil.NewTarWriter(tempFilePath)
	assert.NotNil(t, w)
	assert.Equal(t, tempFilePath, w.PathToTarFile)
}

func TestAndCloseOpen(t *testing.T) {
	dir, err := ioutil.TempDir("", "tarwriter_test")
	if err != nil {
		assert.FailNow(t, "Cannot create temp dir", err.Error())
	}
	tempFilePath := filepath.Join(dir, "test_file.tar")
	defer os.RemoveAll(dir)
	w := fileutil.NewTarWriter(tempFilePath)
	defer w.Close()
	err = w.Open()
	assert.Nil(t, err)
	if _, err := os.Stat(w.PathToTarFile); os.IsNotExist(err) {
		assert.Fail(t, "Tar file does not exist at %s", w.PathToTarFile)
	}
	err = w.Close()
	assert.Nil(t, err)
}

func TestAddToArchive(t *testing.T) {
	dir, err := ioutil.TempDir("", "tarwriter_test")
	if err != nil {
		assert.FailNow(t, "Cannot create temp dir", err.Error())
	}
	tempFilePath := filepath.Join(dir, "test_file.tar")
	defer os.RemoveAll(dir)
	w := fileutil.NewTarWriter(tempFilePath)
	defer w.Close()
	err = w.Open()
	assert.Nil(t, err)
	if _, err := os.Stat(w.PathToTarFile); os.IsNotExist(err) {
		assert.Fail(t, "Tar file does not exist at %s", w.PathToTarFile)
	}
	checksums, err := w.AddToArchive(aptrustFile, "file1.json", algorithms)
	assert.Nil(t, err)
	assert.Equal(t, "807c49b728c85a563990c9c00bda2ca8", checksums[constants.MD5])
	assert.Equal(t, "e28918b4941d30f34824f5e53f2c6c98952f577b21a8cc98347c172f3a4806f3", checksums[constants.SHA256])
	checksums, err = w.AddToArchive(dpnFile, "data/subdir/file2.json", algorithms)
	assert.Nil(t, err)
	assert.Equal(t, "519e171ee45707b8bac1e4a2fd514e9a", checksums[constants.MD5])
	assert.Equal(t, "0fa24fecb8a2333ca32fd85c6c1cc073a8e73a5c0371bc56280a9abe11a72310", checksums[constants.SHA256])
	w.Close()

	file, err := os.Open(w.PathToTarFile)
	if file != nil {
		defer file.Close()
	}
	if err != nil {
		assert.FailNow(t, "Could not open tar file", err.Error())
	}
	filesInArchive := make([]string, 0)
	reader := tar.NewReader(file)
	for {
		header, err := reader.Next()
		if err != nil {
			break
		}
		filesInArchive = append(filesInArchive, header.Name)
	}
	assert.Equal(t, "file1.json", filesInArchive[0])
	assert.Equal(t, "data/subdir/file2.json", filesInArchive[1])
}

func TestAddToArchiveWithClosedWriter(t *testing.T) {
	dir, err := ioutil.TempDir("", "tarwriter_test")
	if err != nil {
		assert.FailNow(t, "Cannot create temp dir", err.Error())
	}
	tempFilePath := filepath.Join(dir, "test_file.tar")
	defer os.RemoveAll(dir)
	w := fileutil.NewTarWriter(tempFilePath)

	// Note that we have not opened the writer
	_, err = w.AddToArchive(aptrustFile, "file1.json", algorithms)
	require.NotNil(t, err)
	assert.True(t, strings.HasPrefix(err.Error(), "Underlying TarWriter is nil"))

	// Open and close the writer, so the file exists.
	w.Open()
	w.Close()
	if _, err := os.Stat(w.PathToTarFile); os.IsNotExist(err) {
		assert.Fail(t, "Tar file does not exist at %s", w.PathToTarFile)
	}
	_, err = w.AddToArchive(aptrustFile, "file1.json", algorithms)
	require.NotNil(t, err)
	assert.True(t, strings.HasPrefix(err.Error(), "archive/tar: write after close"))

}

func TestAddToArchiveWithBadFilePath(t *testing.T) {
	dir, err := ioutil.TempDir("", "tarwriter_test")
	if err != nil {
		assert.FailNow(t, "Cannot create temp dir", err.Error())
	}
	tempFilePath := filepath.Join(dir, "test_file.tar")
	defer os.RemoveAll(dir)
	w := fileutil.NewTarWriter(tempFilePath)
	defer w.Close()
	err = w.Open()
	assert.Nil(t, err)
	if _, err := os.Stat(w.PathToTarFile); os.IsNotExist(err) {
		assert.Fail(t, "Tar file does not exist at %s", w.PathToTarFile)
	}

	// This file doesn't exist. Make sure we get the right error.
	_, err = w.AddToArchive(pathToFile("this_file_does_not_exist"), "file1.json", algorithms)
	require.NotNil(t, err)
	assert.True(t, strings.Contains(err.Error(), "no such file or directory"))
}

func pathToFile(filename string) string {
	fullPath, _ := testutil.GetPathToTestProfile(filename)
	return fullPath
}
<commit_msg>Updated checksums to match altered files<commit_after>package fileutil_test

import (
	"archive/tar"
	"github.com/APTrust/easy-store/constants"
	"github.com/APTrust/easy-store/util/fileutil"
	"github.com/APTrust/easy-store/util/testutil"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"testing"
)

var aptrustFile = pathToFile("aptrust_bagit_profile_2.1.json")
var dpnFile = pathToFile("dpn_bagit_profile_2.1.json")
var algorithms = []string{constants.MD5, constants.SHA256}

func TestNewTarWriter(t *testing.T) {
	dir, err := ioutil.TempDir("", "tarwriter_test")
	if err != nil {
		assert.FailNow(t, "Cannot create temp dir", err.Error())
	}
	tempFilePath := filepath.Join(dir, "test_file.tar")
	defer os.RemoveAll(dir)
	w := fileutil.NewTarWriter(tempFilePath)
	assert.NotNil(t, w)
	assert.Equal(t, tempFilePath, w.PathToTarFile)
}

func TestAndCloseOpen(t *testing.T) {
	dir, err := ioutil.TempDir("", "tarwriter_test")
	if err != nil {
		assert.FailNow(t, "Cannot create temp dir", err.Error())
	}
	tempFilePath := filepath.Join(dir, "test_file.tar")
	defer os.RemoveAll(dir)
	w := fileutil.NewTarWriter(tempFilePath)
	defer w.Close()
	err = w.Open()
	assert.Nil(t, err)
	if _, err := os.Stat(w.PathToTarFile); os.IsNotExist(err) {
		assert.Fail(t, "Tar file does not exist at %s", w.PathToTarFile)
	}
	err = w.Close()
	assert.Nil(t, err)
}

func TestAddToArchive(t *testing.T) {
	dir, err := ioutil.TempDir("", "tarwriter_test")
	if err != nil {
		assert.FailNow(t, "Cannot create temp dir", err.Error())
	}
	tempFilePath := filepath.Join(dir, "test_file.tar")
	defer os.RemoveAll(dir)
	w := fileutil.NewTarWriter(tempFilePath)
	defer w.Close()
	err = w.Open()
	assert.Nil(t, err)
	if _, err := os.Stat(w.PathToTarFile); os.IsNotExist(err) {
		assert.Fail(t, "Tar file does not exist at %s", w.PathToTarFile)
	}
	checksums, err := w.AddToArchive(aptrustFile, "file1.json", algorithms)
	assert.Nil(t, err)
	assert.Equal(t, "567751e62ba368249c31e2248ba6af6b", checksums[constants.MD5])
	assert.Equal(t, "987b2570a72937297f076c3d7b0fd74516aa470bee37db3f95cfe7d9e2795c62", checksums[constants.SHA256])
	checksums, err = w.AddToArchive(dpnFile, "data/subdir/file2.json", algorithms)
	assert.Nil(t, err)
	assert.Equal(t, "5b87c90ff42fb31e0923773924cd7017", checksums[constants.MD5])
	assert.Equal(t, "fd8037350267d6920da34358bdc19f0a29ec7e9b237b42b2f9f7d3ac3d556213", checksums[constants.SHA256])
	w.Close()

	file, err := os.Open(w.PathToTarFile)
	if file != nil {
		defer file.Close()
	}
	if err != nil {
		assert.FailNow(t, "Could not open tar file", err.Error())
	}
	filesInArchive := make([]string, 0)
	reader := tar.NewReader(file)
	for {
		header, err := reader.Next()
		if err != nil {
			break
		}
		filesInArchive = append(filesInArchive, header.Name)
	}
	assert.Equal(t, "file1.json", filesInArchive[0])
	assert.Equal(t, "data/subdir/file2.json", filesInArchive[1])
}

func TestAddToArchiveWithClosedWriter(t *testing.T) {
	dir, err := ioutil.TempDir("", "tarwriter_test")
	if err != nil {
		assert.FailNow(t, "Cannot create temp dir", err.Error())
	}
	tempFilePath := filepath.Join(dir, "test_file.tar")
	defer os.RemoveAll(dir)
	w := fileutil.NewTarWriter(tempFilePath)

	// Note that we have not opened the writer
	_, err = w.AddToArchive(aptrustFile, "file1.json", algorithms)
	require.NotNil(t, err)
	assert.True(t, strings.HasPrefix(err.Error(), "Underlying TarWriter is nil"))

	// Open and close the writer, so the file exists.
	w.Open()
	w.Close()
	if _, err := os.Stat(w.PathToTarFile); os.IsNotExist(err) {
		assert.Fail(t, "Tar file does not exist at %s", w.PathToTarFile)
	}
	_, err = w.AddToArchive(aptrustFile, "file1.json", algorithms)
	require.NotNil(t, err)
	assert.True(t, strings.HasPrefix(err.Error(), "archive/tar: write after close"))

}

func TestAddToArchiveWithBadFilePath(t *testing.T) {
	dir, err := ioutil.TempDir("", "tarwriter_test")
	if err != nil {
		assert.FailNow(t, "Cannot create temp dir", err.Error())
	}
	tempFilePath := filepath.Join(dir, "test_file.tar")
	defer os.RemoveAll(dir)
	w := fileutil.NewTarWriter(tempFilePath)
	defer w.Close()
	err = w.Open()
	assert.Nil(t, err)
	if _, err := os.Stat(w.PathToTarFile); os.IsNotExist(err) {
		assert.Fail(t, "Tar file does not exist at %s", w.PathToTarFile)
	}

	// This file doesn't exist. Make sure we get the right error.
	_, err = w.AddToArchive(pathToFile("this_file_does_not_exist"), "file1.json", algorithms)
	require.NotNil(t, err)
	assert.True(t, strings.Contains(err.Error(), "no such file or directory"))
}

func pathToFile(filename string) string {
	fullPath, _ := testutil.GetPathToTestProfile(filename)
	return fullPath
}
<|endoftext|>


<commit_before>/* Collection chunks coordination test. */
package db

import (
	"encoding/json"
	"fmt"
	"github.com/HouzuoGuo/tiedot/chunkfile"
	"os"
	"strconv"
	"strings"
	"testing"
)

func TestDocIndexAndCRUD(t *testing.T) {
	tmp := "/tmp/tiedot_col_test"
	os.RemoveAll(tmp)
	defer os.RemoveAll(tmp)
	col, err := OpenCol(tmp)
	if err != nil {
		t.Fatal(err)
	}
	// Create an index
	if err = col.Index([]string{"id"}); err != nil {
		t.Fatal(err)
	}
	// Insert 10 long documents, afterwards there should be 5 chunks
	longDocIDs := make([]uint64, 10)
	for i := 0; i < 10; i++ {
		var longDoc interface{}
		json.Unmarshal([]byte(`{"a": "`+strings.Repeat("1", int(chunkfile.DOC_MAX_ROOM/6))+`", "id": `+strconv.Itoa(i)+`}`), &longDoc)
		if longDocIDs[i], err = col.Insert(longDoc); err != nil {
			t.Fatal(err)
		}
	}
	if !(len(col.ChunkMutexes) == 5 && len(col.Chunks) == 5 && col.NumChunks == 5) {
		t.Fatalf("Wrong number of chunks, got %d instead", col.NumChunks)
	}
	// There should still be enough room for 10 very short documents - even if they all go into one chunk
	shortDocIDs := make([]uint64, 10)
	for i := 0; i < 10; i++ {
		var shortDoc interface{}
		json.Unmarshal([]byte(`{"id": `+strconv.Itoa(i)+`}`), &shortDoc)
		if shortDocIDs[i], err = col.Insert(shortDoc); err != nil {
			t.Fatal(err)
		}
	}
	if !(len(col.ChunkMutexes) == 5 && len(col.Chunks) == 5 && col.NumChunks == 5) {
		t.Fatalf("There are too many chunks %d", col.NumChunks)
	}
	// Verify that docs can be read back using IDs
	matchIDAttr := func(doc interface{}, toMatch int) {
		docMap, _ := doc.(map[string]interface{})
		idAttr := docMap["id"]
		idAttrFloat, _ := idAttr.(float64)
		if int(idAttrFloat) != toMatch {
			t.Fatal("ID attribute mismatch")
		}
	}
	for i := 0; i < 10; i++ {
		var doc interface{}
		if err = col.Read(longDocIDs[i], &doc); err != nil {
			t.Fatal(err)
		}
		matchIDAttr(doc, i)
		if err = col.Read(shortDocIDs[i], &doc); err != nil {
			t.Fatal(err)
		}
		matchIDAttr(doc, i)
	}
	// Update a short document - no relocation
	var shortDoc interface{}
	err = json.Unmarshal([]byte(`{"id": 11}`), &shortDoc)
	if err != nil {
		t.Fatal(err)
	}
	shortNewID, err := col.Update(shortDocIDs[0], shortDoc)
	if err != nil {
		t.Fatal(err)
	} else if shortNewID != shortDocIDs[0] {
		t.Fatal("Doc was relocated, but it should not happen")
	}
	col.Read(shortNewID, &shortDoc)
	matchIDAttr(shortDoc, 11)
	// Reopen the collection
	col.Close()
	if col, err = OpenCol(tmp); err != nil {
		t.Fatal(err)
	}
	// Update a long document - relocation happens
	var longDoc interface{}
	json.Unmarshal([]byte(`{"a": "`+strings.Repeat("1", int(chunkfile.DOC_MAX_ROOM/3+2048))+`", "id": 11}`), &longDoc)
	longNewID, err := col.Update(longDocIDs[0], longDoc)
	if err != nil {
		t.Fatal(err)
	} else if longNewID == longDocIDs[0] {
		t.Fatal("Doc should be relocated, it did not happen")
	} else if col.NumChunks != 6 {
		// Relocated document has to go to a new chunk, therefore chunk number should be 6
		t.Fatal("Relocated doc should cause a new chunk to be created")
	}
	col.Read(longNewID, &longDoc)
	matchIDAttr(longDoc, 11)
	// Scrub the entire collection, number of chunks should remain
	if col.Scrub() != 20 {
		t.Fatal("Scrub recovered wrong number of documents")
	}
	if col.NumChunks != 6 {
		t.Fatal("Scrub caused chunk number change")
	}
	if err = col.Flush(); err != nil {
		t.Fatal(err)
	}
	// All chunks should have identical indexes
	for _, chunk := range col.Chunks {
		if _, ok := chunk.Path2HT["id"]; !ok {
			t.Fatal("Chunk does not have id index")
		}
	}
	// Delete a document
	col.Delete(longDocIDs[7])
	if err = col.Read(longDocIDs[7], &longDoc); err == nil {
		t.Fatal("Did not delete the document")
	}

	// Two collection scan
	counter := 0
	var template interface{}
	var throwAway interface{}
	col.DeserializeAll(&template, func(id uint64) bool {
		if err = col.Read(id, &throwAway); err != nil {
			t.Fatal(err)
		}
		counter++
		return true
	})
	if counter != 19 { // 10 long docs, 10 short docs, one less long doc
		t.Fatal("Collection scan wrong number of docs")
	}
	counter = 0
	col.ForAll(func(id uint64, doc interface{}) bool {
		if err = col.Read(id, &throwAway); err != nil {
			t.Fatal(err)
		}
		counter++
		return true
	})
	if counter != 19 {
		t.Fatal("Collection scan wrong number of docs")
	}
	// Remove index - it should be removed from all chunks
	if err = col.Unindex([]string{"id"}); err != nil {
		t.Fatal(err)
	}
	for _, chunk := range col.Chunks {
		if _, ok := chunk.Path2HT["id"]; ok {
			t.Fatal("Chunk should not have id index")
		}
	}
	// Out of bound access
	if err = col.Read(9999999, &longDoc); err == nil {
		t.Fatal("Out of bound access did not return error")
	}
	if _, err = col.Update(9999999, longDoc); err == nil {
		t.Fatal("Out of bound access did not return error")
	}
	col.Delete(99999999) // shall not crash
	col.Close()
}

func TestUIDDocCRUD(t *testing.T) {
	tmp := "/tmp/tiedot_col_test"
	os.RemoveAll(tmp)
	defer os.RemoveAll(tmp)
	col, err := OpenCol(tmp)
	if err != nil {
		t.Fatalf("Failed to open: %v", err)
		return
	}
	docs := []string{
		`{"a": {"b": {"c": 1}}, "d": 1, "more": "` + strings.Repeat(" ", int(chunkfile.DOC_MAX_ROOM/3)) + `"}`,
		`{"a": {"b": {"c": 2}}, "d": 2, "more": "` + strings.Repeat(" ", int(chunkfile.DOC_MAX_ROOM/3)) + `"}`,
		`{"a": {"b": {"c": 3}}, "d": 3, "more": "` + strings.Repeat(" ", int(chunkfile.DOC_MAX_ROOM/3)) + `"}`}
	var jsonDocs [3]interface{}
	var ids [3]uint64
	var uids [3]string
	json.Unmarshal([]byte(docs[0]), &jsonDocs[0])
	json.Unmarshal([]byte(docs[1]), &jsonDocs[1])
	json.Unmarshal([]byte(docs[2]), &jsonDocs[2])
	// insert
	ids[0], uids[0], err = col.InsertWithUID(jsonDocs[0])
	if err != nil {
		t.Fatal("insert error")
	}
	ids[1], uids[1], err = col.InsertWithUID(jsonDocs[1])
	if err != nil {
		t.Fatal("insert error")
	}
	ids[2], uids[2], err = col.InsertWithUID(jsonDocs[2])
	if err != nil {
		t.Fatal("insert error")
	}
	if len(uids[0]) != 32 || len(uids[1]) != 32 || len(uids[2]) != 32 ||
		uids[0] == uids[1] || uids[1] == uids[2] || uids[2] == uids[0] ||
		ids[0] == ids[1] || ids[1] == ids[2] || ids[2] == ids[0] {
		t.Fatalf("Malformed UIDs or IDs: %v %v", uids, ids)
	}
	// read - inexisting UID
	var readDoc interface{}
	if _, readErr := col.ReadByUID("abcde", &readDoc); readErr == nil {
		t.Fatal("It should have triggered UID not found error")
	}
	// read - existing UID
	readID, readErr := col.ReadByUID(uids[1], &readDoc)
	if readErr != nil {
		t.Fatal(readErr)
	}
	docMap1 := readDoc.(map[string]interface{})
	docMap2 := jsonDocs[1].(map[string]interface{})
	if readID != ids[1] || fmt.Sprint(docMap1["a"]) != fmt.Sprint(docMap2["a"]) {
		t.Fatalf("Cannot read back original document by UID: %d %d", readID, ids[1])
	}
	// update
	var docWithoutUID interface{}
	json.Unmarshal([]byte(docs[1]), &docWithoutUID)
	if _, err := col.UpdateByUID(uids[0], docWithoutUID); err != nil { // intentionally remove UID
		t.Fatal(err)
	}
	if _, err = col.ReadByUID(uids[0], &readDoc); err == nil { // UID was removed therefore the UID is not found
		t.Fatalf("UpdateByUID did not work, still read %v", readDoc)
	}
	// update (reassign UID)
	_, newUID, err := col.ReassignUID(ids[0])
	if len(newUID) != 32 || err != nil {
		t.Fatalf("ReassignUID did not work: %v %v %v", ids[0], newUID, err)
	}
	if _, err = col.ReadByUID(uids[1], &readDoc); err != nil {
		t.Fatalf("col failed UID index? %s %v", uids[1], err)
	}
	// after UID reassignment, the old UID should be gone
	if _, readErr := col.ReadByUID(uids[0], &readDoc); readErr == nil {
		t.Fatal("It should have triggered UID not found error")
	}
	if _, err = col.ReadByUID(newUID, &readDoc); err != nil { // UID was reassigned, the error should NOT happen
		t.Fatalf("ReassignUID did not work")
	}
	// delete
	col.DeleteByUID(newUID)
	if _, err = col.ReadByUID(newUID, &readDoc); err == nil {
		t.Fatalf("DeleteByUID did not work")
	}
	if _, err = col.ReadByUID(uids[1], &readDoc); err != nil {
		t.Fatalf("col failed UID index? %s %v", uids[1], err)
	}
	col.Close()
	// Reopen and test read again
	reopen, err := OpenCol(tmp)
	if err != nil {
		t.Fatal(err)
	}
	// UID index should work
	if _, err = reopen.ReadByUID(uids[1], &readDoc); err != nil {
		t.Fatalf("Reopen failed UID index? %s %v", uids[1], err)
	}
	reopen.Close()
}
<commit_msg>fix a minor test case mistake<commit_after>/* Collection chunks coordination test. */
package db

import (
	"encoding/json"
	"fmt"
	"github.com/HouzuoGuo/tiedot/chunkfile"
	"os"
	"strconv"
	"strings"
	"testing"
)

func TestDocIndexAndCRUD(t *testing.T) {
	tmp := "/tmp/tiedot_col_test"
	os.RemoveAll(tmp)
	defer os.RemoveAll(tmp)
	col, err := OpenCol(tmp)
	if err != nil {
		t.Fatal(err)
	}
	// Create an index
	if err = col.Index([]string{"id"}); err != nil {
		t.Fatal(err)
	}
	// Insert 10 long documents, afterwards there should be 5 chunks
	longDocIDs := make([]uint64, 10)
	for i := 0; i < 10; i++ {
		var longDoc interface{}
		json.Unmarshal([]byte(`{"a": "`+strings.Repeat("1", int(chunkfile.DOC_MAX_ROOM/6))+`", "id": `+strconv.Itoa(i)+`}`), &longDoc)
		if longDocIDs[i], err = col.Insert(longDoc); err != nil {
			t.Fatal(err)
		}
	}
	if !(len(col.ChunkMutexes) == 5 && len(col.Chunks) == 5 && col.NumChunks == 5) {
		t.Fatalf("Wrong number of chunks, got %d instead", col.NumChunks)
	}
	// There should still be enough room for 10 very short documents - even if they all go into one chunk
	shortDocIDs := make([]uint64, 10)
	for i := 0; i < 10; i++ {
		var shortDoc interface{}
		json.Unmarshal([]byte(`{"id": `+strconv.Itoa(i)+`}`), &shortDoc)
		if shortDocIDs[i], err = col.Insert(shortDoc); err != nil {
			t.Fatal(err)
		}
	}
	if !(len(col.ChunkMutexes) == 5 && len(col.Chunks) == 5 && col.NumChunks == 5) {
		t.Fatalf("There are too many chunks %d", col.NumChunks)
	}
	// Verify that docs can be read back using IDs
	matchIDAttr := func(doc interface{}, toMatch int) {
		docMap, _ := doc.(map[string]interface{})
		idAttr := docMap["id"]
		idAttrFloat, _ := idAttr.(float64)
		if int(idAttrFloat) != toMatch {
			t.Fatal("ID attribute mismatch")
		}
	}
	for i := 0; i < 10; i++ {
		var doc interface{}
		if err = col.Read(longDocIDs[i], &doc); err != nil {
			t.Fatal(err)
		}
		matchIDAttr(doc, i)
		if err = col.Read(shortDocIDs[i], &doc); err != nil {
			t.Fatal(err)
		}
		matchIDAttr(doc, i)
	}
	// Update a short document - no relocation
	var shortDoc interface{}
	err = json.Unmarshal([]byte(`{"id": 11}`), &shortDoc)
	if err != nil {
		t.Fatal(err)
	}
	shortNewID, err := col.Update(shortDocIDs[0], shortDoc)
	if err != nil {
		t.Fatal(err)
	} else if shortNewID != shortDocIDs[0] {
		t.Fatal("Doc was relocated, but it should not happen")
	}
	col.Read(shortNewID, &shortDoc)
	matchIDAttr(shortDoc, 11)
	// Reopen the collection
	col.Close()
	if col, err = OpenCol(tmp); err != nil {
		t.Fatal(err)
	}
	// Update a long document - relocation happens
	var longDoc interface{}
	json.Unmarshal([]byte(`{"a": "`+strings.Repeat("1", int(chunkfile.DOC_MAX_ROOM/3+2048))+`", "id": 11}`), &longDoc)
	longNewID, err := col.Update(longDocIDs[0], longDoc)
	if err != nil {
		t.Fatal(err)
	} else if longNewID == longDocIDs[0] {
		t.Fatal("Doc should be relocated, it did not happen")
	} else if col.NumChunks != 6 {
		// Relocated document has to go to a new chunk, therefore chunk number should be 6
		t.Fatal("Relocated doc should cause a new chunk to be created")
	}
	col.Read(longNewID, &longDoc)
	matchIDAttr(longDoc, 11)
	// Scrub the entire collection, number of chunks should remain
	if col.Scrub() != 20 {
		t.Fatal("Scrub recovered wrong number of documents")
	}
	if col.NumChunks != 6 {
		t.Fatal("Scrub caused chunk number change")
	}
	if err = col.Flush(); err != nil {
		t.Fatal(err)
	}
	// All chunks should have identical indexes
	for _, chunk := range col.Chunks {
		if _, ok := chunk.Path2HT["id"]; !ok {
			t.Fatal("Chunk does not have id index")
		}
	}
	// Delete a document
	col.Delete(longDocIDs[7])
	if err = col.Read(longDocIDs[7], &longDoc); err == nil {
		t.Fatal("Did not delete the document")
	}

	// Two collection scan
	counter := 0
	var template interface{}
	var throwAway interface{}
	col.DeserializeAll(&template, func(id uint64) bool {
		if err = col.ReadNoLock(id, &throwAway); err != nil {
			t.Fatal(err)
		}
		counter++
		return true
	})
	if counter != 19 { // 10 long docs, 10 short docs, one less long doc
		t.Fatal("Collection scan wrong number of docs")
	}
	counter = 0
	col.ForAll(func(id uint64, doc interface{}) bool {
		if err = col.ReadNoLock(id, &throwAway); err != nil {
			t.Fatal(err)
		}
		counter++
		return true
	})
	if counter != 19 {
		t.Fatal("Collection scan wrong number of docs")
	}
	// Remove index - it should be removed from all chunks
	if err = col.Unindex([]string{"id"}); err != nil {
		t.Fatal(err)
	}
	for _, chunk := range col.Chunks {
		if _, ok := chunk.Path2HT["id"]; ok {
			t.Fatal("Chunk should not have id index")
		}
	}
	// Out of bound access
	if err = col.Read(9999999, &longDoc); err == nil {
		t.Fatal("Out of bound access did not return error")
	}
	if _, err = col.Update(9999999, longDoc); err == nil {
		t.Fatal("Out of bound access did not return error")
	}
	col.Delete(99999999) // shall not crash
	col.Close()
}

func TestUIDDocCRUD(t *testing.T) {
	tmp := "/tmp/tiedot_col_test"
	os.RemoveAll(tmp)
	defer os.RemoveAll(tmp)
	col, err := OpenCol(tmp)
	if err != nil {
		t.Fatalf("Failed to open: %v", err)
		return
	}
	docs := []string{
		`{"a": {"b": {"c": 1}}, "d": 1, "more": "` + strings.Repeat(" ", int(chunkfile.DOC_MAX_ROOM/3)) + `"}`,
		`{"a": {"b": {"c": 2}}, "d": 2, "more": "` + strings.Repeat(" ", int(chunkfile.DOC_MAX_ROOM/3)) + `"}`,
		`{"a": {"b": {"c": 3}}, "d": 3, "more": "` + strings.Repeat(" ", int(chunkfile.DOC_MAX_ROOM/3)) + `"}`}
	var jsonDocs [3]interface{}
	var ids [3]uint64
	var uids [3]string
	json.Unmarshal([]byte(docs[0]), &jsonDocs[0])
	json.Unmarshal([]byte(docs[1]), &jsonDocs[1])
	json.Unmarshal([]byte(docs[2]), &jsonDocs[2])
	// insert
	ids[0], uids[0], err = col.InsertWithUID(jsonDocs[0])
	if err != nil {
		t.Fatal("insert error")
	}
	ids[1], uids[1], err = col.InsertWithUID(jsonDocs[1])
	if err != nil {
		t.Fatal("insert error")
	}
	ids[2], uids[2], err = col.InsertWithUID(jsonDocs[2])
	if err != nil {
		t.Fatal("insert error")
	}
	if len(uids[0]) != 32 || len(uids[1]) != 32 || len(uids[2]) != 32 ||
		uids[0] == uids[1] || uids[1] == uids[2] || uids[2] == uids[0] ||
		ids[0] == ids[1] || ids[1] == ids[2] || ids[2] == ids[0] {
		t.Fatalf("Malformed UIDs or IDs: %v %v", uids, ids)
	}
	// read - inexisting UID
	var readDoc interface{}
	if _, readErr := col.ReadByUID("abcde", &readDoc); readErr == nil {
		t.Fatal("It should have triggered UID not found error")
	}
	// read - existing UID
	readID, readErr := col.ReadByUID(uids[1], &readDoc)
	if readErr != nil {
		t.Fatal(readErr)
	}
	docMap1 := readDoc.(map[string]interface{})
	docMap2 := jsonDocs[1].(map[string]interface{})
	if readID != ids[1] || fmt.Sprint(docMap1["a"]) != fmt.Sprint(docMap2["a"]) {
		t.Fatalf("Cannot read back original document by UID: %d %d", readID, ids[1])
	}
	// update
	var docWithoutUID interface{}
	json.Unmarshal([]byte(docs[1]), &docWithoutUID)
	if _, err := col.UpdateByUID(uids[0], docWithoutUID); err != nil { // intentionally remove UID
		t.Fatal(err)
	}
	if _, err = col.ReadByUID(uids[0], &readDoc); err == nil { // UID was removed therefore the UID is not found
		t.Fatalf("UpdateByUID did not work, still read %v", readDoc)
	}
	// update (reassign UID)
	_, newUID, err := col.ReassignUID(ids[0])
	if len(newUID) != 32 || err != nil {
		t.Fatalf("ReassignUID did not work: %v %v %v", ids[0], newUID, err)
	}
	if _, err = col.ReadByUID(uids[1], &readDoc); err != nil {
		t.Fatalf("col failed UID index? %s %v", uids[1], err)
	}
	// after UID reassignment, the old UID should be gone
	if _, readErr := col.ReadByUID(uids[0], &readDoc); readErr == nil {
		t.Fatal("It should have triggered UID not found error")
	}
	if _, err = col.ReadByUID(newUID, &readDoc); err != nil { // UID was reassigned, the error should NOT happen
		t.Fatalf("ReassignUID did not work")
	}
	// delete
	col.DeleteByUID(newUID)
	if _, err = col.ReadByUID(newUID, &readDoc); err == nil {
		t.Fatalf("DeleteByUID did not work")
	}
	if _, err = col.ReadByUID(uids[1], &readDoc); err != nil {
		t.Fatalf("col failed UID index? %s %v", uids[1], err)
	}
	col.Close()
	// Reopen and test read again
	reopen, err := OpenCol(tmp)
	if err != nil {
		t.Fatal(err)
	}
	// UID index should work
	if _, err = reopen.ReadByUID(uids[1], &readDoc); err != nil {
		t.Fatalf("Reopen failed UID index? %s %v", uids[1], err)
	}
	reopen.Close()
}
<|endoftext|>