<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/APTrust/exchange/context"
	dpn_models "github.com/APTrust/exchange/dpn/models"
	"github.com/APTrust/exchange/dpn/network"
	"github.com/APTrust/exchange/models"
	"net/url"
	"os"
	"time"
)

var InstitutionIdMap map[string]int

// dpn_sync syncs data in our local DPN registry by pulling data about
// bags, replication requests, etc. from other nodes. See printUsage().
func main() {
	pathToConfigFile := parseCommandLine()
	config, err := models.LoadConfigFile(pathToConfigFile)
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	_context := context.NewContext(config)
	err = initInstitutionIdMap(_context)
	if err != nil {
		// Use "%s" so percent signs in the error message are not
		// interprested as formatting directives.
		_context.MessageLog.Error("%s", err.Error())
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	}
	err = syncToPharos(_context)
	if err != nil {
		_context.MessageLog.Error("%s", err.Error())
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	}
}

func initInstitutionIdMap(ctx *context.Context) error {
	ctx.MessageLog.Info("Caching institutions")
	InstitutionIdMap = make(map[string]int)
	params := url.Values{}
	params.Add("page", "1")
	params.Add("per_page", "100")
	resp := ctx.PharosClient.InstitutionList(params)
	ctx.MessageLog.Info(resp.Request.URL.Opaque)
	if resp.Error != nil {
		return resp.Error
	}
	for _, inst := range resp.Institutions() {
		if inst.DPNUUID != "" {
			InstitutionIdMap[inst.DPNUUID] = inst.Id
			ctx.MessageLog.Info("(%d) %s: %s", inst.Id, inst.Name, inst.DPNUUID)
		}
	}
	return nil
}

// getLatestTimestamp returns the latest UpdatedAt timestamp
// from the DPN bags table in Pharos.
func getLatestTimestamp(ctx *context.Context) (time.Time, error) {
	ctx.MessageLog.Info("Getting latest timestamp from Pharos")
	params := url.Values{}
	params.Add("sort", "dpn_updated_at DESC")
	resp := ctx.PharosClient.DPNBagList(params)
	ctx.MessageLog.Info(resp.Request.URL.Opaque)
	if resp.Error != nil {
		return time.Time{}, resp.Error
	}
	if resp.DPNBag() == nil {
		return time.Time{}, nil
	}
	return resp.DPNBag().DPNUpdatedAt, nil
}

func syncToPharos(ctx *context.Context) error {
	timestamp, err := getLatestTimestamp(ctx)
	if err != nil {
		return err
	}
	ctx.MessageLog.Info("Most recent DPN bag has update timestamp of %s", timestamp.Format(time.RFC3339))

	dpnClient, err := network.NewDPNRestClient(
		ctx.Config.DPN.RestClient.LocalServiceURL,
		ctx.Config.DPN.DPNAPIVersion,
		ctx.Config.DPN.RestClient.LocalAuthToken,
		ctx.Config.DPN.LocalNode,
		ctx.Config.DPN)
	if err != nil {
		return fmt.Errorf("Error creating local DPN REST client: %v", err)
	}
	ctx.MessageLog.Info("Set up DPN client for %s", ctx.Config.DPN.RestClient.LocalServiceURL)

	params := url.Values{}
	params.Add("after", timestamp.Format(time.RFC3339))
	params.Add("ingest_node", ctx.Config.DPN.LocalNode) // only bags we ingested
	params.Add("page", "1")
	params.Add("page_size", "100")

	ctx.MessageLog.Info("Checking for bags updated since %s", timestamp.Format(time.RFC3339))
	for {
		resp := dpnClient.DPNBagList(params)
		ctx.MessageLog.Info("%s", resp.Request.URL.String())

		if resp.Response != nil {
			ctx.MessageLog.Info("Server responded: %s", resp.Response.Status)
		} else {
			ctx.MessageLog.Warning("Server did not return a response")
		}

		// VERBOSE LOGGING
		// ctx.MessageLog.Info("%v", resp.Request)
		// ctx.MessageLog.Info("%v", resp.Response)
		// data, _ := resp.RawResponseData()
		// ctx.MessageLog.Info(string(data))
		// END VERBOSE LOGGING

		if resp.Error != nil {
			return resp.Error
		}
		ctx.MessageLog.Info("Request returned %d bags", len(resp.Bags()))
		for i, dpnBag := range resp.Bags() {
			if dpnBag == nil {
				ctx.MessageLog.Info("Item %d in bag list is nil", i)
				continue
			}
			// Quit early if this happens. It shouldn't.
			if InstitutionIdMap[dpnBag.Member] == 0 {
				return fmt.Errorf("Pharos has no institution record for DPN member %s",
					dpnBag.Member)
			}
			existingBag := getExistingPharosDPNBag(ctx, dpnBag.UUID)
			existingBagId := 0
			if existingBag != nil {
				existingBagId = existingBag.Id
			}
			updatedBag := convertToPharos(dpnBag, existingBagId)
			saveResponse := ctx.PharosClient.DPNBagSave(updatedBag)
			if saveResponse.Error != nil {
				ctx.MessageLog.Error("Error saving DPN Bag %s to Pharos: %v",
					dpnBag.UUID, saveResponse.Error)
			} else {
				ctx.MessageLog.Info("Saved DPN Bag %s with id %d",
					dpnBag.UUID, saveResponse.DPNBag().Id)
			}
		}
		if !resp.HasNextPage() {
			break
		} else {
			params = resp.ParamsForNextPage()
		}
	}

	return nil
}

func convertToPharos(dpnBag *dpn_models.DPNBag, existingBagId int) *models.PharosDPNBag {
	pharosDPNBag := &models.PharosDPNBag{
		Id: existingBagId,
	}
	pharosDPNBag.InstitutionId = InstitutionIdMap[dpnBag.Member]
	pharosDPNBag.ObjectIdentifier = dpnBag.LocalId
	pharosDPNBag.DPNIdentifier = dpnBag.UUID
	pharosDPNBag.DPNSize = dpnBag.Size
	if len(dpnBag.ReplicatingNodes) > 2 {
		pharosDPNBag.Node3 = dpnBag.ReplicatingNodes[2]
	}
	if len(dpnBag.ReplicatingNodes) > 1 {
		pharosDPNBag.Node2 = dpnBag.ReplicatingNodes[1]
	}
	if len(dpnBag.ReplicatingNodes) > 0 {
		pharosDPNBag.Node1 = dpnBag.ReplicatingNodes[0]
	}
	pharosDPNBag.DPNCreatedAt = dpnBag.CreatedAt
	pharosDPNBag.DPNUpdatedAt = dpnBag.UpdatedAt
	return pharosDPNBag
}

func getExistingPharosDPNBag(ctx *context.Context, dpnUUID string) *models.PharosDPNBag {
	params := url.Values{}
	params.Add("dpn_identifier", dpnUUID)
	params.Add("page", "1")
	params.Add("page_size", "10")
	resp := ctx.PharosClient.DPNBagList(params)
	ctx.MessageLog.Info(resp.Request.URL.Opaque)
	if resp.Error != nil {
		// Quit here, so we don't corrupt the DPNBags table.
		// We don't want to insert a bag with a DPN UUID that's already
		// in the table.
		ctx.MessageLog.Error(resp.Error.Error())
		fmt.Fprintf(os.Stderr, resp.Error.Error())
		os.Exit(1)
	}
	count := len(resp.DPNBags())
	if count > 1 {
		// Again: quit. Duplicate records need to be fixed
		// before we proceed.
		ctx.MessageLog.Error("Fatal: Found %d records for DPN Bag %s", count, dpnUUID)
		os.Exit(1)
	}
	if count == 0 {
		return nil
	}
	return resp.DPNBag()
}

// See if you can figure out from the function name what this does.
func parseCommandLine() (configFile string) {
	var pathToConfigFile string
	flag.StringVar(&pathToConfigFile, "config", "", "Path to APTrust config file")
	flag.Parse()
	if pathToConfigFile == "" {
		printUsage()
		os.Exit(1)
	}
	return pathToConfigFile
}

// Tell the user about the program.
func printUsage() {
	message := `
dpn_pharos_sync syncs data from our local DPN registry to Pharos.

Usage: dpn_sync -config=<absolute path to APTrust config file>

Param -config is required.
`
	fmt.Println(message)
}
<commit_msg>Set page param manually for next request because params returned by DPN server case 500 error<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/APTrust/exchange/context"
	dpn_models "github.com/APTrust/exchange/dpn/models"
	"github.com/APTrust/exchange/dpn/network"
	"github.com/APTrust/exchange/models"
	"net/url"
	"os"
	"time"
)

var InstitutionIdMap map[string]int

// dpn_sync syncs data in our local DPN registry by pulling data about
// bags, replication requests, etc. from other nodes. See printUsage().
func main() {
	pathToConfigFile := parseCommandLine()
	config, err := models.LoadConfigFile(pathToConfigFile)
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	_context := context.NewContext(config)
	err = initInstitutionIdMap(_context)
	if err != nil {
		// Use "%s" so percent signs in the error message are not
		// interprested as formatting directives.
		_context.MessageLog.Error("%s", err.Error())
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	}
	err = syncToPharos(_context)
	if err != nil {
		_context.MessageLog.Error("%s", err.Error())
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	}
}

func initInstitutionIdMap(ctx *context.Context) error {
	ctx.MessageLog.Info("Caching institutions")
	InstitutionIdMap = make(map[string]int)
	params := url.Values{}
	params.Add("page", "1")
	params.Add("per_page", "100")
	resp := ctx.PharosClient.InstitutionList(params)
	ctx.MessageLog.Info(resp.Request.URL.Opaque)
	if resp.Error != nil {
		return resp.Error
	}
	for _, inst := range resp.Institutions() {
		if inst.DPNUUID != "" {
			InstitutionIdMap[inst.DPNUUID] = inst.Id
			ctx.MessageLog.Info("(%d) %s: %s", inst.Id, inst.Name, inst.DPNUUID)
		}
	}
	return nil
}

// getLatestTimestamp returns the latest UpdatedAt timestamp
// from the DPN bags table in Pharos.
func getLatestTimestamp(ctx *context.Context) (time.Time, error) {
	ctx.MessageLog.Info("Getting latest timestamp from Pharos")
	params := url.Values{}
	params.Add("sort", "dpn_updated_at DESC")
	resp := ctx.PharosClient.DPNBagList(params)
	ctx.MessageLog.Info(resp.Request.URL.Opaque)
	if resp.Error != nil {
		return time.Time{}, resp.Error
	}
	if resp.DPNBag() == nil {
		return time.Time{}, nil
	}
	return resp.DPNBag().DPNUpdatedAt, nil
}

func syncToPharos(ctx *context.Context) error {
	timestamp, err := getLatestTimestamp(ctx)
	if err != nil {
		return err
	}
	ctx.MessageLog.Info("Most recent DPN bag has update timestamp of %s", timestamp.Format(time.RFC3339))

	dpnClient, err := network.NewDPNRestClient(
		ctx.Config.DPN.RestClient.LocalServiceURL,
		ctx.Config.DPN.DPNAPIVersion,
		ctx.Config.DPN.RestClient.LocalAuthToken,
		ctx.Config.DPN.LocalNode,
		ctx.Config.DPN)
	if err != nil {
		return fmt.Errorf("Error creating local DPN REST client: %v", err)
	}
	ctx.MessageLog.Info("Set up DPN client for %s", ctx.Config.DPN.RestClient.LocalServiceURL)

	params := url.Values{}
	params.Add("after", timestamp.Format(time.RFC3339))
	params.Add("ingest_node", ctx.Config.DPN.LocalNode) // only bags we ingested
	params.Add("page", "1")
	params.Add("page_size", "100")

	ctx.MessageLog.Info("Checking for bags updated since %s", timestamp.Format(time.RFC3339))
	page := 1
	for {
		resp := dpnClient.DPNBagList(params)
		ctx.MessageLog.Info("%s", resp.Request.URL.String())

		if resp.Response != nil {
			ctx.MessageLog.Info("Server responded: %s", resp.Response.Status)
		} else {
			ctx.MessageLog.Warning("Server did not return a response")
		}

		// VERBOSE LOGGING
		// ctx.MessageLog.Info("%v", resp.Request)
		// ctx.MessageLog.Info("%v", resp.Response)
		// data, _ := resp.RawResponseData()
		// ctx.MessageLog.Info(string(data))
		// END VERBOSE LOGGING

		if resp.Error != nil {
			return resp.Error
		}
		ctx.MessageLog.Info("Request returned %d bags", len(resp.Bags()))
		for i, dpnBag := range resp.Bags() {
			if dpnBag == nil {
				ctx.MessageLog.Info("Item %d in bag list is nil", i)
				continue
			}
			// Quit early if this happens. It shouldn't.
			if InstitutionIdMap[dpnBag.Member] == 0 {
				return fmt.Errorf("Pharos has no institution record for DPN member %s",
					dpnBag.Member)
			}
			existingBag := getExistingPharosDPNBag(ctx, dpnBag.UUID)
			existingBagId := 0
			if existingBag != nil {
				existingBagId = existingBag.Id
			}
			updatedBag := convertToPharos(dpnBag, existingBagId)
			saveResponse := ctx.PharosClient.DPNBagSave(updatedBag)
			if saveResponse.Error != nil {
				ctx.MessageLog.Error("Error saving DPN Bag %s to Pharos: %v",
					dpnBag.UUID, saveResponse.Error)
			} else {
				ctx.MessageLog.Info("Saved DPN Bag %s with id %d",
					dpnBag.UUID, saveResponse.DPNBag().Id)
			}
		}
		if !resp.HasNextPage() {
			break
		} else {
			page += 1
			params.Set("page", fmt.Sprintf("%d", page))
		}
	}

	return nil
}

func convertToPharos(dpnBag *dpn_models.DPNBag, existingBagId int) *models.PharosDPNBag {
	pharosDPNBag := &models.PharosDPNBag{
		Id: existingBagId,
	}
	pharosDPNBag.InstitutionId = InstitutionIdMap[dpnBag.Member]
	pharosDPNBag.ObjectIdentifier = dpnBag.LocalId
	pharosDPNBag.DPNIdentifier = dpnBag.UUID
	pharosDPNBag.DPNSize = dpnBag.Size
	if len(dpnBag.ReplicatingNodes) > 2 {
		pharosDPNBag.Node3 = dpnBag.ReplicatingNodes[2]
	}
	if len(dpnBag.ReplicatingNodes) > 1 {
		pharosDPNBag.Node2 = dpnBag.ReplicatingNodes[1]
	}
	if len(dpnBag.ReplicatingNodes) > 0 {
		pharosDPNBag.Node1 = dpnBag.ReplicatingNodes[0]
	}
	pharosDPNBag.DPNCreatedAt = dpnBag.CreatedAt
	pharosDPNBag.DPNUpdatedAt = dpnBag.UpdatedAt
	return pharosDPNBag
}

func getExistingPharosDPNBag(ctx *context.Context, dpnUUID string) *models.PharosDPNBag {
	params := url.Values{}
	params.Add("dpn_identifier", dpnUUID)
	params.Add("page", "1")
	params.Add("page_size", "10")
	resp := ctx.PharosClient.DPNBagList(params)
	ctx.MessageLog.Info(resp.Request.URL.Opaque)
	if resp.Error != nil {
		// Quit here, so we don't corrupt the DPNBags table.
		// We don't want to insert a bag with a DPN UUID that's already
		// in the table.
		ctx.MessageLog.Error(resp.Error.Error())
		fmt.Fprintf(os.Stderr, resp.Error.Error())
		os.Exit(1)
	}
	count := len(resp.DPNBags())
	if count > 1 {
		// Again: quit. Duplicate records need to be fixed
		// before we proceed.
		ctx.MessageLog.Error("Fatal: Found %d records for DPN Bag %s", count, dpnUUID)
		os.Exit(1)
	}
	if count == 0 {
		return nil
	}
	return resp.DPNBag()
}

// See if you can figure out from the function name what this does.
func parseCommandLine() (configFile string) {
	var pathToConfigFile string
	flag.StringVar(&pathToConfigFile, "config", "", "Path to APTrust config file")
	flag.Parse()
	if pathToConfigFile == "" {
		printUsage()
		os.Exit(1)
	}
	return pathToConfigFile
}

// Tell the user about the program.
func printUsage() {
	message := `
dpn_pharos_sync syncs data from our local DPN registry to Pharos.

Usage: dpn_sync -config=<absolute path to APTrust config file>

Param -config is required.
`
	fmt.Println(message)
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A small helper for using gcsfuse with mount(8).
//
// Can be invoked using a command-line of the form expected for mount helpers.
// Calls the gcsfuse binary, which must be in $PATH, and waits for it to
// complete. The device is passed as --bucket, and other known options are
// converted to appropriate flags.
//
// This binary does not daemonize, and therefore must be used with a wrapper
// that performs daemonization if it is to be used directly with mount(8).
package main

// Example invocation on OS X:
//
//     mount -t porp -o key_file=/some\ file.json -o ro,blah bucket ~/tmp/mp
//
// becomes the following arguments:
//
//     Arg 0: "/path/to/gcsfuse_mount_helper"
//     Arg 1: "-o"
//     Arg 2: "key_file=/some file.json"
//     Arg 3: "-o"
//     Arg 4: "ro"
//     Arg 5: "-o"
//     Arg 6: "blah"
//     Arg 7: "bucket"
//     Arg 8: "/path/to/mp"
//
// On Linux, the fstab entry
//
//     bucket /path/to/mp porp user,key_file=/some\040file.json
//
// becomes
//
//     Arg 0: "/path/to/gcsfuse_mount_helper"
//     Arg 1: "bucket"
//     Arg 2: "/path/to/mp"
//     Arg 3: "-o"
//     Arg 4: "rw,noexec,nosuid,nodev,user,key_file=/some file.json"
//

import (
	"fmt"
	"log"
	"os"
	"os/exec"

	"github.com/googlecloudplatform/gcsfuse/mount"
)

// Turn mount-style options into gcsfuse arguments. Skip known detritus that
// the mount command gives us.
//
// The result of this function should be appended to exec.Command.Args.
func makeGcsfuseArgs(
	device string,
	mountPoint string,
	opts map[string]string) (args []string, err error) {
	// Deal with options.
	for name, value := range opts {
		switch name {
		case "key_file":
			args = append(args, "--key_file="+value)

		case "fuse_debug":
			args = append(args, "--fuse.debug")

		case "gcs_debug":
			args = append(args, "--gcs.debug")

		case "uid":
			args = append(args, "--uid="+value)

		case "gid":
			args = append(args, "--gid="+value)

		case "file_mode":
			args = append(args, "--file_mode="+value)

		case "dir_mode":
			args = append(args, "--dir_mode="+value)

		// Pass through everything else.
		default:
			var formatted string
			if value == "" {
				formatted = name
			} else {
				formatted = fmt.Sprintf("%s=%s", name, value)
			}

			args = append(args, "-o", formatted)
		}
	}

	// Set the bucket.
	args = append(args, "--bucket="+device)

	// Set the mount point.
	args = append(args, "--mount_point="+mountPoint)

	return
}

// Parse the supplied command-line arguments from a mount(8) invocation on OS X
// or Linux.
func parseArgs(
	args []string) (
	device string,
	mountPoint string,
	opts map[string]string,
	err error) {
	opts = make(map[string]string)

	// Process each argument in turn.
	positionalCount := 0
	for i, s := range args {
		switch {
		// Skip the program name.
		case i == 0:
			continue

		// "-o" is illegal only when at the end. We handle its argument in the case
		// below.
		case s == "-o" && i == len(args)-1:
			err = fmt.Errorf("Unexpected -o at end of args.")
			return

		// Is this an options string following a "-o"?
		case i > 0 && args[i-1] == "-o":
			err = mount.ParseOptions(opts, s)
			if err != nil {
				err = fmt.Errorf("ParseOptions(%q): %v", s, err)
				return
			}

		// Is this the device?
		case positionalCount == 0:
			device = s
			positionalCount++

		// Is this the mount point?
		case positionalCount == 1:
			mountPoint = s
			positionalCount++

		default:
			err = fmt.Errorf("Unexpected arg %d: %q", i, s)
			return
		}
	}

	return
}

func main() {
	// Print out each argument.
	args := os.Args
	for i, arg := range args {
		log.Printf("Arg %d: %q", i, arg)
	}

	// Attempt to parse arguments.
	device, mountPoint, opts, err := parseArgs(args)
	if err != nil {
		log.Fatalf("parseArgs: %v", err)
	}

	// Print what we gleaned.
	log.Printf("Device: %q", device)
	log.Printf("Mount point: %q", mountPoint)
	for name, value := range opts {
		log.Printf("Option %q: %q", name, value)
	}

	// Choose gcsfuse args.
	gcsfuseArgs, err := makeGcsfuseArgs(device, mountPoint, opts)
	if err != nil {
		log.Fatalf("makeGcsfuseArgs: %v", err)
	}

	for _, a := range gcsfuseArgs {
		log.Printf("gcsfuse arg: %q", a)
	}

	// Run gcsfuse and wait for it to complete.
	cmd := exec.Command("gcsfuse", gcsfuseArgs...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	err = cmd.Run()
	if err != nil {
		log.Fatalf("gcsfuse failed or failed to run: %v", err)
	}

	log.Println("gcsfuse completed successfully.")
}
<commit_msg>Fixed a parsing bug.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A small helper for using gcsfuse with mount(8).
//
// Can be invoked using a command-line of the form expected for mount helpers.
// Calls the gcsfuse binary, which must be in $PATH, and waits for it to
// complete. The device is passed as --bucket, and other known options are
// converted to appropriate flags.
//
// This binary does not daemonize, and therefore must be used with a wrapper
// that performs daemonization if it is to be used directly with mount(8).
package main

// Example invocation on OS X:
//
//     mount -t porp -o key_file=/some\ file.json -o ro,blah bucket ~/tmp/mp
//
// becomes the following arguments:
//
//     Arg 0: "/path/to/gcsfuse_mount_helper"
//     Arg 1: "-o"
//     Arg 2: "key_file=/some file.json"
//     Arg 3: "-o"
//     Arg 4: "ro"
//     Arg 5: "-o"
//     Arg 6: "blah"
//     Arg 7: "bucket"
//     Arg 8: "/path/to/mp"
//
// On Linux, the fstab entry
//
//     bucket /path/to/mp porp user,key_file=/some\040file.json
//
// becomes
//
//     Arg 0: "/path/to/gcsfuse_mount_helper"
//     Arg 1: "bucket"
//     Arg 2: "/path/to/mp"
//     Arg 3: "-o"
//     Arg 4: "rw,noexec,nosuid,nodev,user,key_file=/some file.json"
//

import (
	"fmt"
	"log"
	"os"
	"os/exec"

	"github.com/googlecloudplatform/gcsfuse/mount"
)

// Turn mount-style options into gcsfuse arguments. Skip known detritus that
// the mount command gives us.
//
// The result of this function should be appended to exec.Command.Args.
func makeGcsfuseArgs(
	device string,
	mountPoint string,
	opts map[string]string) (args []string, err error) {
	// Deal with options.
	for name, value := range opts {
		switch name {
		case "key_file":
			args = append(args, "--key_file="+value)

		case "fuse_debug":
			args = append(args, "--fuse.debug")

		case "gcs_debug":
			args = append(args, "--gcs.debug")

		case "uid":
			args = append(args, "--uid="+value)

		case "gid":
			args = append(args, "--gid="+value)

		case "file_mode":
			args = append(args, "--file_mode="+value)

		case "dir_mode":
			args = append(args, "--dir_mode="+value)

		// Pass through everything else.
		default:
			var formatted string
			if value == "" {
				formatted = name
			} else {
				formatted = fmt.Sprintf("%s=%s", name, value)
			}

			args = append(args, "-o", formatted)
		}
	}

	// Set the bucket.
	args = append(args, "--bucket="+device)

	// Set the mount point.
	args = append(args, "--mount_point="+mountPoint)

	return
}

// Parse the supplied command-line arguments from a mount(8) invocation on OS X
// or Linux.
func parseArgs(
	args []string) (
	device string,
	mountPoint string,
	opts map[string]string,
	err error) {
	opts = make(map[string]string)

	// Process each argument in turn.
	positionalCount := 0
	for i, s := range args {
		switch {
		// Skip the program name.
		case i == 0:
			continue

		// "-o" is illegal only when at the end. We handle its argument in the case
		// below.
		case s == "-o":
			if i == len(args)-1 {
				err = fmt.Errorf("Unexpected -o at end of args.")
				return
			}

		// Is this an options string following a "-o"?
		case i > 0 && args[i-1] == "-o":
			err = mount.ParseOptions(opts, s)
			if err != nil {
				err = fmt.Errorf("ParseOptions(%q): %v", s, err)
				return
			}

		// Is this the device?
		case positionalCount == 0:
			device = s
			positionalCount++

		// Is this the mount point?
		case positionalCount == 1:
			mountPoint = s
			positionalCount++

		default:
			err = fmt.Errorf("Unexpected arg %d: %q", i, s)
			return
		}
	}

	return
}

func main() {
	// Print out each argument.
	args := os.Args
	for i, arg := range args {
		log.Printf("Arg %d: %q", i, arg)
	}

	// Attempt to parse arguments.
	device, mountPoint, opts, err := parseArgs(args)
	if err != nil {
		log.Fatalf("parseArgs: %v", err)
	}

	// Print what we gleaned.
	log.Printf("Device: %q", device)
	log.Printf("Mount point: %q", mountPoint)
	for name, value := range opts {
		log.Printf("Option %q: %q", name, value)
	}

	// Choose gcsfuse args.
	gcsfuseArgs, err := makeGcsfuseArgs(device, mountPoint, opts)
	if err != nil {
		log.Fatalf("makeGcsfuseArgs: %v", err)
	}

	for _, a := range gcsfuseArgs {
		log.Printf("gcsfuse arg: %q", a)
	}

	// Run gcsfuse and wait for it to complete.
	cmd := exec.Command("gcsfuse", gcsfuseArgs...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	err = cmd.Run()
	if err != nil {
		log.Fatalf("gcsfuse failed or failed to run: %v", err)
	}

	log.Println("gcsfuse completed successfully.")
}
<|endoftext|>


<commit_before>package sls

import (
	"fmt"
	"github.com/denverdino/aliyungo/common"
	"net/http"
	"time"
	"encoding/json"
)

type Client struct {
	accessKeyId     string //Access Key Id
	accessKeySecret string //Access Key Secret
	debug           bool
	httpClient      *http.Client
	version         string
	internal        bool
	region          common.Region
	endpoint        string
}

type Project struct {
	client      *Client
	Name        string `json:"projectName,omitempty"`
	Description string `json:"description,omitempty"`
}

type LogItem struct {
	Time    time.Time
	Content map[string]string
}

type LogGroupItem struct {
	Logs   []*LogItem
	Topic  string
	Source string
}

const (
	SLSDefaultEndpoint = "sls.aliyuncs.com"
	SLSAPIVersion = "0.6.0"
	METHOD_GET = "GET"
	METHOD_POST = "POST"
	METHOD_PUT = "PUT"
	METHOD_DELETE = "DELETE"
)

// NewClient creates a new instance of ECS client
func NewClient(region common.Region, internal bool, accessKeyId, accessKeySecret string) *Client {
	return &Client{
		accessKeyId:     accessKeyId,
		accessKeySecret: accessKeySecret,
		internal:        internal,
		region:          region,
		version:         SLSAPIVersion,
		endpoint:        SLSDefaultEndpoint,
		httpClient:      &http.Client{},
	}
}

func (client *Client) Project(name string) (*Project, error) {

	newClient := client.forProject(name)

	req := &request{
		method: METHOD_GET,
		path: "/",
	}

	project := &Project{}

	if err := newClient.requestWithJsonResponse(req, project); err != nil {
		return nil, err
	}
	project.client = newClient
	return project, nil
}

func (client *Client) forProject(name string) *Client {
	newclient := *client

	region := string(client.region)
	if client.internal {
		region = fmt.Sprintf("%s-intranet", region)
	}
	newclient.endpoint = fmt.Sprintf("%s.%s.%s", name, region, SLSDefaultEndpoint)
	return &newclient
}

func (client *Client) DeleteProject(name string) error {
	req := &request{
		method: METHOD_DELETE,
		path: "/",
	}

	newClient := client.forProject(name)
	return newClient.requestWithClose(req)
}

func (client *Client) CreateProject(name string, description string) error {
	project := &Project{
		Name: name,
		Description: description,
	}
	data, err := json.Marshal(project)
	if err != nil {
		return err
	}

	req := &request{
		method: METHOD_POST,
		path: "/",
		payload:     data,
		contentType: "application/json",
	}

	newClient := client.forProject(name)
	return newClient.requestWithClose(req)
}

//
//func marshal() ([]byte, error) {
//
//	logGroups := []*LogGroup{}
//	tmp := []*LogGroupItem
//	for _, logGroupItem := range tmp {
//
//		logs := []*Log{}
//		for _, logItem := range logGroupItem.Logs {
//			contents := []*Log_Content{}
//			for key, value := range logItem.Content {
//				contents = append(contents, &Log_Content{
//					Key: proto.String(key),
//					Value: proto.String(value),
//				})
//			}
//
//			logs = append(logs, &Log{
//				Time: proto.Uint32(uint32(LogItem.Time.Unix())),
//				Contents: contents,
//			})
//		}
//
//		logGroup := &LogGroup{
//			Topic: proto.String(LogGroupItem.Topic),
//			Source: proto.String(LogGroupItem.Source),
//			Logs: logs,
//		}
//		logGroups = append(logGroups, logGroup)
//	}
//
//	return proto.Marshal(&LogGroupList{
//		LogGroupList: logGroups,
//	})
//}
<commit_msg>workaround sls bug：通过API获取Project容易出现502，改成本地直接构造<commit_after>package sls

import (
	"fmt"
	"github.com/denverdino/aliyungo/common"
	"net/http"
	"time"
	"encoding/json"
)

type Client struct {
	accessKeyId     string //Access Key Id
	accessKeySecret string //Access Key Secret
	debug           bool
	httpClient      *http.Client
	version         string
	internal        bool
	region          common.Region
	endpoint        string
}

type Project struct {
	client      *Client
	Name        string `json:"projectName,omitempty"`
	Description string `json:"description,omitempty"`
}

type LogItem struct {
	Time    time.Time
	Content map[string]string
}

type LogGroupItem struct {
	Logs   []*LogItem
	Topic  string
	Source string
}

const (
	SLSDefaultEndpoint = "sls.aliyuncs.com"
	SLSAPIVersion = "0.6.0"
	METHOD_GET = "GET"
	METHOD_POST = "POST"
	METHOD_PUT = "PUT"
	METHOD_DELETE = "DELETE"
)

// NewClient creates a new instance of ECS client
func NewClient(region common.Region, internal bool, accessKeyId, accessKeySecret string) *Client {
	return &Client{
		accessKeyId:     accessKeyId,
		accessKeySecret: accessKeySecret,
		internal:        internal,
		region:          region,
		version:         SLSAPIVersion,
		endpoint:        SLSDefaultEndpoint,
		httpClient:      &http.Client{},
	}
}

func (client *Client) Project(name string) (*Project, error) {

//	newClient := client.forProject(name)
//
//	req := &request{
//		method: METHOD_GET,
//		path: "/",
//	}
//
//	project := &Project{}
//
//	if err := newClient.requestWithJsonResponse(req, project); err != nil {
//		return nil, err
//	}
//	project.client = newClient
//	return project, nil
	return &Project {
		Name: name,
		client: client.forProject(name),
	}, nil
}

func (client *Client) forProject(name string) *Client {
	newclient := *client

	region := string(client.region)
	if client.internal {
		region = fmt.Sprintf("%s-intranet", region)
	}
	newclient.endpoint = fmt.Sprintf("%s.%s.%s", name, region, SLSDefaultEndpoint)
	return &newclient
}

func (client *Client) DeleteProject(name string) error {
	req := &request{
		method: METHOD_DELETE,
		path: "/",
	}

	newClient := client.forProject(name)
	return newClient.requestWithClose(req)
}

func (client *Client) CreateProject(name string, description string) error {
	project := &Project{
		Name: name,
		Description: description,
	}
	data, err := json.Marshal(project)
	if err != nil {
		return err
	}

	req := &request{
		method: METHOD_POST,
		path: "/",
		payload:     data,
		contentType: "application/json",
	}

	newClient := client.forProject(name)
	return newClient.requestWithClose(req)
}

//
//func marshal() ([]byte, error) {
//
//	logGroups := []*LogGroup{}
//	tmp := []*LogGroupItem
//	for _, logGroupItem := range tmp {
//
//		logs := []*Log{}
//		for _, logItem := range logGroupItem.Logs {
//			contents := []*Log_Content{}
//			for key, value := range logItem.Content {
//				contents = append(contents, &Log_Content{
//					Key: proto.String(key),
//					Value: proto.String(value),
//				})
//			}
//
//			logs = append(logs, &Log{
//				Time: proto.Uint32(uint32(LogItem.Time.Unix())),
//				Contents: contents,
//			})
//		}
//
//		logGroup := &LogGroup{
//			Topic: proto.String(LogGroupItem.Topic),
//			Source: proto.String(LogGroupItem.Source),
//			Logs: logs,
//		}
//		logGroups = append(logGroups, logGroup)
//	}
//
//	return proto.Marshal(&LogGroupList{
//		LogGroupList: logGroups,
//	})
//}
<|endoftext|>


<commit_before>// Copyright 2014 Caleb Brose, Chris Fogerty, Rob Sheehy, Zach Taylor, Nick Miller
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
    "fmt"
    "flag"
    "log"
    "net/http"
    "encoding/json"
    "io/ioutil"
    "strings"

    "github.com/mgutz/ansi"

    "github.com/zenazn/goji/web"
    "github.com/zenazn/goji/graceful"
    "github.com/zenazn/goji/web/middleware"

    "github.com/lighthouse/beacon/auth"
    "github.com/lighthouse/beacon/drivers"
    "github.com/lighthouse/beacon/structs"
)

var pemFile = flag.String("pem", "", "Path to Cert file")
var keyFile = flag.String("key", "", "Path to Key file")
var address = flag.String("h", "127.0.0.1:5000", "Address to host under")

var App *web.Mux
var Driver *structs.Driver

func init() {
    App = web.New()
    App.Use(middleware.Logger)
    App.Use(auth.Middleware)

    App.Handle("/d/*", func(c web.C, w http.ResponseWriter, r *http.Request) {
        target := fmt.Sprintf("http://%s",
            strings.SplitN(r.URL.Path, "/", 3)[2])

        req, err := http.NewRequest(r.Method, target, r.Body)
        if err != nil {
            w.WriteHeader(http.StatusInternalServerError)
            fmt.Fprint(w, err)
            return
        }

        resp, err := http.DefaultClient.Do(req)

        if err != nil {
            w.WriteHeader(http.StatusInternalServerError)
            fmt.Fprint(w, err)
            return
        }

        body, err := ioutil.ReadAll(resp.Body)

        if err != nil {
            w.WriteHeader(http.StatusInternalServerError)
            fmt.Fprint(w, err)
            return
        }

        w.WriteHeader(resp.StatusCode)
        w.Write(body)
    })

    App.Get("/vms", func(c web.C, w http.ResponseWriter, r *http.Request) {
        response, _ := json.Marshal(Driver.GetVMs())
        w.Write(response)
    })

    App.Get("/which", func(c web.C, w http.ResponseWriter, r *http.Request) {
        response, _ := json.Marshal(Driver.Name)
        w.Write(response)
    })

    App.Compile()
}

func main() {
    log.Printf(ansi.Color("Starting Beacon...", "white+b"))

    if !flag.Parsed() {
        flag.Parse()
    }

    Driver = drivers.Decide()

    log.Printf("Provider Interface: %s\n", ansi.Color(Driver.Name, "cyan+b"))
    log.Printf("Authentication Token: %s\n", ansi.Color(*auth.Token, "cyan+b"))

    graceful.HandleSignals()

    graceful.PreHook(func() {
        log.Printf(ansi.Color("Gracefully Shutting Down...", "white+b"))
    })
    graceful.PostHook(func() {
        log.Printf(ansi.Color("Done!", "white+b"))
    })

    defer graceful.Wait()

    http.Handle("/", App)
    log.Printf("Listening on %s", *address)

    var err error

    if *pemFile != "" && *keyFile != "" {
        log.Printf("Setting up secure server...")
        err = graceful.ListenAndServeTLS(*address, *pemFile, *keyFile, http.DefaultServeMux)
    } else {
        log.Printf(ansi.Color("Setting up unsecure server...", "yellow+b"))
        err = graceful.ListenAndServe(*address, http.DefaultServeMux)
    }

    if err != nil {
        log.Fatal(err)
    }
}
<commit_msg>fix for new boot2docker header complaints<commit_after>// Copyright 2014 Caleb Brose, Chris Fogerty, Rob Sheehy, Zach Taylor, Nick Miller
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
    "fmt"
    "flag"
    "log"
    "net/http"
    "encoding/json"
    "io/ioutil"
    "strings"

    "github.com/mgutz/ansi"

    "github.com/zenazn/goji/web"
    "github.com/zenazn/goji/graceful"
    "github.com/zenazn/goji/web/middleware"

    "github.com/lighthouse/beacon/auth"
    "github.com/lighthouse/beacon/drivers"
    "github.com/lighthouse/beacon/structs"
)

var pemFile = flag.String("pem", "", "Path to Cert file")
var keyFile = flag.String("key", "", "Path to Key file")
var address = flag.String("h", "127.0.0.1:5000", "Address to host under")

var App *web.Mux
var Driver *structs.Driver

func init() {
    App = web.New()
    App.Use(middleware.Logger)
    App.Use(auth.Middleware)

    App.Handle("/d/*", func(c web.C, w http.ResponseWriter, r *http.Request) {
        target := fmt.Sprintf("http://%s",
            strings.SplitN(r.URL.Path, "/", 3)[2])

        req, err := http.NewRequest(r.Method, target, r.Body)
        if err != nil {
            w.WriteHeader(http.StatusInternalServerError)
            fmt.Fprint(w, err)
            return
        }

        // TESTING header addition
        req.Header.Add("Content-Type", "application/json")

        resp, err := http.DefaultClient.Do(req)

        if err != nil {
            w.WriteHeader(http.StatusInternalServerError)
            fmt.Fprint(w, err)
            return
        }

        body, err := ioutil.ReadAll(resp.Body)

        if err != nil {
            w.WriteHeader(http.StatusInternalServerError)
            fmt.Fprint(w, err)
            return
        }

        w.WriteHeader(resp.StatusCode)
        w.Write(body)
    })

    App.Get("/vms", func(c web.C, w http.ResponseWriter, r *http.Request) {
        response, _ := json.Marshal(Driver.GetVMs())
        w.Write(response)
    })

    App.Get("/which", func(c web.C, w http.ResponseWriter, r *http.Request) {
        response, _ := json.Marshal(Driver.Name)
        w.Write(response)
    })

    App.Compile()
}

func main() {
    log.Printf(ansi.Color("Starting Beacon...", "white+b"))

    if !flag.Parsed() {
        flag.Parse()
    }

    Driver = drivers.Decide()

    log.Printf("Provider Interface: %s\n", ansi.Color(Driver.Name, "cyan+b"))
    log.Printf("Authentication Token: %s\n", ansi.Color(*auth.Token, "cyan+b"))

    graceful.HandleSignals()

    graceful.PreHook(func() {
        log.Printf(ansi.Color("Gracefully Shutting Down...", "white+b"))
    })
    graceful.PostHook(func() {
        log.Printf(ansi.Color("Done!", "white+b"))
    })

    defer graceful.Wait()

    http.Handle("/", App)
    log.Printf("Listening on %s", *address)

    var err error

    if *pemFile != "" && *keyFile != "" {
        log.Printf("Setting up secure server...")
        err = graceful.ListenAndServeTLS(*address, *pemFile, *keyFile, http.DefaultServeMux)
    } else {
        log.Printf(ansi.Color("Setting up unsecure server...", "yellow+b"))
        err = graceful.ListenAndServe(*address, http.DefaultServeMux)
    }

    if err != nil {
        log.Fatal(err)
    }
}
<|endoftext|>


<commit_before>package devices

import (
	"github.com/ninjasphere/go-ninja/api"
	"github.com/ninjasphere/go-ninja/logger"
	"github.com/ninjasphere/go-ninja/model"
)

type baseDevice struct {
	log       *logger.Logger
	driver    ninja.Driver
	info      *model.Device
	conn      *ninja.Connection
	sendEvent func(event string, payload interface{}) error
}

func (d *baseDevice) GetDeviceInfo() *model.Device {
	return d.info
}

func (d *baseDevice) GetDriver() ninja.Driver {
	return d.driver
}

func (d *baseDevice) SetEventHandler(sendEvent func(event string, payload interface{}) error) {
	d.sendEvent = sendEvent
}
<commit_msg>Expose base device logger to consumers - otherwise it isn't very useful!<commit_after>package devices

import (
	"github.com/ninjasphere/go-ninja/api"
	"github.com/ninjasphere/go-ninja/logger"
	"github.com/ninjasphere/go-ninja/model"
)

type baseDevice struct {
	log       *logger.Logger
	driver    ninja.Driver
	info      *model.Device
	conn      *ninja.Connection
	sendEvent func(event string, payload interface{}) error
}

func (d *baseDevice) GetDeviceInfo() *model.Device {
	return d.info
}

func (d *baseDevice) GetDriver() ninja.Driver {
	return d.driver
}

func (d *baseDevice) SetEventHandler(sendEvent func(event string, payload interface{}) error) {
	d.sendEvent = sendEvent
}

func (d *baseDevice) Log() *logger.Logger {
	return d.log
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"errors"
	"flag"
	"fmt"
	"github.com/bwmarrin/discordgo"
	"github.com/facebookgo/flagconfig"
	"io/ioutil"
	"mime"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"os/signal"
	"path"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"syscall"
	"time"
)

const (
	ImageRegex      string = `(https?:\/\/.*\.(?:png|jpg))`
	DownloadTimeout int    = 60
)

var (
	token    = flag.String("token", "", "Bot Token")
	baseUrl  = flag.String("baseUrl", "", "Base url of local server where static is saved")
	faces    = flag.String("faces", "/home/sites/faces", "Faces to add to photo")
	cascade  = flag.String("cascade", "/home/sites/faces/cascade.xml", "Haar cascade to find faces")
	basePath = "/var/www/static"
)

func main() {
	flag.Parse()
	flagconfig.Parse()

	runtime.GOMAXPROCS(4)
	fmt.Printf("GOMAXPROCS is %d\n", runtime.GOMAXPROCS(0))

	if *token == "" {
		fmt.Println("No token provided")
	}

	dg, err := discordgo.New("Bot " + *token)

	if err != nil {
		fmt.Println(err)

		return
	}

	dg.AddHandler(ready)
	dg.AddHandler(messageCreate)

	err = dg.Open()
	if err != nil {
		fmt.Println("Error opening Discord session: ", err)
	}

	defer dg.Close()

	ctx, cancel := context.WithCancel(context.Background())

	go signalHandler(cancel)

	fmt.Println("PandaBot is now running.  Press CTRL-C to exit.")

	<-ctx.Done()
	return
}

func ready(s *discordgo.Session, event *discordgo.Ready) {
	//  Set the playing status.
	_ = s.UpdateStatus(0, "Dirty Games")
}

func messageCreate(s *discordgo.Session, m *discordgo.MessageCreate) {
	if strings.HasPrefix(m.Content, "!pandabot") {
		_, _ = s.ChannelMessageSend(m.ChannelID, "PandaBot")
		return
	}

	if strings.HasPrefix(m.Content, "!confify") {
		fmt.Println("Starting confify image")
		defer fmt.Println("Finishing confify image")

		imgCh := make(chan string)
		ticksWaiting := 1
		message, _ := s.ChannelMessageSend(m.ChannelID, "Processing"+strings.Repeat(".", ticksWaiting%4))

		regexpImage, err := regexp.Compile(ImageRegex)

		if err != nil {
			fmt.Println("Error: ", err)
			return
		}

		var imageString string

		if imageString = regexpImage.FindString(m.Content); imageString == "" {
			s.ChannelMessageEdit(m.ChannelID, message.ID, "Please, provide image link with PNG or JPEG extension")

			return
		}

		go processImage(imgCh, imageString)

		for {
			time.Sleep(200 * time.Millisecond)
			select {
			case image := <-imgCh:
				s.ChannelMessageEdit(m.ChannelID, message.ID, "Processed file: "+*baseUrl+image)

				if image == "" {
					s.ChannelMessageEdit(
						m.ChannelID,
						message.ID,
						"Error during processing, please, notify PandaSam about it")
				}

				return
			default:
				fmt.Println("Waiting for image being procesed")
				ticksWaiting += 1
				s.ChannelMessageEdit(m.ChannelID, message.ID, "Processing"+strings.Repeat(".", ticksWaiting%4))
				if ticksWaiting > 50 {
					s.ChannelMessageEdit(m.ChannelID, message.ID, "Processing time exceeed")
					return
				}
			}
		}

		return
	}
}

func processImage(imgCh chan<- string, imageString string) {
	fmt.Println("Started image processing")
	defer fmt.Println("Finished image processing")

	downloadedFilename, downloadedFilePath, err := downloadFromUrl(imageString, "", basePath)

	if err != nil {
		fmt.Println(err)
		imgCh <- ""
		return
	}

	outputFilePath := basePath + "/processed_" + downloadedFilename

	args := []string{
		"--faces", *faces,
		"--haar", *cascade,
		downloadedFilePath}

	cmd := exec.Command("chrisify", args...)

	out, err := cmd.CombinedOutput()

	if err != nil {
		fmt.Println("Non-zero exit code: " + err.Error() + ", " + string(out))
	}

	err = ioutil.WriteFile(outputFilePath, out, 0644)

	fmt.Println("Image processed, putting in channel")

	imgCh <- "processed_" + downloadedFilename
	return
}

func downloadFromUrl(dUrl string, filename string, path string) (string, string, error) {
	err := os.MkdirAll(path, 755)
	if err != nil {
		return "", "", errors.New(fmt.Sprintln("Error while creating folder", path, "-", err))
	}

	timeout := time.Duration(time.Duration(DownloadTimeout) * time.Second)
	client := &http.Client{
		Timeout: timeout,
	}
	request, err := http.NewRequest("GET", dUrl, nil)
	if err != nil {
		return "", "", errors.New(fmt.Sprintln("Error while downloading", dUrl, "-", err))
	}
	request.Header.Add("Accept-Encoding", "identity")
	response, err := client.Do(request)
	if err != nil {
		return "", "", errors.New(fmt.Sprintln("Error while downloading", dUrl, "-", err))
	}
	defer response.Body.Close()

	if filename == "" {
		filename = filenameFromUrl(response.Request.URL.String())
		for key, iHeader := range response.Header {
			if key == "Content-Disposition" {
				_, params, err := mime.ParseMediaType(iHeader[0])
				if err == nil {
					newFilename, err := url.QueryUnescape(params["filename"])
					if err != nil {
						newFilename = params["filename"]
					}
					if newFilename != "" {
						filename = newFilename
					}
				}
			}
		}
	}

	completePath := path + string(os.PathSeparator) + filename
	if _, err := os.Stat(completePath); err == nil {
		tmpPath := completePath
		i := 1
		for {
			completePath = tmpPath[0:len(tmpPath)-len(filepath.Ext(tmpPath))] +
				"-" + strconv.Itoa(i) + filepath.Ext(tmpPath)
			if _, err := os.Stat(completePath); os.IsNotExist(err) {
				break
			}
			i = i + 1
		}
		fmt.Printf("[%s] Saving possible duplicate (filenames match): %s to %s\n", time.Now().Format(time.Stamp), tmpPath, completePath)
	}

	bodyOfResp, err := ioutil.ReadAll(response.Body)

	if err != nil {
		return "", "", errors.New(fmt.Sprintln("Could not read response", dUrl, "-", err))
	}

	contentType := http.DetectContentType(bodyOfResp)
	contentTypeParts := strings.Split(contentType, "/")

	if contentTypeParts[0] != "image" && contentTypeParts[0] != "video" {
		return "", "", errors.New(fmt.Sprintln("No image or video found at", dUrl))
	}

	err = ioutil.WriteFile(completePath, bodyOfResp, 0644)

	if err != nil {
		return "", "", errors.New(fmt.Sprintln("Error while writing to disk", dUrl, "-", err))
	}

	return filename, completePath, err
}

func filenameFromUrl(dUrl string) string {
	base := path.Base(dUrl)
	parts := strings.Split(base, "?")
	return parts[0]
}

func signalHandler(cancel context.CancelFunc) {
	sigCh := make(chan os.Signal, 1)
	signal.Notify(sigCh, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGINT)
	for {
		<-sigCh
		fmt.Println("Got stop signal, safely shutting down")
		cancel()
		return
	}
}
<commit_msg>hash filename and make it case insensitive<commit_after>package main

import (
	"context"
	"crypto/md5"
	"errors"
	"flag"
	"fmt"
	"github.com/bwmarrin/discordgo"
	"github.com/facebookgo/flagconfig"
	"io/ioutil"
	"mime"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"os/signal"
	"path"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"syscall"
	"time"
)

const (
	ImageRegex      string = `(?i)(https?:\/\/.*\.(?:png|jpe?g))`
	DownloadTimeout int    = 60
)

var (
	token    = flag.String("token", "", "Bot Token")
	baseUrl  = flag.String("baseUrl", "", "Base url of local server where static is saved")
	faces    = flag.String("faces", "/home/sites/faces", "Faces to add to photo")
	cascade  = flag.String("cascade", "/home/sites/faces/cascade.xml", "Haar cascade to find faces")
	basePath = "/var/www/static"
)

func main() {
	flag.Parse()
	flagconfig.Parse()

	runtime.GOMAXPROCS(4)
	fmt.Printf("GOMAXPROCS is %d\n", runtime.GOMAXPROCS(0))

	if *token == "" {
		fmt.Println("No token provided")
	}

	dg, err := discordgo.New("Bot " + *token)

	if err != nil {
		fmt.Println(err)

		return
	}

	dg.AddHandler(ready)
	dg.AddHandler(messageCreate)

	err = dg.Open()
	if err != nil {
		fmt.Println("Error opening Discord session: ", err)
	}

	defer dg.Close()

	ctx, cancel := context.WithCancel(context.Background())

	go signalHandler(cancel)

	fmt.Println("PandaBot is now running.  Press CTRL-C to exit.")

	<-ctx.Done()
	return
}

func ready(s *discordgo.Session, event *discordgo.Ready) {
	//  Set the playing status.
	_ = s.UpdateStatus(0, "Dirty Games")
}

func messageCreate(s *discordgo.Session, m *discordgo.MessageCreate) {
	if strings.HasPrefix(m.Content, "!pandabot") {
		_, _ = s.ChannelMessageSend(m.ChannelID, "PandaBot")
		return
	}

	if strings.HasPrefix(m.Content, "!confify") {
		fmt.Println("Starting confify image")
		defer fmt.Println("Finishing confify image")

		imgCh := make(chan string)
		ticksWaiting := 1
		message, _ := s.ChannelMessageSend(m.ChannelID, "Processing"+strings.Repeat(".", ticksWaiting%4))

		regexpImage, err := regexp.Compile(ImageRegex)

		if err != nil {
			fmt.Println("Error: ", err)
			return
		}

		var imageString string

		if imageString = regexpImage.FindString(m.Content); imageString == "" {
			s.ChannelMessageEdit(m.ChannelID, message.ID, "Please, provide image link with PNG or JPEG extension")

			return
		}

		go processImage(imgCh, imageString)

		for {
			time.Sleep(200 * time.Millisecond)
			select {
			case image := <-imgCh:
				s.ChannelMessageEdit(m.ChannelID, message.ID, "Processed file: "+*baseUrl+image)

				if image == "" {
					s.ChannelMessageEdit(
						m.ChannelID,
						message.ID,
						"Error during processing, please, notify PandaSam about it")
				}

				return
			default:
				fmt.Println("Waiting for image being procesed")
				ticksWaiting += 1
				s.ChannelMessageEdit(m.ChannelID, message.ID, "Processing"+strings.Repeat(".", ticksWaiting%4))
				if ticksWaiting > 50 {
					s.ChannelMessageEdit(m.ChannelID, message.ID, "Processing time exceeed")
					return
				}
			}
		}

		return
	}
}

func processImage(imgCh chan<- string, imageString string) {
	fmt.Println("Started image processing")
	defer fmt.Println("Finished image processing")

	downloadedFilename, downloadedFilePath, err := downloadFromUrl(imageString, "", basePath)

	if err != nil {
		fmt.Println(err)
		imgCh <- ""
		return
	}
	splittedString := strings.Split(downloadedFilename, ".")
	fileExtension := splittedString[len(splittedString)-1]
	outputFileName := fmt.Sprintf("%x", md5.Sum([]byte("processed_"+downloadedFilename+time.Now().Format(time.RFC3339Nano)))) + "." + fileExtension
	outputFilePath := basePath + "/" + outputFileName

	args := []string{
		"--faces", *faces,
		"--haar", *cascade,
		downloadedFilePath}

	cmd := exec.Command("chrisify", args...)

	out, err := cmd.CombinedOutput()

	if err != nil {
		fmt.Println("Non-zero exit code: " + err.Error() + ", " + string(out))
	}

	err = ioutil.WriteFile(outputFilePath, out, 0644)

	fmt.Println("Image processed, putting in channel")

	imgCh <- outputFileName
	return
}

func downloadFromUrl(dUrl string, filename string, path string) (string, string, error) {
	err := os.MkdirAll(path, 755)
	if err != nil {
		return "", "", errors.New(fmt.Sprintln("Error while creating folder", path, "-", err))
	}

	timeout := time.Duration(time.Duration(DownloadTimeout) * time.Second)
	client := &http.Client{
		Timeout: timeout,
	}
	request, err := http.NewRequest("GET", dUrl, nil)
	if err != nil {
		return "", "", errors.New(fmt.Sprintln("Error while downloading", dUrl, "-", err))
	}
	request.Header.Add("Accept-Encoding", "identity")
	response, err := client.Do(request)
	if err != nil {
		return "", "", errors.New(fmt.Sprintln("Error while downloading", dUrl, "-", err))
	}
	defer response.Body.Close()

	if filename == "" {
		filename = filenameFromUrl(response.Request.URL.String())
		for key, iHeader := range response.Header {
			if key == "Content-Disposition" {
				_, params, err := mime.ParseMediaType(iHeader[0])
				if err == nil {
					newFilename, err := url.QueryUnescape(params["filename"])
					if err != nil {
						newFilename = params["filename"]
					}
					if newFilename != "" {
						filename = newFilename
					}
				}
			}
		}
	}

	completePath := path + string(os.PathSeparator) + filename
	if _, err := os.Stat(completePath); err == nil {
		tmpPath := completePath
		i := 1
		for {
			completePath = tmpPath[0:len(tmpPath)-len(filepath.Ext(tmpPath))] +
				"-" + strconv.Itoa(i) + filepath.Ext(tmpPath)
			if _, err := os.Stat(completePath); os.IsNotExist(err) {
				break
			}
			i = i + 1
		}
		fmt.Printf("[%s] Saving possible duplicate (filenames match): %s to %s\n", time.Now().Format(time.Stamp), tmpPath, completePath)
	}

	bodyOfResp, err := ioutil.ReadAll(response.Body)

	if err != nil {
		return "", "", errors.New(fmt.Sprintln("Could not read response", dUrl, "-", err))
	}

	contentType := http.DetectContentType(bodyOfResp)
	contentTypeParts := strings.Split(contentType, "/")

	if contentTypeParts[0] != "image" && contentTypeParts[0] != "video" {
		return "", "", errors.New(fmt.Sprintln("No image or video found at", dUrl))
	}

	err = ioutil.WriteFile(completePath, bodyOfResp, 0644)

	if err != nil {
		return "", "", errors.New(fmt.Sprintln("Error while writing to disk", dUrl, "-", err))
	}

	return filename, completePath, err
}

func filenameFromUrl(dUrl string) string {
	base := path.Base(dUrl)
	parts := strings.Split(base, "?")
	return parts[0]
}

func signalHandler(cancel context.CancelFunc) {
	sigCh := make(chan os.Signal, 1)
	signal.Notify(sigCh, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGINT)
	for {
		<-sigCh
		fmt.Println("Got stop signal, safely shutting down")
		cancel()
		return
	}
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"sync"
	"time"

	"golang.org/x/net/context"

	log15 "gopkg.in/inconshreveable/log15.v2"

	"github.com/CyCoreSystems/ari"
	"github.com/CyCoreSystems/ari/client/nc"
	v2 "github.com/CyCoreSystems/ari/v2"
)

var log = log15.New()
var wg sync.WaitGroup

func main() {

	<-time.After(20 * time.Second)

	if i := run(); i != 0 {
		os.Exit(i)
	}
}

func channelHandler(cl *ari.Client, h *ari.ChannelHandle) {
	log.Info("Running channel handler")
	defer wg.Done()

	// TODO: this subscription /should/ be channel specific. Not all Channel events are setup this way yet
	sub := h.Subscribe("ChannelHangupRequest")
	wg.Add(1)

	go func() {
		defer wg.Done()
		log.Info("Waiting for channel hangup request")
		<-sub.Events()
		log.Info("Got Channel hangup request")
	}()

	data, err := h.Data()
	if err != nil {
		log.Error("Error getting data", "error", err)
		return
	}
	log.Info("Channel Data", "data", data)

	h.Answer()

	h.Hangup()
}

func run() int {

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	// setup logging

	nc.Logger = log15.New()

	// connect

	cl, err := connect(ctx)
	if err != nil {
		log.Error("Failed to build nc ARI client", "error", err)
		return -1
	}

	// setup app

	log.Info("Starting listener app")

	go listenApp(ctx, cl, channelHandler)

	// make sample call

	wg.Add(1)
	log.Info("Make sample call")

	_, err = createCall(cl)
	if err != nil {
		log.Error("Failed to create call", "error", err)
	}

	wg.Wait()

	return 0
}

func listenApp(ctx context.Context, cl *ari.Client, handler func(cl *ari.Client, h *ari.ChannelHandle)) {
	sub := cl.Bus.Subscribe("StasisStart")

	select {
	case e := <-sub.Events():
		log.Info("Got stasis start")
		stasisStartEvent := e.(*v2.StasisStart)
		go handler(cl, cl.Channel.Get(stasisStartEvent.Channel.Id))
	case <-ctx.Done():
		return
	}

}

func createCall(cl *ari.Client) (h *ari.ChannelHandle, err error) {
	h, err = cl.Channel.Create(ari.OriginateRequest{
		Endpoint: "Local/1000",
		App:      "example",
	})

	return
}

func connect(ctx context.Context) (cl *ari.Client, err error) {

	opts := nc.Options{
		URL: "nats://nats:4222",
	}

	log.Info("Connecting")

	cl, err = nc.New(opts)
	return
}
<commit_msg>examples - additional event handling workflow<commit_after>package main

import (
	"os"
	"sync"
	"time"

	"golang.org/x/net/context"

	log15 "gopkg.in/inconshreveable/log15.v2"

	"github.com/CyCoreSystems/ari"
	"github.com/CyCoreSystems/ari/client/nc"
	v2 "github.com/CyCoreSystems/ari/v2"
)

var log = log15.New()
var wg sync.WaitGroup

func main() {

	<-time.After(20 * time.Second)

	if i := run(); i != 0 {
		os.Exit(i)
	}
}

func channelHandler(cl *ari.Client, h *ari.ChannelHandle) {
	log.Info("Running channel handler")

	stateChange := h.Subscribe("ChannelStateChange")
	defer stateChange.Cancel()

	hangup := h.Subscribe("ChannelHangupRequest")
	defer hangup.Cancel()

	data, err := h.Data()
	if err != nil {
		log.Error("Error getting data", "error", err)
		return
	}
	log.Info("Channel State", "state", data.State)

	go func() {
		log.Info("Waiting for channel events")

		defer wg.Done()

		for {
			select {
			case <-hangup.Events():
				log.Info("Got hangup")
				return
			case <-stateChange.Events():
				log.Info("Got state change request")

				data, err = h.Data()
				if err != nil {
					log.Error("Error getting data", "error", err)
					return
				}
				log.Info("New Channel State", "state", data.State)

				h.Hangup()
			}
		}

	}()

	h.Answer()

	wg.Wait()
}

func run() int {

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	// setup logging

	nc.Logger = log15.New()

	// connect

	cl, err := connect(ctx)
	if err != nil {
		log.Error("Failed to build nc ARI client", "error", err)
		return -1
	}

	// setup app

	log.Info("Starting listener app")

	go listenApp(ctx, cl, channelHandler)

	// make sample call

	wg.Add(1)
	log.Info("Make sample call")
	_, err = createCall(cl)
	if err != nil {
		log.Error("Failed to create call", "error", err)
	}

	wg.Wait()

	return 0
}

func listenApp(ctx context.Context, cl *ari.Client, handler func(cl *ari.Client, h *ari.ChannelHandle)) {
	sub := cl.Bus.Subscribe("StasisStart")

	select {
	case e := <-sub.Events():
		log.Info("Got stasis start")
		stasisStartEvent := e.(*v2.StasisStart)
		go handler(cl, cl.Channel.Get(stasisStartEvent.Channel.Id))
	case <-ctx.Done():
		return
	}

}

func createCall(cl *ari.Client) (h *ari.ChannelHandle, err error) {
	h, err = cl.Channel.Create(ari.OriginateRequest{
		Endpoint: "Local/1000",
		App:      "example",
	})

	return
}

func connect(ctx context.Context) (cl *ari.Client, err error) {

	opts := nc.Options{
		URL: "nats://nats:4222",
	}

	log.Info("Connecting")

	cl, err = nc.New(opts)
	return
}
<|endoftext|>


<commit_before>package serf

import (
	"net"
	"testing"

	"github.com/hashicorp/serf/serf"
	log "github.com/mgutz/logxi/v1"
	"github.com/stretchr/testify/suite"
)

// In order for 'go test' to run this suite, we need to create
// a normal test function and pass our suite to suite.Run
func TestEventHandlerSuite(t *testing.T) {
	suite.Run(t, new(EventHandlerTestSuite))
}

// Define the suite, and absorb the built-in basic suite
// functionality from testify - including a T() method which
// returns the current testing context
type EventHandlerTestSuite struct {
	suite.Suite
	Handler SerfEventHandler
	Member  serf.Member
}

// Make sure that VariableThatShouldStartAtFive is set to five
// before each test
func (suite *EventHandlerTestSuite) SetupTest() {
	suite.Handler = SerfEventHandler{
		Logger: &log.NullLogger{},
	}

	suite.Member = serf.Member{
		Name:        "",
		Addr:        net.ParseIP("127.0.0.1"),
		Port:        9022,
		Tags:        make(map[string]string),
		Status:      serf.StatusAlive,
		ProtocolMin: serf.ProtocolVersionMin,
		ProtocolMax: serf.ProtocolVersionMax,
		ProtocolCur: serf.ProtocolVersionMax,
		DelegateMin: serf.ProtocolVersionMin,
		DelegateMax: serf.ProtocolVersionMax,
		DelegateCur: serf.ProtocolVersionMax,
	}
}

// Test NodeJoin events are processed properly
func (suite *EventHandlerTestSuite) TestNodeJoined() {

	// Create Member Event
	evt := serf.MemberEvent{
		serf.EventMemberJoin,
		[]serf.Member{suite.Member},
	}

	// Add NodeJoined handler
	m := &MockMemberEventHandler{}
	m.On("HandleMemberEvent", evt).Return()
	suite.Handler.NodeJoined = m

	// Add Reconciler
	r := &MockReconciler{}
	r.On("Reconcile", evt).Return()
	suite.Handler.Reconciler = r

	// Process event
	suite.Handler.HandleEvent(evt)
	m.AssertCalled(suite.T(), "HandleMemberEvent", evt)
	r.AssertCalled(suite.T(), "Reconcile", evt)
}

// Test NodeLeave messages are dispatched properly
func (suite *EventHandlerTestSuite) TestNodeLeave() {

	// Create Member Event
	evt := serf.MemberEvent{
		serf.EventMemberLeave,
		[]serf.Member{suite.Member},
	}

	// Add NodeLeft handler
	m := &MockMemberEventHandler{}
	m.On("HandleMemberEvent", evt).Return()
	suite.Handler.NodeLeft = m

	// Add Reconciler
	r := &MockReconciler{}
	r.On("Reconcile", evt).Return()
	suite.Handler.Reconciler = r

	// Process event
	suite.Handler.HandleEvent(evt)
	m.AssertCalled(suite.T(), "HandleMemberEvent", evt)
	r.AssertCalled(suite.T(), "Reconcile", evt)
}

// Test NodeFailed messages are dispatched properly
func (suite *EventHandlerTestSuite) TestNodeFailed() {

	// Create Member Event
	evt := serf.MemberEvent{
		serf.EventMemberFailed,
		[]serf.Member{suite.Member},
	}

	// Add NodeFailed handler
	m := &MockMemberEventHandler{}
	m.On("HandleMemberEvent", evt).Return()
	suite.Handler.NodeFailed = m

	// Add Reconciler
	r := &MockReconciler{}
	r.On("Reconcile", evt).Return()
	suite.Handler.Reconciler = r

	// Process event
	suite.Handler.HandleEvent(evt)
	m.AssertCalled(suite.T(), "HandleMemberEvent", evt)
	r.AssertCalled(suite.T(), "Reconcile", evt)
}

// Test NodeReaped messages are dispatched properly
func (suite *EventHandlerTestSuite) TestNodeReaped() {

	// Create Member Event
	evt := serf.MemberEvent{
		serf.EventMemberReap,
		[]serf.Member{suite.Member},
	}

	// Add NodeReaped handler
	m := &MockMemberEventHandler{}
	m.On("HandleMemberEvent", evt).Return()
	suite.Handler.NodeReaped = m

	// Add Reconciler
	r := &MockReconciler{}
	r.On("Reconcile", evt).Return()
	suite.Handler.Reconciler = r

	// Process event
	suite.Handler.HandleEvent(evt)
	m.AssertCalled(suite.T(), "HandleMemberEvent", evt)
	r.AssertCalled(suite.T(), "Reconcile", evt)
}

// Test UserEvent messages are dispatched properly
func (suite *EventHandlerTestSuite) TestUserEvent() {

	// Create Member Event
	evt := serf.UserEvent{
		LTime:    serf.LamportTime(0),
		Name:     "Event",
		Payload:  make([]byte, 0),
		Coalesce: false,
	}

	// Add UserEvent handler
	m := &MockUserEventHandler{}
	m.On("HandleUserEvent", evt).Return()
	suite.Handler.UserEvent = m

	// Add Reconciler
	r := &MockReconciler{}
	r.On("Reconcile", evt).Return()
	suite.Handler.Reconciler = r

	// Process event
	suite.Handler.HandleEvent(evt)
	m.AssertCalled(suite.T(), "HandleUserEvent", evt)
	r.AssertNotCalled(suite.T(), "Reconcile", evt)
}

// Test NodeUpdated messages are dispatched properly
func (suite *EventHandlerTestSuite) TestNodeUpdated() {

	// Create Member Event
	evt := serf.MemberEvent{
		serf.EventMemberUpdate,
		[]serf.Member{suite.Member},
	}

	// Add NodeReaped handler
	m := &MockMemberEventHandler{}
	m.On("HandleMemberEvent", evt).Return()
	suite.Handler.NodeUpdated = m

	// Add Reconciler
	r := &MockReconciler{}
	r.On("Reconcile", evt).Return()
	suite.Handler.Reconciler = r

	// Process event
	suite.Handler.HandleEvent(evt)
	m.AssertCalled(suite.T(), "HandleMemberEvent", evt)
	r.AssertCalled(suite.T(), "Reconcile", evt)
}

// Test QueryEvent messages are dispatched properly
func (suite *EventHandlerTestSuite) TestQueryEvent() {

	// Create Query
	query := serf.Query{
		LTime:   serf.LamportTime(0),
		Name:    "Event",
		Payload: make([]byte, 0),
	}

	// Add UserEvent handler
	m := &MockQueryEventHandler{}
	m.On("HandleQueryEvent", query).Return()
	suite.Handler.QueryHandler = m

	// Add Reconciler
	r := &MockReconciler{}
	r.On("Reconcile", query).Return()
	suite.Handler.Reconciler = r

	// Process event
	suite.Handler.HandleEvent(&query)
	m.AssertCalled(suite.T(), "HandleQueryEvent", query)
	r.AssertNotCalled(suite.T(), "Reconcile", query)
}

// Test nil messages are not dispatched properly
func (suite *EventHandlerTestSuite) TestNilEvent() {

	// Add NodeJoined handler
	m1 := &MockMemberEventHandler{}
	suite.Handler.NodeJoined = m1

	// Add NodeLeft handler
	m2 := &MockMemberEventHandler{}
	suite.Handler.NodeLeft = m2

	// Add NodeFailed handler
	m3 := &MockMemberEventHandler{}
	suite.Handler.NodeFailed = m3

	// Add NodeReaped handler
	m4 := &MockMemberEventHandler{}
	suite.Handler.NodeReaped = m4

	// Add NodeUpdated handler
	m5 := &MockMemberEventHandler{}
	suite.Handler.NodeUpdated = m5

	// Add UserEvent handler
	u1 := &MockUserEventHandler{}
	suite.Handler.UserEvent = u1

	// Add UserEvent handler
	q1 := &MockQueryEventHandler{}
	suite.Handler.QueryHandler = q1

	// Add Reconciler
	r1 := &MockReconciler{}
	suite.Handler.Reconciler = r1

	// Process event
	suite.Handler.HandleEvent(nil)
	m1.AssertNotCalled(suite.T(), "HandleMemberEvent")
	m2.AssertNotCalled(suite.T(), "HandleMemberEvent")
	m3.AssertNotCalled(suite.T(), "HandleMemberEvent")
	m4.AssertNotCalled(suite.T(), "HandleMemberEvent")
	m5.AssertNotCalled(suite.T(), "HandleMemberEvent")
	u1.AssertNotCalled(suite.T(), "HandleUserEvent")
	q1.AssertNotCalled(suite.T(), "HandleQueryEvent")
	r1.AssertNotCalled(suite.T(), "Reconcile")
}

// Test unknown messages are not dispatched properly
func (suite *EventHandlerTestSuite) TestUnknownEvent() {

	// Add NodeJoined handler
	m1 := &MockMemberEventHandler{}
	suite.Handler.NodeJoined = m1

	// Add NodeLeft handler
	m2 := &MockMemberEventHandler{}
	suite.Handler.NodeLeft = m2

	// Add NodeFailed handler
	m3 := &MockMemberEventHandler{}
	suite.Handler.NodeFailed = m3

	// Add NodeReaped handler
	m4 := &MockMemberEventHandler{}
	suite.Handler.NodeReaped = m4

	// Add NodeUpdated handler
	m5 := &MockMemberEventHandler{}
	suite.Handler.NodeUpdated = m5

	// Add UserEvent handler
	u1 := &MockUserEventHandler{}
	suite.Handler.UserEvent = u1

	// Add UserEvent handler
	q1 := &MockQueryEventHandler{}
	suite.Handler.QueryHandler = q1

	// Add Reconciler
	r1 := &MockReconciler{}
	suite.Handler.Reconciler = r1

	// Process event
	t1 := &MockEvent{Name: "UnknownType", Type: serf.EventType(-1)}
	t1.On("EventType").Return()
	suite.Handler.HandleEvent(t1)

	// Test Assertions
	t1.AssertCalled(suite.T(), "EventType")
	m1.AssertNotCalled(suite.T(), "HandleMemberEvent")
	m2.AssertNotCalled(suite.T(), "HandleMemberEvent")
	m3.AssertNotCalled(suite.T(), "HandleMemberEvent")
	m4.AssertNotCalled(suite.T(), "HandleMemberEvent")
	m5.AssertNotCalled(suite.T(), "HandleMemberEvent")
	u1.AssertNotCalled(suite.T(), "HandleUserEvent")
	q1.AssertNotCalled(suite.T(), "HandleQueryEvent")
	r1.AssertNotCalled(suite.T(), "Reconcile")
}
<commit_msg>Update docs and rename package<commit_after>package serfer

import (
	"net"
	"testing"

	"github.com/hashicorp/serf/serf"
	log "github.com/mgutz/logxi/v1"
	"github.com/stretchr/testify/suite"
)

// In order for 'go test' to run this suite, we need to create
// a normal test function and pass our suite to suite.Run
func TestEventHandlerSuite(t *testing.T) {
	suite.Run(t, new(EventHandlerTestSuite))
}

// Define the suite, and absorb the built-in basic suite
// functionality from testify - including a T() method which
// returns the current testing context
type EventHandlerTestSuite struct {
	suite.Suite
	Handler SerfEventHandler
	Member  serf.Member
}

// Make sure that Handler and Member are set before each test
func (suite *EventHandlerTestSuite) SetupTest() {
	suite.Handler = SerfEventHandler{
		Logger: &log.NullLogger{},
	}

	suite.Member = serf.Member{
		Name:        "",
		Addr:        net.ParseIP("127.0.0.1"),
		Port:        9022,
		Tags:        make(map[string]string),
		Status:      serf.StatusAlive,
		ProtocolMin: serf.ProtocolVersionMin,
		ProtocolMax: serf.ProtocolVersionMax,
		ProtocolCur: serf.ProtocolVersionMax,
		DelegateMin: serf.ProtocolVersionMin,
		DelegateMax: serf.ProtocolVersionMax,
		DelegateCur: serf.ProtocolVersionMax,
	}
}

// Test NodeJoin events are processed properly
func (suite *EventHandlerTestSuite) TestNodeJoined() {

	// Create Member Event
	evt := serf.MemberEvent{
		serf.EventMemberJoin,
		[]serf.Member{suite.Member},
	}

	// Add NodeJoined handler
	m := &MockMemberEventHandler{}
	m.On("HandleMemberEvent", evt).Return()
	suite.Handler.NodeJoined = m

	// Add Reconciler
	r := &MockReconciler{}
	r.On("Reconcile", evt).Return()
	suite.Handler.Reconciler = r

	// Process event
	suite.Handler.HandleEvent(evt)
	m.AssertCalled(suite.T(), "HandleMemberEvent", evt)
	r.AssertCalled(suite.T(), "Reconcile", evt)
}

// Test NodeLeave messages are dispatched properly
func (suite *EventHandlerTestSuite) TestNodeLeave() {

	// Create Member Event
	evt := serf.MemberEvent{
		serf.EventMemberLeave,
		[]serf.Member{suite.Member},
	}

	// Add NodeLeft handler
	m := &MockMemberEventHandler{}
	m.On("HandleMemberEvent", evt).Return()
	suite.Handler.NodeLeft = m

	// Add Reconciler
	r := &MockReconciler{}
	r.On("Reconcile", evt).Return()
	suite.Handler.Reconciler = r

	// Process event
	suite.Handler.HandleEvent(evt)
	m.AssertCalled(suite.T(), "HandleMemberEvent", evt)
	r.AssertCalled(suite.T(), "Reconcile", evt)
}

// Test NodeFailed messages are dispatched properly
func (suite *EventHandlerTestSuite) TestNodeFailed() {

	// Create Member Event
	evt := serf.MemberEvent{
		serf.EventMemberFailed,
		[]serf.Member{suite.Member},
	}

	// Add NodeFailed handler
	m := &MockMemberEventHandler{}
	m.On("HandleMemberEvent", evt).Return()
	suite.Handler.NodeFailed = m

	// Add Reconciler
	r := &MockReconciler{}
	r.On("Reconcile", evt).Return()
	suite.Handler.Reconciler = r

	// Process event
	suite.Handler.HandleEvent(evt)
	m.AssertCalled(suite.T(), "HandleMemberEvent", evt)
	r.AssertCalled(suite.T(), "Reconcile", evt)
}

// Test NodeReaped messages are dispatched properly
func (suite *EventHandlerTestSuite) TestNodeReaped() {

	// Create Member Event
	evt := serf.MemberEvent{
		serf.EventMemberReap,
		[]serf.Member{suite.Member},
	}

	// Add NodeReaped handler
	m := &MockMemberEventHandler{}
	m.On("HandleMemberEvent", evt).Return()
	suite.Handler.NodeReaped = m

	// Add Reconciler
	r := &MockReconciler{}
	r.On("Reconcile", evt).Return()
	suite.Handler.Reconciler = r

	// Process event
	suite.Handler.HandleEvent(evt)
	m.AssertCalled(suite.T(), "HandleMemberEvent", evt)
	r.AssertCalled(suite.T(), "Reconcile", evt)
}

// Test UserEvent messages are dispatched properly
func (suite *EventHandlerTestSuite) TestUserEvent() {

	// Create Member Event
	evt := serf.UserEvent{
		LTime:    serf.LamportTime(0),
		Name:     "Event",
		Payload:  make([]byte, 0),
		Coalesce: false,
	}

	// Add UserEvent handler
	m := &MockUserEventHandler{}
	m.On("HandleUserEvent", evt).Return()
	suite.Handler.UserEvent = m

	// Add Reconciler
	r := &MockReconciler{}
	r.On("Reconcile", evt).Return()
	suite.Handler.Reconciler = r

	// Process event
	suite.Handler.HandleEvent(evt)
	m.AssertCalled(suite.T(), "HandleUserEvent", evt)
	r.AssertNotCalled(suite.T(), "Reconcile", evt)
}

// Test NodeUpdated messages are dispatched properly
func (suite *EventHandlerTestSuite) TestNodeUpdated() {

	// Create Member Event
	evt := serf.MemberEvent{
		serf.EventMemberUpdate,
		[]serf.Member{suite.Member},
	}

	// Add NodeReaped handler
	m := &MockMemberEventHandler{}
	m.On("HandleMemberEvent", evt).Return()
	suite.Handler.NodeUpdated = m

	// Add Reconciler
	r := &MockReconciler{}
	r.On("Reconcile", evt).Return()
	suite.Handler.Reconciler = r

	// Process event
	suite.Handler.HandleEvent(evt)
	m.AssertCalled(suite.T(), "HandleMemberEvent", evt)
	r.AssertCalled(suite.T(), "Reconcile", evt)
}

// Test QueryEvent messages are dispatched properly
func (suite *EventHandlerTestSuite) TestQueryEvent() {

	// Create Query
	query := serf.Query{
		LTime:   serf.LamportTime(0),
		Name:    "Event",
		Payload: make([]byte, 0),
	}

	// Add UserEvent handler
	m := &MockQueryEventHandler{}
	m.On("HandleQueryEvent", query).Return()
	suite.Handler.QueryHandler = m

	// Add Reconciler
	r := &MockReconciler{}
	r.On("Reconcile", query).Return()
	suite.Handler.Reconciler = r

	// Process event
	suite.Handler.HandleEvent(&query)
	m.AssertCalled(suite.T(), "HandleQueryEvent", query)
	r.AssertNotCalled(suite.T(), "Reconcile", query)
}

// Test nil messages are not dispatched properly
func (suite *EventHandlerTestSuite) TestNilEvent() {

	// Add NodeJoined handler
	m1 := &MockMemberEventHandler{}
	suite.Handler.NodeJoined = m1

	// Add NodeLeft handler
	m2 := &MockMemberEventHandler{}
	suite.Handler.NodeLeft = m2

	// Add NodeFailed handler
	m3 := &MockMemberEventHandler{}
	suite.Handler.NodeFailed = m3

	// Add NodeReaped handler
	m4 := &MockMemberEventHandler{}
	suite.Handler.NodeReaped = m4

	// Add NodeUpdated handler
	m5 := &MockMemberEventHandler{}
	suite.Handler.NodeUpdated = m5

	// Add UserEvent handler
	u1 := &MockUserEventHandler{}
	suite.Handler.UserEvent = u1

	// Add UserEvent handler
	q1 := &MockQueryEventHandler{}
	suite.Handler.QueryHandler = q1

	// Add Reconciler
	r1 := &MockReconciler{}
	suite.Handler.Reconciler = r1

	// Process event
	suite.Handler.HandleEvent(nil)
	m1.AssertNotCalled(suite.T(), "HandleMemberEvent")
	m2.AssertNotCalled(suite.T(), "HandleMemberEvent")
	m3.AssertNotCalled(suite.T(), "HandleMemberEvent")
	m4.AssertNotCalled(suite.T(), "HandleMemberEvent")
	m5.AssertNotCalled(suite.T(), "HandleMemberEvent")
	u1.AssertNotCalled(suite.T(), "HandleUserEvent")
	q1.AssertNotCalled(suite.T(), "HandleQueryEvent")
	r1.AssertNotCalled(suite.T(), "Reconcile")
}

// Test unknown messages are not dispatched properly
func (suite *EventHandlerTestSuite) TestUnknownEvent() {

	// Add NodeJoined handler
	m1 := &MockMemberEventHandler{}
	suite.Handler.NodeJoined = m1

	// Add NodeLeft handler
	m2 := &MockMemberEventHandler{}
	suite.Handler.NodeLeft = m2

	// Add NodeFailed handler
	m3 := &MockMemberEventHandler{}
	suite.Handler.NodeFailed = m3

	// Add NodeReaped handler
	m4 := &MockMemberEventHandler{}
	suite.Handler.NodeReaped = m4

	// Add NodeUpdated handler
	m5 := &MockMemberEventHandler{}
	suite.Handler.NodeUpdated = m5

	// Add UserEvent handler
	u1 := &MockUserEventHandler{}
	suite.Handler.UserEvent = u1

	// Add UserEvent handler
	q1 := &MockQueryEventHandler{}
	suite.Handler.QueryHandler = q1

	// Add Reconciler
	r1 := &MockReconciler{}
	suite.Handler.Reconciler = r1

	// Process event
	t1 := &MockEvent{Name: "UnknownType", Type: serf.EventType(-1)}
	t1.On("EventType").Return()
	suite.Handler.HandleEvent(t1)

	// Test Assertions
	t1.AssertCalled(suite.T(), "EventType")
	m1.AssertNotCalled(suite.T(), "HandleMemberEvent")
	m2.AssertNotCalled(suite.T(), "HandleMemberEvent")
	m3.AssertNotCalled(suite.T(), "HandleMemberEvent")
	m4.AssertNotCalled(suite.T(), "HandleMemberEvent")
	m5.AssertNotCalled(suite.T(), "HandleMemberEvent")
	u1.AssertNotCalled(suite.T(), "HandleUserEvent")
	q1.AssertNotCalled(suite.T(), "HandleQueryEvent")
	r1.AssertNotCalled(suite.T(), "Reconcile")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
)

type testrouter struct {
	path             string
	body             []byte
	deleteCalledWith string
	deleteBool       bool
}

func (tr *testrouter) Get(_ string) (http.Handler, bool) {
	if len(tr.body) == 0 {
		return nil, false
	}
	h := func(rw http.ResponseWriter, _ *http.Request) {
		rw.Write(tr.body)
	}
	return http.HandlerFunc(h), true
}

func (tr *testrouter) Set(path string, req *http.Request) error {
	var err error
	tr.body, err = ioutil.ReadAll(req.Body)
	tr.path = path
	return err
}

func (tr *testrouter) Del(path string) bool {
	tr.deleteCalledWith = path
	return tr.deleteBool
}

func TestGetHandler(t *testing.T) {
	type checkFunc func(*httptest.ResponseRecorder) error
	check := func(fns ...checkFunc) []checkFunc { return fns }

	hasStatus := func(want int) checkFunc {
		return func(rec *httptest.ResponseRecorder) error {
			if rec.Code != want {
				return fmt.Errorf("expected status %d, found %d", want, rec.Code)
			}
			return nil
		}
	}
	hasContents := func(want string) checkFunc {
		return func(rec *httptest.ResponseRecorder) error {
			if have := rec.Body.String(); have != want {
				return fmt.Errorf("expected body %q, found %q", want, have)
			}
			return nil
		}
	}

	handlerWithBody := func(body string) *testrouter {
		return &testrouter{body: []byte(body)}
	}

	tests := [...]struct {
		name   string
		store  *testrouter
		checks []checkFunc
	}{
		{
			"gets",
			handlerWithBody("hey"),
			check(
				hasStatus(200),
				hasContents("hey"),
			),
		},
		{
			"miss is 404",
			&testrouter{},
			check(
				hasStatus(404),
			),
		},
	}

	req, _ := http.NewRequest("GET", "http://foo.com/", strings.NewReader(""))
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			h := getHandler(tc.store)
			rec := httptest.NewRecorder()
			h(rec, req)
			for _, check := range tc.checks {
				if err := check(rec); err != nil {
					t.Error(err)
				}
			}
		})
	}
}

func TestPutHandler(t *testing.T) {
	type checkFunc func(*testrouter, *httptest.ResponseRecorder) error
	check := func(fns ...checkFunc) []checkFunc { return fns }

	responseHasStatus := func(want int) checkFunc {
		return func(_ *testrouter, rec *httptest.ResponseRecorder) error {
			if rec.Code != want {
				return fmt.Errorf("expected status %d, found %d", want, rec.Code)
			}
			return nil
		}
	}
	responseHasContents := func(want string) checkFunc {
		return func(_ *testrouter, rec *httptest.ResponseRecorder) error {
			if have := rec.Body.String(); have != want {
				return fmt.Errorf("expected body %q, found %q", want, have)
			}
			return nil
		}
	}
	storeHasPath := func(want string) checkFunc {
		return func(router *testrouter, _ *httptest.ResponseRecorder) error {
			if have := router.path; have != want {
				return fmt.Errorf("expected new path %q, found %q", want, have)
			}
			return nil
		}
	}
	storeHasBody := func(want string) checkFunc {
		return func(router *testrouter, _ *httptest.ResponseRecorder) error {
			if have := string(router.body); have != want {
				return fmt.Errorf("expected new body %q, found %q", want, have)
			}
			return nil
		}
	}

	tests := [...]struct {
		name   string
		path   string
		body   string
		checks []checkFunc
	}{
		{
			"stores a new entry",
			"/wow",
			`{"content": "NEW!"}`,
			check(
				storeHasPath("/wow"),
				storeHasBody(`{"content": "NEW!"}`),
			),
		},
		{
			"returns the newly created entry",
			"/wow",
			`{"content": "NEW!"}`,
			check(
				responseHasStatus(200),
				responseHasContents(`{"content": "NEW!"}`),
			),
		},
	}

	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			req, _ := http.NewRequest("PUT", tc.path, strings.NewReader(tc.body))
			store := &testrouter{}
			h := putHandler(store)
			rec := httptest.NewRecorder()
			h(rec, req)
			for _, check := range tc.checks {
				if err := check(store, rec); err != nil {
					t.Error(err)
				}
			}
		})
	}
}

func TestDeleteHandler(t *testing.T) {
	type checkFunc func(*testrouter, *httptest.ResponseRecorder) error
	check := func(fns ...checkFunc) []checkFunc { return fns }

	responseHasStatus := func(want int) checkFunc {
		return func(_ *testrouter, rec *httptest.ResponseRecorder) error {
			if rec.Code != want {
				return fmt.Errorf("expected status %d, found %d", want, rec.Code)
			}
			return nil
		}
	}
	deleteCalledWith := func(want string) checkFunc {
		return func(router *testrouter, _ *httptest.ResponseRecorder) error {
			if have := router.deleteCalledWith; have != want {
				return fmt.Errorf("expected Del called with path %q, found %q", want, have)
			}
			return nil
		}
	}

	tests := [...]struct {
		name   string
		path   string
		store  *testrouter
		checks []checkFunc
	}{
		{
			"deletes an entry",
			"/wow",
			&testrouter{deleteBool: true},
			check(
				deleteCalledWith("/wow"),
				responseHasStatus(204),
			),
		},
		{
			"returns 404 for unknown routes",
			"/wow",
			&testrouter{deleteBool: false},
			check(
				deleteCalledWith("/wow"),
				responseHasStatus(404),
			),
		},
	}

	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			req, _ := http.NewRequest("DELETE", tc.path, strings.NewReader(""))
			h := deleteHandler(tc.store)
			rec := httptest.NewRecorder()
			h(rec, req)
			for _, check := range tc.checks {
				if err := check(tc.store, rec); err != nil {
					t.Error(err)
				}
			}
		})
	}
}
<commit_msg>Test OPTIONS handler<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
)

type testrouter struct {
	path             string
	body             []byte
	deleteCalledWith string
	deleteBool       bool
}

func (tr *testrouter) Get(_ string) (http.Handler, bool) {
	if len(tr.body) == 0 {
		return nil, false
	}
	h := func(rw http.ResponseWriter, _ *http.Request) {
		rw.Write(tr.body)
	}
	return http.HandlerFunc(h), true
}

func (tr *testrouter) Set(path string, req *http.Request) error {
	var err error
	tr.body, err = ioutil.ReadAll(req.Body)
	tr.path = path
	return err
}

func (tr *testrouter) Del(path string) bool {
	tr.deleteCalledWith = path
	return tr.deleteBool
}

func TestGetHandler(t *testing.T) {
	type checkFunc func(*httptest.ResponseRecorder) error
	check := func(fns ...checkFunc) []checkFunc { return fns }

	hasStatus := func(want int) checkFunc {
		return func(rec *httptest.ResponseRecorder) error {
			if rec.Code != want {
				return fmt.Errorf("expected status %d, found %d", want, rec.Code)
			}
			return nil
		}
	}
	hasContents := func(want string) checkFunc {
		return func(rec *httptest.ResponseRecorder) error {
			if have := rec.Body.String(); have != want {
				return fmt.Errorf("expected body %q, found %q", want, have)
			}
			return nil
		}
	}

	handlerWithBody := func(body string) *testrouter {
		return &testrouter{body: []byte(body)}
	}

	tests := [...]struct {
		name   string
		store  *testrouter
		checks []checkFunc
	}{
		{
			"gets",
			handlerWithBody("hey"),
			check(
				hasStatus(200),
				hasContents("hey"),
			),
		},
		{
			"miss is 404",
			&testrouter{},
			check(
				hasStatus(404),
			),
		},
	}

	req, _ := http.NewRequest("GET", "http://foo.com/", strings.NewReader(""))
	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			h := getHandler(tc.store)
			rec := httptest.NewRecorder()
			h(rec, req)
			for _, check := range tc.checks {
				if err := check(rec); err != nil {
					t.Error(err)
				}
			}
		})
	}
}

func TestPutHandler(t *testing.T) {
	type checkFunc func(*testrouter, *httptest.ResponseRecorder) error
	check := func(fns ...checkFunc) []checkFunc { return fns }

	responseHasStatus := func(want int) checkFunc {
		return func(_ *testrouter, rec *httptest.ResponseRecorder) error {
			if rec.Code != want {
				return fmt.Errorf("expected status %d, found %d", want, rec.Code)
			}
			return nil
		}
	}
	responseHasContents := func(want string) checkFunc {
		return func(_ *testrouter, rec *httptest.ResponseRecorder) error {
			if have := rec.Body.String(); have != want {
				return fmt.Errorf("expected body %q, found %q", want, have)
			}
			return nil
		}
	}
	storeHasPath := func(want string) checkFunc {
		return func(router *testrouter, _ *httptest.ResponseRecorder) error {
			if have := router.path; have != want {
				return fmt.Errorf("expected new path %q, found %q", want, have)
			}
			return nil
		}
	}
	storeHasBody := func(want string) checkFunc {
		return func(router *testrouter, _ *httptest.ResponseRecorder) error {
			if have := string(router.body); have != want {
				return fmt.Errorf("expected new body %q, found %q", want, have)
			}
			return nil
		}
	}

	tests := [...]struct {
		name   string
		path   string
		body   string
		checks []checkFunc
	}{
		{
			"stores a new entry",
			"/wow",
			`{"content": "NEW!"}`,
			check(
				storeHasPath("/wow"),
				storeHasBody(`{"content": "NEW!"}`),
			),
		},
		{
			"returns the newly created entry",
			"/wow",
			`{"content": "NEW!"}`,
			check(
				responseHasStatus(200),
				responseHasContents(`{"content": "NEW!"}`),
			),
		},
	}

	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			req, _ := http.NewRequest("PUT", tc.path, strings.NewReader(tc.body))
			store := &testrouter{}
			h := putHandler(store)
			rec := httptest.NewRecorder()
			h(rec, req)
			for _, check := range tc.checks {
				if err := check(store, rec); err != nil {
					t.Error(err)
				}
			}
		})
	}
}

func TestDeleteHandler(t *testing.T) {
	type checkFunc func(*testrouter, *httptest.ResponseRecorder) error
	check := func(fns ...checkFunc) []checkFunc { return fns }

	responseHasStatus := func(want int) checkFunc {
		return func(_ *testrouter, rec *httptest.ResponseRecorder) error {
			if rec.Code != want {
				return fmt.Errorf("expected status %d, found %d", want, rec.Code)
			}
			return nil
		}
	}
	deleteCalledWith := func(want string) checkFunc {
		return func(router *testrouter, _ *httptest.ResponseRecorder) error {
			if have := router.deleteCalledWith; have != want {
				return fmt.Errorf("expected Del called with path %q, found %q", want, have)
			}
			return nil
		}
	}

	tests := [...]struct {
		name   string
		path   string
		store  *testrouter
		checks []checkFunc
	}{
		{
			"deletes an entry",
			"/wow",
			&testrouter{deleteBool: true},
			check(
				deleteCalledWith("/wow"),
				responseHasStatus(204),
			),
		},
		{
			"returns 404 for unknown routes",
			"/wow",
			&testrouter{deleteBool: false},
			check(
				deleteCalledWith("/wow"),
				responseHasStatus(404),
			),
		},
	}

	for _, tc := range tests {
		t.Run(tc.name, func(t *testing.T) {
			req, _ := http.NewRequest("DELETE", tc.path, strings.NewReader(""))
			h := deleteHandler(tc.store)
			rec := httptest.NewRecorder()
			h(rec, req)
			for _, check := range tc.checks {
				if err := check(tc.store, rec); err != nil {
					t.Error(err)
				}
			}
		})
	}
}

func TestOptionsHandler(t *testing.T) {
	t.Run("returns 204", func(t *testing.T) {
		req, _ := http.NewRequest("OPTIONS", "/", strings.NewReader(""))
		rec := httptest.NewRecorder()
		optionsHandler(rec, req)
		if want, have := 204, rec.Code; want != have {
			t.Errorf("expected status code %d, found %d", want, have)
		}
	})
}
<|endoftext|>


<commit_before>package storage

import (
	"fmt"

	k8sv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/utils/pointer"

	v1 "kubevirt.io/client-go/api/v1"
	"kubevirt.io/kubevirt/tests/flags"
)

func RenderNFSServer(generateName string, hostPath string) *k8sv1.Pod {
	image := fmt.Sprintf("%s/nfs-server:%s", flags.KubeVirtRepoPrefix, flags.KubeVirtVersionTag)
	resources := k8sv1.ResourceRequirements{}
	resources.Requests = make(k8sv1.ResourceList)
	resources.Requests[k8sv1.ResourceMemory] = resource.MustParse("256M")
	resources.Requests[k8sv1.ResourceCPU] = resource.MustParse("500m")
	hostPathType := k8sv1.HostPathDirectory
	pod := &k8sv1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: generateName,
			Labels: map[string]string{
				v1.AppLabel: generateName,
			},
		},
		Spec: k8sv1.PodSpec{
			RestartPolicy: k8sv1.RestartPolicyNever,
			Volumes: []k8sv1.Volume{
				{
					Name: "nfsdata",
					VolumeSource: k8sv1.VolumeSource{
						HostPath: &k8sv1.HostPathVolumeSource{
							Path: hostPath,
							Type: &hostPathType,
						},
					},
				},
			},
			Containers: []k8sv1.Container{
				{
					Name:            generateName,
					Image:           image,
					ImagePullPolicy: k8sv1.PullAlways,
					Resources:       resources,
					SecurityContext: &k8sv1.SecurityContext{
						Privileged: pointer.BoolPtr(true),
					},
					VolumeMounts: []k8sv1.VolumeMount{
						{
							Name:      "nfsdata",
							MountPath: "/data/nfs",
						},
					},
				},
			},
		},
	}
	return pod
}
<commit_msg>change the image pull policy for the NFS test pod to IfNotPresent<commit_after>package storage

import (
	"fmt"

	k8sv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/utils/pointer"

	v1 "kubevirt.io/client-go/api/v1"
	"kubevirt.io/kubevirt/tests/flags"
)

func RenderNFSServer(generateName string, hostPath string) *k8sv1.Pod {
	image := fmt.Sprintf("%s/nfs-server:%s", flags.KubeVirtRepoPrefix, flags.KubeVirtVersionTag)
	resources := k8sv1.ResourceRequirements{}
	resources.Requests = make(k8sv1.ResourceList)
	resources.Requests[k8sv1.ResourceMemory] = resource.MustParse("256M")
	resources.Requests[k8sv1.ResourceCPU] = resource.MustParse("500m")
	hostPathType := k8sv1.HostPathDirectory
	pod := &k8sv1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: generateName,
			Labels: map[string]string{
				v1.AppLabel: generateName,
			},
		},
		Spec: k8sv1.PodSpec{
			RestartPolicy: k8sv1.RestartPolicyNever,
			Volumes: []k8sv1.Volume{
				{
					Name: "nfsdata",
					VolumeSource: k8sv1.VolumeSource{
						HostPath: &k8sv1.HostPathVolumeSource{
							Path: hostPath,
							Type: &hostPathType,
						},
					},
				},
			},
			Containers: []k8sv1.Container{
				{
					Name:            generateName,
					Image:           image,
					ImagePullPolicy: k8sv1.PullIfNotPresent,
					Resources:       resources,
					SecurityContext: &k8sv1.SecurityContext{
						Privileged: pointer.BoolPtr(true),
					},
					VolumeMounts: []k8sv1.VolumeMount{
						{
							Name:      "nfsdata",
							MountPath: "/data/nfs",
						},
					},
				},
			},
		},
	}
	return pod
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/codegangsta/cli"
	"github.com/fsouza/go-dockerclient"
	"fmt"
	"os"
	"sync"
	"time"
)

func collectAPIImages(images []docker.APIImages, client *docker.Client, ctx *cli.Context, excludes []string) {
	var imageSync sync.WaitGroup
	grace := ctx.Duration("grace")
	quiet := ctx.Bool("quiet")
	options := docker.RemoveImageOptions{
		Force:   ctx.Bool("force"),
		NoPrune: ctx.Bool("no-prune"),
	}
	for _, image := range images {
		imageSync.Add(1)
		go func(image docker.APIImages) {
			defer imageSync.Done()
			imageDetail, _ := client.InspectImage(image.ID)
			now := time.Now()
			if now.Sub(imageDetail.Created) >= grace {
				if err := client.RemoveImageExtended(imageDetail.ID, options); err == nil {
					if !quiet {
						fmt.Printf("Deleted image: %s.\n", imageDetail.ID)
					}
				}
			}
		}(image)
	}
	imageSync.Wait()
}

func collectAPIContainers(containers []docker.APIContainers, client *docker.Client, ctx *cli.Context, excludes []string) {
	var containerSync sync.WaitGroup
	grace := ctx.Duration("grace")
	quiet := ctx.Bool("quiet")
	for _, container := range containers {
		containerSync.Add(1)
		go func(container docker.APIContainers){
			defer containerSync.Done()
			containerDetail, _ := client.InspectContainer(container.ID)
			now := time.Now()
			if now.Sub(containerDetail.Created) >= grace {
				options := docker.RemoveContainerOptions{
					ID:            containerDetail.ID,
					RemoveVolumes: ctx.Bool("remove-volumes"),
					Force:         ctx.Bool("force"),
				}
				if err := client.RemoveContainer(options); err == nil {
					if !quiet {
						fmt.Printf("Deleted container: %s.\n", containerDetail.ID)
					}
				}
			}
		}(container)
	}
	containerSync.Wait()
}

func runDgc(ctx *cli.Context) {
	var dgcSync sync.WaitGroup
	client, _ := docker.NewClient(ctx.String("socket"))
	images, _ := client.ListImages(docker.ListImagesOptions{All: true})
	containers, _ := client.ListContainers(docker.ListContainersOptions{All: true})
	excludes, _ := readLines(ctx.String("exclude"))
	dgcSync.Add(2)
	go func() {
		defer dgcSync.Done()
		collectAPIContainers(containers, client, ctx, excludes)
	}()
	go func() {
		defer dgcSync.Done()
		collectAPIImages(images, client, ctx, excludes)
	}()
	dgcSync.Wait()
}

func main() {
	dgc := cli.NewApp()
	dgc.EnableBashCompletion = true
	dgc.Name = "dgc"
	dgc.Usage = "A minimal docker garbage collector"
	dgc.Version = "0.1.0"
	dgc.Author = "David J Felix <davidjfelix@davidjfelix.com>"
	dgc.Action = runDgc
	dgc.Flags = []cli.Flag{
		cli.DurationFlag{
			Name:   "grace, g",
			Value:  time.Duration(3600) * time.Second,
			Usage:  "the grace period for a container. Accepted compostable time units: [h, m, s, ms, ns us]",
			EnvVar: "GRACE_PERIOD_SECONDS,GRACE_PERIOD",
		},
		cli.StringFlag{
			Name:   "socket, s",
			Value:  "unix:///var/run/docker.sock",
			Usage:  "the docker remote socket",
			EnvVar: "DOCKER_SOCKET",
		},
		cli.StringFlag{
			Name:   "exclude, e",
			Value:  "/etc/docker-gc-exclude",
			Usage:  "the list of containers to exclude from garbage collection, as a file or directory",
			EnvVar: "EXCLUDE_FROM_GC",
		},
		cli.BoolFlag{
			Name:  "quiet, q",
			Usage: "don't print name of garbage-collected containers",
		},
		cli.BoolTFlag{
			Name:  "remove-volumes, r",
			Usage: "remove volumes with the container",
		},
		cli.BoolFlag{
			Name:  "force, f",
			Usage: "force images and containers to stop and be collected",
		},
		cli.BoolFlag{
			Name:  "no-prune, n",
			Usage: "don't prune parent images to a GC'd image",
		},
	}
	dgc.Run(os.Args)
}
<commit_msg>Update dgc.go<commit_after>package main

import (
	"github.com/codegangsta/cli"
	"github.com/fsouza/go-dockerclient"
	"fmt"
	"os"
	"sync"
	"time"
)

func readExcludes() {
}

func collectAPIImages(images []docker.APIImages, client *docker.Client, ctx *cli.Context, excludes []string) {
	var imageSync sync.WaitGroup
	grace := ctx.Duration("grace")
	quiet := ctx.Bool("quiet")
	options := docker.RemoveImageOptions{
		Force:   ctx.Bool("force"),
		NoPrune: ctx.Bool("no-prune"),
	}
	for _, image := range images {
		imageSync.Add(1)
		go func(image docker.APIImages) {
			defer imageSync.Done()
			imageDetail, _ := client.InspectImage(image.ID)
			now := time.Now()
			if now.Sub(imageDetail.Created) >= grace {
				if err := client.RemoveImageExtended(imageDetail.ID, options); err == nil {
					if !quiet {
						fmt.Printf("Deleted image: %s.\n", imageDetail.ID)
					}
				}
			}
		}(image)
	}
	imageSync.Wait()
}

func collectAPIContainers(containers []docker.APIContainers, client *docker.Client, ctx *cli.Context, excludes []string) {
	var containerSync sync.WaitGroup
	grace := ctx.Duration("grace")
	quiet := ctx.Bool("quiet")
	for _, container := range containers {
		containerSync.Add(1)
		go func(container docker.APIContainers){
			defer containerSync.Done()
			containerDetail, _ := client.InspectContainer(container.ID)
			now := time.Now()
			if now.Sub(containerDetail.Created) >= grace {
				options := docker.RemoveContainerOptions{
					ID:            containerDetail.ID,
					RemoveVolumes: ctx.Bool("remove-volumes"),
					Force:         ctx.Bool("force"),
				}
				if err := client.RemoveContainer(options); err == nil {
					if !quiet {
						fmt.Printf("Deleted container: %s.\n", containerDetail.ID)
					}
				}
			}
		}(container)
	}
	containerSync.Wait()
}

func runDgc(ctx *cli.Context) {
	var dgcSync sync.WaitGroup
	client, _ := docker.NewClient(ctx.String("socket"))
	images, _ := client.ListImages(docker.ListImagesOptions{All: true})
	containers, _ := client.ListContainers(docker.ListContainersOptions{All: true})
	excludes, _ := readLines(ctx.String("exclude"))
	dgcSync.Add(2)
	go func() {
		defer dgcSync.Done()
		collectAPIContainers(containers, client, ctx, excludes)
	}()
	go func() {
		defer dgcSync.Done()
		collectAPIImages(images, client, ctx, excludes)
	}()
	dgcSync.Wait()
}

func main() {
	dgc := cli.NewApp()
	dgc.EnableBashCompletion = true
	dgc.Name = "dgc"
	dgc.Usage = "A minimal docker garbage collector"
	dgc.Version = "0.1.0"
	dgc.Author = "David J Felix <davidjfelix@davidjfelix.com>"
	dgc.Action = runDgc
	dgc.Flags = []cli.Flag{
		cli.DurationFlag{
			Name:   "grace, g",
			Value:  time.Duration(3600) * time.Second,
			Usage:  "the grace period for a container. Accepted compostable time units: [h, m, s, ms, ns us]",
			EnvVar: "GRACE_PERIOD_SECONDS,GRACE_PERIOD",
		},
		cli.StringFlag{
			Name:   "socket, s",
			Value:  "unix:///var/run/docker.sock",
			Usage:  "the docker remote socket",
			EnvVar: "DOCKER_SOCKET",
		},
		cli.StringFlag{
			Name:   "exclude, e",
			Value:  "/etc/docker-gc-exclude",
			Usage:  "the list of containers to exclude from garbage collection, as a file or directory",
			EnvVar: "EXCLUDE_FROM_GC",
		},
		cli.BoolFlag{
			Name:  "quiet, q",
			Usage: "don't print name of garbage-collected containers",
		},
		cli.BoolTFlag{
			Name:  "remove-volumes, r",
			Usage: "remove volumes with the container",
		},
		cli.BoolFlag{
			Name:  "force, f",
			Usage: "force images and containers to stop and be collected",
		},
		cli.BoolFlag{
			Name:  "no-prune, n",
			Usage: "don't prune parent images to a GC'd image",
		},
	}
	dgc.Run(os.Args)
}
<|endoftext|>


<commit_before>package main

import (
	"strings"

	"github.com/ikawaha/kagome/tokenizer"

	"unicode/utf8"
)

type FeatureVector []string

func ExtractNounFeatures(s string, prefix string) FeatureVector {
	var fv FeatureVector
	if s == "" {
		return fv
	}
	t := tokenizer.New()
	tokens := t.Tokenize(s)
	for _, token := range tokens {
		if token.Pos() == "名詞" {
			fv = append(fv, prefix+":"+token.Surface)
		}
	}
	return fv
}

func ExtractFeatures(e Example) FeatureVector {
	var fv FeatureVector
	fv = append(fv, "BIAS")

	html := strings.ToLower(strings.Replace(e.RawHTML, " ", "", -1))
	if !utf8.ValidString(html) {
		return fv
	}

	if !utf8.ValidString(e.Title) {
		return fv
	}
	fv = append(fv, "VALID_TITLE")
	fv = append(fv, ExtractNounFeatures(e.Title, "TITLE")...)

	if !utf8.ValidString(e.Description) {
		return fv
	}
	fv = append(fv, "VALID_DESCRIPTION")
	fv = append(fv, ExtractNounFeatures(e.Description, "DESCRIPTION")...)

	if !utf8.ValidString(e.Body) {
		return fv
	}
	fv = append(fv, "VALID_TITLE")
	fv = append(fv, ExtractNounFeatures(e.Body, "BODY")...)

	return fv
}
<commit_msg>kagome側で文字コードの対応がされたので、ValidStringはしなくてよい<commit_after>package main

import (
	"github.com/ikawaha/kagome/tokenizer"
)

type FeatureVector []string

func ExtractNounFeatures(s string, prefix string) FeatureVector {
	var fv FeatureVector
	if s == "" {
		return fv
	}
	t := tokenizer.New()
	tokens := t.Tokenize(s)
	for _, token := range tokens {
		if token.Pos() == "名詞" {
			fv = append(fv, prefix+":"+token.Surface)
		}
	}
	return fv
}

func ExtractFeatures(e Example) FeatureVector {
	var fv FeatureVector
	fv = append(fv, "BIAS")
	fv = append(fv, ExtractNounFeatures(e.Title, "TITLE")...)
	fv = append(fv, ExtractNounFeatures(e.Description, "DESCRIPTION")...)
	fv = append(fv, ExtractNounFeatures(e.Body, "BODY")...)
	return fv
}
<|endoftext|>


<commit_before>package cmd

// Version defines the current Pop version.
const Version = "development"
<commit_msg>version bump: v4.9.4-rc.2<commit_after>package cmd

// Version defines the current Pop version.
const Version = "v4.9.4-rc.2"
<|endoftext|>