<commit_before>package integration_test

import (
	"github.com/APTrust/exchange/context"
	"github.com/APTrust/exchange/dpn/network"
	"github.com/APTrust/exchange/models"
	"github.com/APTrust/exchange/util/testutil"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"net/url"
	"path/filepath"
	"strings"
	"testing"
)

// getObjectIdentifiers returns the identifiers of objects that
// apps/test_push_to_dpn marks as "Push to DPN". These objects
// should wind up in the dpn_copy queue.
func identifiersPushedToDPN() ([]string) {
	identifiers := make([]string, 7)
	for index, s3Key := range testutil.INTEGRATION_GOOD_BAGS[0:7] {
		identifier := strings.Replace(s3Key, "aptrust.receiving.test.", "", 1)
		identifier = strings.Replace(identifier, ".tar", "", 1)
		identifiers[index] = identifier
	}
	return identifiers
}

func getContext(t *testing.T) (*context.Context) {
	configFile := filepath.Join("config", "integration.json")
	config, err := models.LoadConfigFile(configFile)
	require.Nil(t, err)
	config.ExpandFilePaths()
	return context.NewContext(config)
}

// We should have created one WorkItem for each DPN ingest request.
func TestWorkItemsCreatedAndQueued(t *testing.T) {
	if !testutil.ShouldRunIntegrationTests() {
		t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.")
	}
	expectedIdentifiers := identifiersPushedToDPN()
	_context := getContext(t)
	params := url.Values{}
	params.Set("item_action", "DPN")
	params.Set("page", "1")
	params.Set("per_page", "100")
	resp := _context.PharosClient.WorkItemList(params)
	require.Nil(t, resp.Error)
	assert.Equal(t, len(expectedIdentifiers), resp.Count)
	for _, workItem := range resp.WorkItems() {
		found := false
		queued := false
		currentIdentifier := ""
		for _, identifier := range expectedIdentifiers {
			currentIdentifier = identifier
			if workItem.ObjectIdentifier == identifier {
				found = true
				if workItem.QueuedAt != nil && !workItem.QueuedAt.IsZero() {
					queued = true
				}
				break
			}
		}
		assert.True(t, found, "No WorkItem for object %s", currentIdentifier)
		assert.True(t, queued, "Object %s was not queued", currentIdentifier)
	}

	// In addition to checking whether Pharos thinks the items are queued,
	// let's ask NSQ as well.
	stats, err := _context.NSQClient.GetStats()
	require.Nil(t, err)
	for _, topic := range stats.Data.Topics {
		if topic.TopicName == "fetch_topic" {
			// We fetch 16 bags in our integration tests.
			// They're not all valid, but we should have that many in the queue.
			assert.EqualValues(t, uint64(16), topic.MessageCount)
		} else if topic.TopicName == "store_topic" {
			// All of the 11 valid bags should have made it into the store topic.
			assert.EqualValues(t, uint64(11), topic.MessageCount)
		} else if topic.TopicName == "record_topic" {
			// All of the 11 valid bags should have made it into the record topic.
			assert.EqualValues(t, uint64(11), topic.MessageCount)
		}
	}
}

// We should have created one DPNWorkItem for each replication request
// that we synched from the other nodes.
func TestDPNWorkItemsCreatedAndQueued(t *testing.T) {
	if !testutil.ShouldRunIntegrationTests() {
		t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.")
	}
	// Use local DPN REST client to check for all replication requests
	// where ToNode is our LocalNode.
	//
	// Then check Pharos for a DPNWorkItem for each of these replications.
	// The DPNWorkItem should exist, and should have a QueuedAt timestamp.
	_context := getContext(t)
	dpnClient, err := network.NewDPNRestClient(
		_context.Config.DPN.RestClient.LocalServiceURL,
		_context.Config.DPN.RestClient.LocalAPIRoot,
		_context.Config.DPN.RestClient.LocalAuthToken,
		_context.Config.DPN.LocalNode,
		_context.Config.DPN)
	require.Nil(t, err)
	xferParams := url.Values{}
	xferParams.Set("to_node", _context.Config.DPN.LocalNode)
	dpnResp := dpnClient.ReplicationTransferList(xferParams)
	require.Nil(t, dpnResp.Error)
	for _, xfer := range dpnResp.ReplicationTransfers() {
		params := url.Values{}
		params.Set("identifier", xfer.ReplicationId)
		params.Set("task", "replication")
		pharosResp := _context.PharosClient.DPNWorkItemList(params)
		require.Nil(t, pharosResp.Error)
		require.Equal(t, 1, pharosResp.Count)
		dpnWorkItem := pharosResp.DPNWorkItem()
		require.NotNil(t, dpnWorkItem.QueuedAt)
		assert.False(t, dpnWorkItem.QueuedAt.IsZero())
	}
}
<commit_msg>Check NSQ for DPN ingest items<commit_after>package integration_test

import (
	"github.com/APTrust/exchange/context"
	"github.com/APTrust/exchange/dpn/network"
	"github.com/APTrust/exchange/models"
	"github.com/APTrust/exchange/util/testutil"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"net/url"
	"path/filepath"
	"strings"
	"testing"
)

// getObjectIdentifiers returns the identifiers of objects that
// apps/test_push_to_dpn marks as "Push to DPN". These objects
// should wind up in the dpn_copy queue.
func identifiersPushedToDPN() ([]string) {
	identifiers := make([]string, 7)
	for index, s3Key := range testutil.INTEGRATION_GOOD_BAGS[0:7] {
		identifier := strings.Replace(s3Key, "aptrust.receiving.test.", "", 1)
		identifier = strings.Replace(identifier, ".tar", "", 1)
		identifiers[index] = identifier
	}
	return identifiers
}

func getContext(t *testing.T) (*context.Context) {
	configFile := filepath.Join("config", "integration.json")
	config, err := models.LoadConfigFile(configFile)
	require.Nil(t, err)
	config.ExpandFilePaths()
	return context.NewContext(config)
}

// We should have created one WorkItem for each DPN ingest request.
func TestWorkItemsCreatedAndQueued(t *testing.T) {
	if !testutil.ShouldRunIntegrationTests() {
		t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.")
	}
	expectedIdentifiers := identifiersPushedToDPN()
	_context := getContext(t)
	params := url.Values{}
	params.Set("item_action", "DPN")
	params.Set("page", "1")
	params.Set("per_page", "100")
	resp := _context.PharosClient.WorkItemList(params)
	require.Nil(t, resp.Error)
	assert.Equal(t, len(expectedIdentifiers), resp.Count)
	for _, workItem := range resp.WorkItems() {
		found := false
		queued := false
		currentIdentifier := ""
		for _, identifier := range expectedIdentifiers {
			currentIdentifier = identifier
			if workItem.ObjectIdentifier == identifier {
				found = true
				if workItem.QueuedAt != nil && !workItem.QueuedAt.IsZero() {
					queued = true
				}
				break
			}
		}
		assert.True(t, found, "No WorkItem for object %s", currentIdentifier)
		assert.True(t, queued, "Object %s was not queued", currentIdentifier)
	}

	// In addition to checking whether Pharos thinks the items are queued,
	// let's ask NSQ as well.
	stats, err := _context.NSQClient.GetStats()
	require.Nil(t, err)
	for _, topic := range stats.Data.Topics {
		if topic.TopicName == "fetch_topic" {
			// We fetch 16 bags in our integration tests.
			// They're not all valid, but we should have that many in the queue.
			assert.EqualValues(t, uint64(16), topic.MessageCount)
		} else if topic.TopicName == "store_topic" {
			// All of the 11 valid bags should have made it into the store topic.
			assert.EqualValues(t, uint64(11), topic.MessageCount)
		} else if topic.TopicName == "record_topic" {
			// All of the 11 valid bags should have made it into the record topic.
			assert.EqualValues(t, uint64(11), topic.MessageCount)
		}
	}
}

// We should have created one DPNWorkItem for each replication request
// that we synched from the other nodes.
func TestDPNWorkItemsCreatedAndQueued(t *testing.T) {
	if !testutil.ShouldRunIntegrationTests() {
		t.Skip("Skipping integration test. Set ENV var RUN_EXCHANGE_INTEGRATION=true if you want to run them.")
	}
	// Use local DPN REST client to check for all replication requests
	// where ToNode is our LocalNode.
	//
	// Then check Pharos for a DPNWorkItem for each of these replications.
	// The DPNWorkItem should exist, and should have a QueuedAt timestamp.
	_context := getContext(t)
	dpnClient, err := network.NewDPNRestClient(
		_context.Config.DPN.RestClient.LocalServiceURL,
		_context.Config.DPN.RestClient.LocalAPIRoot,
		_context.Config.DPN.RestClient.LocalAuthToken,
		_context.Config.DPN.LocalNode,
		_context.Config.DPN)
	require.Nil(t, err)
	xferParams := url.Values{}
	xferParams.Set("to_node", _context.Config.DPN.LocalNode)
	dpnResp := dpnClient.ReplicationTransferList(xferParams)
	require.Nil(t, dpnResp.Error)
	for _, xfer := range dpnResp.ReplicationTransfers() {
		params := url.Values{}
		params.Set("identifier", xfer.ReplicationId)
		params.Set("task", "replication")
		pharosResp := _context.PharosClient.DPNWorkItemList(params)
		require.Nil(t, pharosResp.Error)
		require.Equal(t, 1, pharosResp.Count)
		dpnWorkItem := pharosResp.DPNWorkItem()
		require.NotNil(t, dpnWorkItem.QueuedAt)
		assert.False(t, dpnWorkItem.QueuedAt.IsZero())
	}

	// Check NSQ as well.
	stats, err := _context.NSQClient.GetStats()
	require.Nil(t, err)
	for _, topic := range stats.Data.Topics {
		if topic.TopicName == "dpn_package_topic" {
			// apps/test_push_to_dpn.go requests that items
			// testutil.INTEGRATION_GOOD_BAGS[0:7] be sent to DPN,
			// so we should find seven items in the package queue
			assert.EqualValues(t, uint64(7), topic.MessageCount)
		}
	}
}
<|endoftext|>


<commit_before>package drive

import (
    "io"
    "io/ioutil"
    "fmt"
    "time"
)

func getProgressReader(r io.Reader, w io.Writer, size int64) io.Reader {
    if w == ioutil.Discard || size < 1024 * 1024 {
        return r
    }

    return &Progress{
        Reader: r,
        Writer: w,
        Size: size,
    }
}

type Progress struct {
    Writer io.Writer
    Reader io.Reader
    Size int64
    progress int64
    rate int64
    rateProgress int64
    rateUpdated time.Time
    updated time.Time
    done bool
}

func (self *Progress) Read(p []byte) (int, error) {
    // Read
    n, err := self.Reader.Read(p)

    now := time.Now()
    isLast := err != nil

    // Increment progress
    newProgress := self.progress + int64(n)
    self.progress = newProgress

    // Initialize rate state
    if self.rateUpdated.IsZero() {
        self.rateUpdated = now
        self.rateProgress = newProgress
    }

    // Update rate every 3 seconds
    if self.rateUpdated.Add(time.Second * 3).Before(now) {
        self.rate = calcRate(newProgress - self.rateProgress, self.rateUpdated, now)
        self.rateUpdated = now
        self.rateProgress = newProgress
    }

    // Draw progress every second
    if self.updated.Add(time.Second).Before(now) || isLast {
        self.Draw(isLast)
    }

    // Update last draw time
    self.updated = now

    // Mark as done if error occurs
    self.done = isLast

    return n, err
}

func (self *Progress) Draw(isLast bool) {
    if self.done {
        return
    }

    // Clear line
    fmt.Fprintf(self.Writer, "\r%50s", "")

    // Print progress
    fmt.Fprintf(self.Writer, "\r%s/%s", formatSize(self.progress, false), formatSize(self.Size, false))

    // Print rate
    if self.rate > 0 {
        fmt.Fprintf(self.Writer, ", Rate: %s/s", formatSize(self.rate, false))
    }

    if isLast {
        fmt.Fprintf(self.Writer, "\n")
    }
}
<commit_msg>Update rate every 5 seconds<commit_after>package drive

import (
    "io"
    "io/ioutil"
    "fmt"
    "time"
)

const MaxDrawInterval = time.Second * 1
const MaxRateInterval = time.Second * 5

func getProgressReader(r io.Reader, w io.Writer, size int64) io.Reader {
    // Don't wrap reader if output is discarded or size is too small
    if w == ioutil.Discard || size < 1024 * 1024 {
        return r
    }

    return &Progress{
        Reader: r,
        Writer: w,
        Size: size,
    }
}

type Progress struct {
    Writer io.Writer
    Reader io.Reader
    Size int64
    progress int64
    rate int64
    rateProgress int64
    rateUpdated time.Time
    updated time.Time
    done bool
}

func (self *Progress) Read(p []byte) (int, error) {
    // Read
    n, err := self.Reader.Read(p)

    now := time.Now()
    isLast := err != nil

    // Increment progress
    newProgress := self.progress + int64(n)
    self.progress = newProgress

    // Initialize rate state
    if self.rateUpdated.IsZero() {
        self.rateUpdated = now
        self.rateProgress = newProgress
    }

    // Update rate every x seconds
    if self.rateUpdated.Add(MaxRateInterval).Before(now) {
        self.rate = calcRate(newProgress - self.rateProgress, self.rateUpdated, now)
        self.rateUpdated = now
        self.rateProgress = newProgress
    }

    // Draw progress every x seconds
    if self.updated.Add(MaxDrawInterval).Before(now) || isLast {
        self.Draw(isLast)
    }

    // Update last draw time
    self.updated = now

    // Mark as done if error occurs
    self.done = isLast

    return n, err
}

func (self *Progress) Draw(isLast bool) {
    if self.done {
        return
    }

    // Clear line
    fmt.Fprintf(self.Writer, "\r%50s", "")

    // Print progress
    fmt.Fprintf(self.Writer, "\r%s/%s", formatSize(self.progress, false), formatSize(self.Size, false))

    // Print rate
    if self.rate > 0 {
        fmt.Fprintf(self.Writer, ", Rate: %s/s", formatSize(self.rate, false))
    }

    if isLast {
        fmt.Fprintf(self.Writer, "\n")
    }
}
<|endoftext|>


<commit_before>package eval

import (
	"time"

	"github.com/abrander/gansoi/checks"
	"github.com/abrander/gansoi/database"
	"github.com/abrander/gansoi/logger"
	"github.com/abrander/gansoi/node"
	"github.com/hashicorp/raft"
)

type (
	// Evaluator will evaluate check results from all nodes on the leader node.
	Evaluator struct {
		node  *node.Node
		peers raft.PeerStore
	}
)

// NewEvaluator will instantiate a new Evaluator listening to cluster changes,
// and evaluating results as they arrive.
func NewEvaluator(n *node.Node, peers raft.PeerStore) *Evaluator {
	e := &Evaluator{
		node:  n,
		peers: peers,
	}

	n.RegisterListener(e)

	return e
}

// evaluate1 will evalute a check result from a single node.
func (e *Evaluator) evaluate1(checkResult *checks.CheckResult) error {
	pe := PartialEvaluation{}
	pe.ID = checkResult.CheckID + ":::" + checkResult.Node

	state := StateDown

	// Evaluate if the check went well for a single node. For now we simply
	// checks for empty error string and assume everything is good if its
	// empty :)
	if checkResult.Error == "" {
		state = StateUp
	}

	err := e.node.One("ID", pe.ID, &pe)
	if err != nil {
		// None was found. Fill out new.
		pe.CheckID = checkResult.CheckID
		pe.NodeID = checkResult.Node
		pe.Start = checkResult.TimeStamp
		pe.End = checkResult.TimeStamp
		pe.State = state
	} else {
		// Check if the state changed.
		if pe.State == state {
			// If the state is the same, just update end time.
			pe.End = checkResult.TimeStamp
		} else {
			// If the state changed, reset both start and end time.
			pe.Start = checkResult.TimeStamp
			pe.End = checkResult.TimeStamp
		}

		pe.State = state
	}

	err = e.node.Save(&pe)

	return err
}

// evaluate2 will evalute if a given check should be considered up or down when
// evaluating the result from all nodes.
// This should only be done on the leader.
func (e *Evaluator) evaluate2(n *PartialEvaluation) error {
	var eval Evaluation

	// FIXME: Locking.

	err := e.node.One("CheckID", n.CheckID, &eval)
	if err != nil {
		// Evaluation is unknown. Start new.
		eval.CheckID = n.CheckID
		eval.Start = n.Start
		eval.End = n.End
	}

	state := StateUp
	states := make(map[State]int)

	nodes, _ := e.peers.Peers()
	for _, nodeID := range nodes {
		ID := n.CheckID + ":::" + nodeID

		var pe PartialEvaluation
		err = e.node.One("ID", ID, &pe)
		if err != nil {
			// Not all nodes have reported yet. Could be StateDegraded instead?
			state = StateUnknown
			break
		}

		// FIXME: Deal with old checks somehow. Maybe we should simply discard
		//        them? I'm not sure it makes much sense to evalute a period
		//        where gansoi were not running for example.

		states[pe.State]++
	}

	if state == StateUp {
		if states[StateUp] == len(nodes) {
			state = StateUp
		} else if states[StateDown] == len(nodes) {
			state = StateDown
		} else {
			state = StateDegraded
		}
	}

	if eval.State == state {
		// There is no change in state. Keep current start time, and update end
		// time and cyclecount.
		eval.End = time.Now()
		eval.Cycles++
	} else {
		// We have a new state. Update both start and end time.
		eval.Start = time.Now()
		eval.End = eval.Start
		eval.Cycles = 0
		eval.PreviousState = eval.State
	}

	eval.State = state

	return e.node.Save(&eval)
}

// PostClusterApply implements node.Listener.
func (e *Evaluator) PostClusterApply(leader bool, command database.Command, data interface{}, err error) {
	// If we're not the leader, we abort. Only the leader should evaluate
	// check results.
	if !leader {
		return
	}

	// We're only interested in saves for now.
	if command != database.CommandSave {
		return
	}

	switch data.(type) {
	case *checks.CheckResult:
		e.evaluate1(data.(*checks.CheckResult))
	case *PartialEvaluation:
		e.evaluate2(data.(*PartialEvaluation))
	case *Evaluation:
		eval := data.(*Evaluation)
		logger.Green("eval", "%s: %s (%s)", eval.CheckID, eval.State.ColorString(), eval.End.Sub(eval.Start).String())
	}
}
<commit_msg>Fixed import order.<commit_after>package eval

import (
	"time"

	"github.com/hashicorp/raft"

	"github.com/abrander/gansoi/checks"
	"github.com/abrander/gansoi/database"
	"github.com/abrander/gansoi/logger"
	"github.com/abrander/gansoi/node"
)

type (
	// Evaluator will evaluate check results from all nodes on the leader node.
	Evaluator struct {
		node  *node.Node
		peers raft.PeerStore
	}
)

// NewEvaluator will instantiate a new Evaluator listening to cluster changes,
// and evaluating results as they arrive.
func NewEvaluator(n *node.Node, peers raft.PeerStore) *Evaluator {
	e := &Evaluator{
		node:  n,
		peers: peers,
	}

	n.RegisterListener(e)

	return e
}

// evaluate1 will evalute a check result from a single node.
func (e *Evaluator) evaluate1(checkResult *checks.CheckResult) error {
	pe := PartialEvaluation{}
	pe.ID = checkResult.CheckID + ":::" + checkResult.Node

	state := StateDown

	// Evaluate if the check went well for a single node. For now we simply
	// checks for empty error string and assume everything is good if its
	// empty :)
	if checkResult.Error == "" {
		state = StateUp
	}

	err := e.node.One("ID", pe.ID, &pe)
	if err != nil {
		// None was found. Fill out new.
		pe.CheckID = checkResult.CheckID
		pe.NodeID = checkResult.Node
		pe.Start = checkResult.TimeStamp
		pe.End = checkResult.TimeStamp
		pe.State = state
	} else {
		// Check if the state changed.
		if pe.State == state {
			// If the state is the same, just update end time.
			pe.End = checkResult.TimeStamp
		} else {
			// If the state changed, reset both start and end time.
			pe.Start = checkResult.TimeStamp
			pe.End = checkResult.TimeStamp
		}

		pe.State = state
	}

	err = e.node.Save(&pe)

	return err
}

// evaluate2 will evalute if a given check should be considered up or down when
// evaluating the result from all nodes.
// This should only be done on the leader.
func (e *Evaluator) evaluate2(n *PartialEvaluation) error {
	var eval Evaluation

	// FIXME: Locking.

	err := e.node.One("CheckID", n.CheckID, &eval)
	if err != nil {
		// Evaluation is unknown. Start new.
		eval.CheckID = n.CheckID
		eval.Start = n.Start
		eval.End = n.End
	}

	state := StateUp
	states := make(map[State]int)

	nodes, _ := e.peers.Peers()
	for _, nodeID := range nodes {
		ID := n.CheckID + ":::" + nodeID

		var pe PartialEvaluation
		err = e.node.One("ID", ID, &pe)
		if err != nil {
			// Not all nodes have reported yet. Could be StateDegraded instead?
			state = StateUnknown
			break
		}

		// FIXME: Deal with old checks somehow. Maybe we should simply discard
		//        them? I'm not sure it makes much sense to evalute a period
		//        where gansoi were not running for example.

		states[pe.State]++
	}

	if state == StateUp {
		if states[StateUp] == len(nodes) {
			state = StateUp
		} else if states[StateDown] == len(nodes) {
			state = StateDown
		} else {
			state = StateDegraded
		}
	}

	if eval.State == state {
		// There is no change in state. Keep current start time, and update end
		// time and cyclecount.
		eval.End = time.Now()
		eval.Cycles++
	} else {
		// We have a new state. Update both start and end time.
		eval.Start = time.Now()
		eval.End = eval.Start
		eval.Cycles = 0
		eval.PreviousState = eval.State
	}

	eval.State = state

	return e.node.Save(&eval)
}

// PostClusterApply implements node.Listener.
func (e *Evaluator) PostClusterApply(leader bool, command database.Command, data interface{}, err error) {
	// If we're not the leader, we abort. Only the leader should evaluate
	// check results.
	if !leader {
		return
	}

	// We're only interested in saves for now.
	if command != database.CommandSave {
		return
	}

	switch data.(type) {
	case *checks.CheckResult:
		e.evaluate1(data.(*checks.CheckResult))
	case *PartialEvaluation:
		e.evaluate2(data.(*PartialEvaluation))
	case *Evaluation:
		eval := data.(*Evaluation)
		logger.Green("eval", "%s: %s (%s)", eval.CheckID, eval.State.ColorString(), eval.End.Sub(eval.Start).String())
	}
}
<|endoftext|>


<commit_before>// +build windows

package speakeasy

import (
	"os"
	"syscall"
)

// SetConsoleMode function can be used to change value of ENABLE_ECHO_INPUT:
// http://msdn.microsoft.com/en-us/library/windows/desktop/ms686033(v=vs.85).aspx
const ENABLE_ECHO_INPUT = 0x0004

func getPassword() (password string, err error) {
	stdinHandler := syscall.Handle(os.Stdin.Fd())
	var oldMode uint32

	err = syscall.GetConsoleMode(stdinHandler, &oldMode)
	if err != nil {
		return
	}

	var newMode uint32 = (oldMode &^ ENABLE_ECHO_INPUT)

	err = setConsoleMode(hStdin, newMode)
	defer setConsoleMode(hStdin, oldMode)
	if err != nil {
		return
	}

	return readline()
}

func setConsoleMode(console syscall.Handle, mode uint32) (err error) {
	dll := syscall.MustLoadDLL("kernel32")
	proc := dll.MustFindProc("SetConsoleMode")
	r, _, err := proc.Call(uintptr(console), uintptr(mode))

	if r == 0 {
		return err
	}
	return nil
}
<commit_msg>windows typo fix<commit_after>// +build windows

package speakeasy

import (
	"os"
	"syscall"
)

// SetConsoleMode function can be used to change value of ENABLE_ECHO_INPUT:
// http://msdn.microsoft.com/en-us/library/windows/desktop/ms686033(v=vs.85).aspx
const ENABLE_ECHO_INPUT = 0x0004

func getPassword() (password string, err error) {
	hStdin := syscall.Handle(os.Stdin.Fd())
	var oldMode uint32

	err = syscall.GetConsoleMode(hStdin, &oldMode)
	if err != nil {
		return
	}

	var newMode uint32 = (oldMode &^ ENABLE_ECHO_INPUT)

	err = setConsoleMode(hStdin, newMode)
	defer setConsoleMode(hStdin, oldMode)
	if err != nil {
		return
	}

	return readline()
}

func setConsoleMode(console syscall.Handle, mode uint32) (err error) {
	dll := syscall.MustLoadDLL("kernel32")
	proc := dll.MustFindProc("SetConsoleMode")
	r, _, err := proc.Call(uintptr(console), uintptr(mode))

	if r == 0 {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package api

import (
	campSender "github.com/supme/gonder/campaign"
	"github.com/supme/gonder/models"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"text/template"
)

func getMailPreview(w http.ResponseWriter, r *http.Request) {
	if user.Right("get-recipients") {
		recipient, err := campSender.GetRecipient(r.FormValue("id"))
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
		}
		if user.CampaignRight(recipient.CampaignID) {
			var tmplFunc func(io.Writer) error
			if r.FormValue("type") != "web" {
				tmplFunc = recipient.WebHTML(false, true)
			} else {
				tmplFunc = recipient.WebHTML(true, true)
			}
			w.Header().Set("Content-Type", "text/html")
			err = tmplFunc(w)
			if err != nil {
				apilog.Println(err)
			}

		} else {
			http.Error(w, "Forbidden get recipients from this campaign", http.StatusForbidden)
		}
	} else {
		http.Error(w, "Forbidden get recipients", http.StatusForbidden)
	}
}

func getUnsubscribePreview(w http.ResponseWriter, r *http.Request) {
	if user.Right("get-recipients") && user.CampaignRight(r.FormValue("campaignId")) {

		var tmpl string
		var content []byte
		var err error
		models.Db.QueryRow("SELECT `group`.`template` FROM `campaign` INNER JOIN `group` ON `campaign`.`group_id`=`group`.`id` WHERE `group`.`template` IS NOT NULL AND `campaign`.`id`=?", r.FormValue("id")).Scan(&tmpl)
		if tmpl == "" {
			tmpl = "default"
		} else {
			if _, err = os.Stat(models.FromRootDir("templates/" + tmpl + "/accept.html")); err != nil {
				tmpl = "default"
			}
			if _, err = os.Stat(models.FromRootDir("templates/" + tmpl + "/success.html")); err != nil {
				tmpl = "default"
			}
		}

		if r.Method == "GET" {
			content, _ = ioutil.ReadFile(models.FromRootDir("templates/" + tmpl + "/accept.html"))
		} else {
			content, _ = ioutil.ReadFile(models.FromRootDir("templates/" + tmpl + "/success.html"))
		}

		t := template.New("unsubscribe")
		t, err = t.Parse(string(content))
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
		err = t.Execute(w, map[string]string{"campaignId": r.FormValue("campaignId")})
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}

	} else {
		http.Error(w, "Forbidden get from this campaign", http.StatusForbidden)
	}

}
<commit_msg>add todo<commit_after>package api

import (
	campSender "github.com/supme/gonder/campaign"
	"github.com/supme/gonder/models"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"text/template"
)

func getMailPreview(w http.ResponseWriter, r *http.Request) {
	if user.Right("get-recipients") {
		recipient, err := campSender.GetRecipient(r.FormValue("id"))
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
		}
		if user.CampaignRight(recipient.CampaignID) {
			var tmplFunc func(io.Writer) error
			if r.FormValue("type") != "web" {
				tmplFunc = recipient.WebHTML(false, true)
			} else {
				tmplFunc = recipient.WebHTML(true, true)
			}
			w.Header().Set("Content-Type", "text/html")
			err = tmplFunc(w)
			if err != nil {
				apilog.Println(err)
			}

		} else {
			http.Error(w, "Forbidden get recipients from this campaign", http.StatusForbidden)
		}
	} else {
		http.Error(w, "Forbidden get recipients", http.StatusForbidden)
	}
}

// ToDo fix get right template then move unsubscribe template from models
func getUnsubscribePreview(w http.ResponseWriter, r *http.Request) {
	if user.Right("get-recipients") && user.CampaignRight(r.FormValue("campaignId")) {

		var tmpl string
		var content []byte
		var err error
		models.Db.QueryRow("SELECT `group`.`template` FROM `campaign` INNER JOIN `group` ON `campaign`.`group_id`=`group`.`id` WHERE `group`.`template` IS NOT NULL AND `campaign`.`id`=?", r.FormValue("id")).Scan(&tmpl)
		if tmpl == "" {
			tmpl = "default"
		} else {
			if _, err = os.Stat(models.FromRootDir("templates/" + tmpl + "/accept.html")); err != nil {
				tmpl = "default"
			}
			if _, err = os.Stat(models.FromRootDir("templates/" + tmpl + "/success.html")); err != nil {
				tmpl = "default"
			}
		}

		if r.Method == "GET" {
			content, _ = ioutil.ReadFile(models.FromRootDir("templates/" + tmpl + "/accept.html"))
		} else {
			content, _ = ioutil.ReadFile(models.FromRootDir("templates/" + tmpl + "/success.html"))
		}

		t := template.New("unsubscribe")
		t, err = t.Parse(string(content))
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
		err = t.Execute(w, map[string]string{"campaignId": r.FormValue("campaignId")})
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}

	} else {
		http.Error(w, "Forbidden get from this campaign", http.StatusForbidden)
	}

}
<|endoftext|>


<commit_before>// Copyright 2019 gf Author(https://github.com/gogf/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://github.com/gogf/gf.

package gins

import (
	"fmt"
	"github.com/gogf/gf/internal/intlog"
	"github.com/gogf/gf/text/gstr"
	"github.com/gogf/gf/util/gutil"

	"github.com/gogf/gf/database/gdb"
	"github.com/gogf/gf/text/gregex"
	"github.com/gogf/gf/util/gconv"
)

const (
	gFRAME_CORE_COMPONENT_NAME_DATABASE = "gf.core.component.database"
	gDATABASE_NODE_NAME                 = "database"
)

// Database returns an instance of database ORM object
// with specified configuration group name.
func Database(name ...string) gdb.DB {
	group := gdb.DEFAULT_GROUP_NAME
	if len(name) > 0 && name[0] != "" {
		group = name[0]
	}
	instanceKey := fmt.Sprintf("%s.%s", gFRAME_CORE_COMPONENT_NAME_DATABASE, group)
	db := instances.GetOrSetFuncLock(instanceKey, func() interface{} {
		var m map[string]interface{}
		// It firstly searches the configuration of the instance name.
		nodeKey, _ := gutil.MapPossibleItemByKey(Config().GetMap("."), gDATABASE_NODE_NAME)
		if nodeKey == "" {
			nodeKey = gDATABASE_NODE_NAME
		}
		if m = Config().GetMap(nodeKey); len(m) == 0 {
			panic(fmt.Sprintf(`database init failed: "%s" node not found, is config file or configuration missing?`, gDATABASE_NODE_NAME))
		}
		// Parse <m> as map-slice and adds it to gdb's global configurations.
		for group, groupConfig := range m {
			cg := gdb.ConfigGroup{}
			switch value := groupConfig.(type) {
			case []interface{}:
				for _, v := range value {
					if node := parseDBConfigNode(v); node != nil {
						cg = append(cg, *node)
					}
				}
			case map[string]interface{}:
				if node := parseDBConfigNode(value); node != nil {
					cg = append(cg, *node)
				}
			}
			if len(cg) > 0 {
				intlog.Printf("%s, %#v", group, cg)
				gdb.SetConfigGroup(group, cg)
			}
		}
		// Parse <m> as a single node configuration,
		// which is the default group configuration.
		if node := parseDBConfigNode(m); node != nil {
			cg := gdb.ConfigGroup{}
			if node.LinkInfo != "" || node.Host != "" {
				cg = append(cg, *node)
			}
			if len(cg) > 0 {
				intlog.Printf("%s, %#v", gdb.DEFAULT_GROUP_NAME, cg)
				gdb.SetConfigGroup(gdb.DEFAULT_GROUP_NAME, cg)
			}
		}

		if db, err := gdb.New(name...); err == nil {
			// Initialize logger for ORM.
			var m map[string]interface{}
			m = Config().GetMap(fmt.Sprintf("%s.%s", nodeKey, gLOGGER_NODE_NAME))
			if len(m) == 0 {
				m = Config().GetMap(nodeKey)
			}
			if len(m) > 0 {
				if err := db.GetLogger().SetConfigWithMap(m); err != nil {
					panic(err)
				}
			}
			return db
		} else {
			panic(err)
		}
		return nil
	})
	if db != nil {
		return db.(gdb.DB)
	}
	return nil
}

func parseDBConfigNode(value interface{}) *gdb.ConfigNode {
	nodeMap, ok := value.(map[string]interface{})
	if !ok {
		return nil
	}
	node := &gdb.ConfigNode{}
	err := gconv.Struct(nodeMap, node)
	if err != nil {
		panic(err)
	}
	if _, v := gutil.MapPossibleItemByKey(nodeMap, "link"); v != nil {
		node.LinkInfo = gconv.String(v)
	}
	// Parse link syntax.
	if node.LinkInfo != "" && node.Type == "" {
		match, _ := gregex.MatchString(`([a-z]+):(.+)`, node.LinkInfo)
		if len(match) == 3 {
			node.Type = gstr.Trim(match[1])
			node.LinkInfo = gstr.Trim(match[2])
		}
	}
	return node
}
<commit_msg>fix issue in database configuration for package gind<commit_after>// Copyright 2019 gf Author(https://github.com/gogf/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://github.com/gogf/gf.

package gins

import (
	"fmt"
	"github.com/gogf/gf/internal/intlog"
	"github.com/gogf/gf/text/gstr"
	"github.com/gogf/gf/util/gutil"

	"github.com/gogf/gf/database/gdb"
	"github.com/gogf/gf/text/gregex"
	"github.com/gogf/gf/util/gconv"
)

const (
	gFRAME_CORE_COMPONENT_NAME_DATABASE = "gf.core.component.database"
	gDATABASE_NODE_NAME                 = "database"
)

// Database returns an instance of database ORM object
// with specified configuration group name.
func Database(name ...string) gdb.DB {
	group := gdb.DEFAULT_GROUP_NAME
	if len(name) > 0 && name[0] != "" {
		group = name[0]
	}
	instanceKey := fmt.Sprintf("%s.%s", gFRAME_CORE_COMPONENT_NAME_DATABASE, group)
	db := instances.GetOrSetFuncLock(instanceKey, func() interface{} {
		// If configuration file does not exist but the gdb configurations are already set.
		if !Config().Available() && gdb.GetConfig(group) != nil {
			db, err := gdb.Instance(group)
			if err != nil {
				panic(err)
			}
			return db
		}
		// Check the configuration file.
		var m map[string]interface{}
		// It firstly searches the configuration of the instance name.
		nodeKey, _ := gutil.MapPossibleItemByKey(Config().GetMap("."), gDATABASE_NODE_NAME)
		if nodeKey == "" {
			nodeKey = gDATABASE_NODE_NAME
		}
		if m = Config().GetMap(nodeKey); len(m) == 0 {
			panic(fmt.Sprintf(`database init failed: "%s" node not found, is config file or configuration missing?`, gDATABASE_NODE_NAME))
		}
		// Parse <m> as map-slice and adds it to gdb's global configurations.
		for group, groupConfig := range m {
			cg := gdb.ConfigGroup{}
			switch value := groupConfig.(type) {
			case []interface{}:
				for _, v := range value {
					if node := parseDBConfigNode(v); node != nil {
						cg = append(cg, *node)
					}
				}
			case map[string]interface{}:
				if node := parseDBConfigNode(value); node != nil {
					cg = append(cg, *node)
				}
			}
			if len(cg) > 0 {
				intlog.Printf("%s, %#v", group, cg)
				gdb.SetConfigGroup(group, cg)
			}
		}
		// Parse <m> as a single node configuration,
		// which is the default group configuration.
		if node := parseDBConfigNode(m); node != nil {
			cg := gdb.ConfigGroup{}
			if node.LinkInfo != "" || node.Host != "" {
				cg = append(cg, *node)
			}
			if len(cg) > 0 {
				intlog.Printf("%s, %#v", gdb.DEFAULT_GROUP_NAME, cg)
				gdb.SetConfigGroup(gdb.DEFAULT_GROUP_NAME, cg)
			}
		}

		if db, err := gdb.New(name...); err == nil {
			// Initialize logger for ORM.
			var m map[string]interface{}
			m = Config().GetMap(fmt.Sprintf("%s.%s", nodeKey, gLOGGER_NODE_NAME))
			if len(m) == 0 {
				m = Config().GetMap(nodeKey)
			}
			if len(m) > 0 {
				if err := db.GetLogger().SetConfigWithMap(m); err != nil {
					panic(err)
				}
			}
			return db
		} else {
			panic(err)
		}
		return nil
	})
	if db != nil {
		return db.(gdb.DB)
	}
	return nil
}

func parseDBConfigNode(value interface{}) *gdb.ConfigNode {
	nodeMap, ok := value.(map[string]interface{})
	if !ok {
		return nil
	}
	node := &gdb.ConfigNode{}
	err := gconv.Struct(nodeMap, node)
	if err != nil {
		panic(err)
	}
	if _, v := gutil.MapPossibleItemByKey(nodeMap, "link"); v != nil {
		node.LinkInfo = gconv.String(v)
	}
	// Parse link syntax.
	if node.LinkInfo != "" && node.Type == "" {
		match, _ := gregex.MatchString(`([a-z]+):(.+)`, node.LinkInfo)
		if len(match) == 3 {
			node.Type = gstr.Trim(match[1])
			node.LinkInfo = gstr.Trim(match[2])
		}
	}
	return node
}
<|endoftext|>


<commit_before>package arel

type FunctionNode struct {
	BaseNode
}
<commit_msg>Added several other types of FunctionNode's<commit_after>package arel

type FunctionNode struct {
	BaseNode
}

type SumNode FunctionNode
type ExistsNode FunctionNode
type MaxNode FunctionNode
type MinNode FunctionNode
type AvgNode FunctionNode
<|endoftext|>


<commit_before>package api

import (
    "fmt"
    "net/url"
    "strconv"
    "github.com/mitchellh/mapstructure"
)

const (
    DEFAULT_PRODUCT_API_LENGTH = 20
    DEFAULT_PRODUCT_MAX_LENGTH = 100

    DEFAULT_PRODUCT_MAX_OFFSET = 50000
)

type ProductService struct {
    ApiId        string `mapstructure:"api_id"`
    AffiliateId  string `mapstructure:"affiliate_id"`
    Site         string `mapstructure:"site"`
    Service      string `mapstructure:"service"`
    Floor        string `mapstructure:"floor"`
    Length       int64  `mapstructure:"hits"`
    Offset       int64  `mapstructure:"offset"`
    Sort         string `mapstructure:"sort"`
    Keyword      string `mapstructure:"keyword"`
}

type ProductRawResponse struct {
    Request ProductService  `mapstructure:"request"`
    Result  ProductResponse `mapstructure:"result"`
}

type ProductResponse struct {
    ResultCount   int64  `mapstructure:"result_count"`
    TotalCount    int64  `mapstructure:"total_count"`
    FirstPosition int64  `mapstructure:"first_position"`
    Items         []Item `mapstructure:"items"`
}

type Item struct {
    AffiliateUrl       string             `mapstructure:"affiliateURL"`
    AffiliateUrlMobile string             `mapstructure:"affiliateURLsp"`
    CategoryName       string             `mapstructure:"category_name"`
    Comment            string             `mapstructure:"comment"`
    ContentId          string             `mapstructure:"content_id"`
    Date               string             `mapstructure:"date"`
    FloorName          string             `mapstructure:"floor_name"`
    ISBN               string             `mapstructure:"isbn"`
    JANCode            string             `mapstructure:"jancode"`
    ProductCode        string             `mapstructure:"maker_product"`
    ProductId          string             `mapstructure:"product_id"`
    ServiceName        string             `mapstructure:"service_name"`
    Stock              string             `mapstructure:"stock"`
    Title              string             `mapstructure:"title"`
    Url                string             `mapstructure:"URL"`
    UrlMoble           string             `mapstructure:"URLsp"`
    Volume             string             `mapstructure:"volume"`
    ImageUrl           ImageUrlList       `mapstructure:"imageURL"`
    SampleImageUrl     SampleImageUrlList `mapstructure:"sampleImageURL"`
    SampleMovieUrl     SampleMovieUrlList `mapstructure:"sampleMovieURL"`
    Review             ReviewInformation  `mapstructure:"review"`
    PriceInformation   PriceInformation   `mapstructure:"prices"`
    ItemInformation    ItemInformation    `mapstructure:"iteminfo"`
    BandaiInformation  BandaiInformation  `mapstructure:"bandaiinfo"`
    CdInformation      CdInformation      `mapstructure:"cdinfo"`
}

type ImageUrlList struct {
    List  string `mapstructure:"list"`
    Small string `mapstructure:"small"`
    Large string `mapstructure:"large"`
}

type SampleImageUrlList struct {
    Sample_s SmallSampleList  `mapstructure:"sample_s"`
}

type SmallSampleList struct {
    Image []string `mapstructure:"image"`
}

type SampleMovieUrlList struct {
    Size_476_306 string `mapstructure:"size_476_306"`
    Size_560_360 string `mapstructure:"size_560_360"`
    Size_644_414 string `mapstructure:"size_644_414"`
    Size_720_480 string `mapstructure:"size_720_480"`
    PC_flag      bool   `mapstructure:"pc_flag"`
    SP_flag      bool   `mapstructure:"sp_flag"`
}

type PriceInformation struct {
    Price       string             `mapstructure:"price"`
    PriceAll    string             `mapstructure:"price_all"`
    RetailPrice string             `mapstructure:"list_price"`
    Distributions DistributionList `mapstructure:"deliveries"`
}

type DistributionList struct {
    Distribution []Distribution `mapstructure:"delivery"`
}

type Distribution struct {
    Type  string `mapstructure:"type"`
    Price string `mapstructure:"price"`
}

type ItemInformation struct {
    Maker     []ItemComponent `mapstructure:"maker"`
    Label     []ItemComponent `mapstructure:"label"`
    Series    []ItemComponent `mapstructure:"series"`
    Keywords  []ItemComponent `mapstructure:"keyword"`
    Genres    []ItemComponent `mapstructure:"genre"`
    Actors    []ItemComponent `mapstructure:"actor"`
    Artists   []ItemComponent `mapstructure:"artist"`
    Authors   []ItemComponent `mapstructure:"author"`
    Directors []ItemComponent `mapstructure:"director"`
    Fighters  []ItemComponent `mapstructure:"fighter"`
    Colors    []ItemComponent `mapstructure:"color"`
    Sizes     []ItemComponent `mapstructure:"size"`
}

type ItemComponent struct {
    Id   string `mapstructure:"id"`
    Name string `mapstructure:"name"`
}

type BandaiInformation struct {
    TitleCode string `mapstructure:"titlecode"`
}

type CdInformation struct {
    Kind string `mapstructure:"kind"`
}

type ReviewInformation struct {
    Count   int64   `mapstructure:"count"`
    Average float64 `mapstructure:"average"`
}

func NewProductService(affiliateId, apiId string) *ProductService {
    return &ProductService{
        ApiId:       apiId,
        AffiliateId: affiliateId,
        Site:        "",
        Service:     "",
        Floor:       "",
        Length:      DEFAULT_PRODUCT_API_LENGTH,
        Offset:      DEFAULT_API_OFFSET,
        Sort:        "",
        Keyword:     "",
    }
}

func (srv *ProductService) Execute() (*ProductResponse, error) {
    result, err := srv.ExecuteWeak()
    if err != nil {
        return nil, err
    }
    var raw ProductRawResponse
    if err = mapstructure.WeakDecode(result, &raw); err != nil {
        return nil, err
    }
    return &raw.Result, nil
}

func (srv *ProductService) ExecuteWeak() (interface{}, error) {
    reqUrl, err := srv.BuildRequestUrl()
    if err != nil {
        return nil, err
    }

    return RequestJson(reqUrl)
}

func (srv *ProductService) SetLength(length int64) *ProductService {
    srv.Length = length
    return srv
}

func (srv *ProductService) SetHits(length int64) *ProductService {
    srv.SetLength(length)
    return srv
}

func (srv *ProductService) SetOffset(offset int64) *ProductService {
    srv.Offset = offset
    return srv
}

func (srv *ProductService) SetKeyword(keyword string) *ProductService {
    srv.Keyword = keyword
    return srv
}

func (srv *ProductService) SetSite(site string) *ProductService {
    srv.Site = site
    return srv
}

func (srv *ProductService) ValidateLength() bool {
    return ValidateRange(srv.Length, 1, DEFAULT_PRODUCT_MAX_LENGTH)
}

func (srv *ProductService) ValidateOffset() bool {
    return ValidateRange(srv.Offset, 1, DEFAULT_PRODUCT_MAX_OFFSET)
}

func (srv *ProductService) BuildRequestUrl() (string, error) {
    if srv.ApiId == "" {
        return "", fmt.Errorf("set invalid ApiId parameter.")
    }
    if !ValidateAffiliateId(srv.AffiliateId) {
        return "", fmt.Errorf("set invalid AffiliateId parameter.")
    }

    if !ValidateSite(srv.Site) {
        return "", fmt.Errorf("set invalid Site parameter.")
    }

    queries := url.Values{}
    queries.Set("api_id", srv.ApiId)
    queries.Set("affiliate_id", srv.AffiliateId)
    queries.Set("site", srv.Site)

    if srv.Length != 0 {
        if !srv.ValidateLength() {
            return "", fmt.Errorf("length out of range: %d", srv.Length)
        }
        queries.Set("hits", strconv.FormatInt(srv.Length, 10))
    }

    if srv.Offset != 0 {
        if !srv.ValidateOffset() {
            return "", fmt.Errorf("offset out of range: %d", srv.Offset)
        }
        queries.Set("offset", strconv.FormatInt(srv.Offset, 10))
    }

    if (srv.Service != "") {
        queries.Set("service", srv.Service)
    }
    if (srv.Floor != "") {
        queries.Set("floor", srv.Floor)
    }
    if (srv.Sort != "") {
        queries.Set("sort", srv.Sort)
    }
    if (srv.Keyword != "") {
        queries.Set("keyword", srv.Keyword)
    }
    return API_BASE_URL + "/ItemList?" + queries.Encode(), nil
}<commit_msg>add functions in product.go<commit_after>package api

import (
    "fmt"
    "net/url"
    "strconv"
    "github.com/mitchellh/mapstructure"
)

const (
    DEFAULT_PRODUCT_API_LENGTH = 20
    DEFAULT_PRODUCT_MAX_LENGTH = 100

    DEFAULT_PRODUCT_MAX_OFFSET = 50000
)

type ProductService struct {
    ApiId        string `mapstructure:"api_id"`
    AffiliateId  string `mapstructure:"affiliate_id"`
    Site         string `mapstructure:"site"`
    Service      string `mapstructure:"service"`
    Floor        string `mapstructure:"floor"`
    Length       int64  `mapstructure:"hits"`
    Offset       int64  `mapstructure:"offset"`
    Sort         string `mapstructure:"sort"`
    Keyword      string `mapstructure:"keyword"`
}

type ProductRawResponse struct {
    Request ProductService  `mapstructure:"request"`
    Result  ProductResponse `mapstructure:"result"`
}

type ProductResponse struct {
    ResultCount   int64  `mapstructure:"result_count"`
    TotalCount    int64  `mapstructure:"total_count"`
    FirstPosition int64  `mapstructure:"first_position"`
    Items         []Item `mapstructure:"items"`
}

type Item struct {
    AffiliateUrl       string             `mapstructure:"affiliateURL"`
    AffiliateUrlMobile string             `mapstructure:"affiliateURLsp"`
    CategoryName       string             `mapstructure:"category_name"`
    Comment            string             `mapstructure:"comment"`
    ContentId          string             `mapstructure:"content_id"`
    Date               string             `mapstructure:"date"`
    FloorName          string             `mapstructure:"floor_name"`
    ISBN               string             `mapstructure:"isbn"`
    JANCode            string             `mapstructure:"jancode"`
    ProductCode        string             `mapstructure:"maker_product"`
    ProductId          string             `mapstructure:"product_id"`
    ServiceName        string             `mapstructure:"service_name"`
    Stock              string             `mapstructure:"stock"`
    Title              string             `mapstructure:"title"`
    Url                string             `mapstructure:"URL"`
    UrlMoble           string             `mapstructure:"URLsp"`
    Volume             string             `mapstructure:"volume"`
    ImageUrl           ImageUrlList       `mapstructure:"imageURL"`
    SampleImageUrl     SampleImageUrlList `mapstructure:"sampleImageURL"`
    SampleMovieUrl     SampleMovieUrlList `mapstructure:"sampleMovieURL"`
    Review             ReviewInformation  `mapstructure:"review"`
    PriceInformation   PriceInformation   `mapstructure:"prices"`
    ItemInformation    ItemInformation    `mapstructure:"iteminfo"`
    BandaiInformation  BandaiInformation  `mapstructure:"bandaiinfo"`
    CdInformation      CdInformation      `mapstructure:"cdinfo"`
}

type ImageUrlList struct {
    List  string `mapstructure:"list"`
    Small string `mapstructure:"small"`
    Large string `mapstructure:"large"`
}

type SampleImageUrlList struct {
    Sample_s SmallSampleList  `mapstructure:"sample_s"`
}

type SmallSampleList struct {
    Image []string `mapstructure:"image"`
}

type SampleMovieUrlList struct {
    Size_476_306 string `mapstructure:"size_476_306"`
    Size_560_360 string `mapstructure:"size_560_360"`
    Size_644_414 string `mapstructure:"size_644_414"`
    Size_720_480 string `mapstructure:"size_720_480"`
    PC_flag      bool   `mapstructure:"pc_flag"`
    SP_flag      bool   `mapstructure:"sp_flag"`
}

type PriceInformation struct {
    Price       string             `mapstructure:"price"`
    PriceAll    string             `mapstructure:"price_all"`
    RetailPrice string             `mapstructure:"list_price"`
    Distributions DistributionList `mapstructure:"deliveries"`
}

type DistributionList struct {
    Distribution []Distribution `mapstructure:"delivery"`
}

type Distribution struct {
    Type  string `mapstructure:"type"`
    Price string `mapstructure:"price"`
}

type ItemInformation struct {
    Maker     []ItemComponent `mapstructure:"maker"`
    Label     []ItemComponent `mapstructure:"label"`
    Series    []ItemComponent `mapstructure:"series"`
    Keywords  []ItemComponent `mapstructure:"keyword"`
    Genres    []ItemComponent `mapstructure:"genre"`
    Actors    []ItemComponent `mapstructure:"actor"`
    Artists   []ItemComponent `mapstructure:"artist"`
    Authors   []ItemComponent `mapstructure:"author"`
    Directors []ItemComponent `mapstructure:"director"`
    Fighters  []ItemComponent `mapstructure:"fighter"`
    Colors    []ItemComponent `mapstructure:"color"`
    Sizes     []ItemComponent `mapstructure:"size"`
}

type ItemComponent struct {
    Id   string `mapstructure:"id"`
    Name string `mapstructure:"name"`
}

type BandaiInformation struct {
    TitleCode string `mapstructure:"titlecode"`
}

type CdInformation struct {
    Kind string `mapstructure:"kind"`
}

type ReviewInformation struct {
    Count   int64   `mapstructure:"count"`
    Average float64 `mapstructure:"average"`
}

func NewProductService(affiliateId, apiId string) *ProductService {
    return &ProductService{
        ApiId:       apiId,
        AffiliateId: affiliateId,
        Site:        "",
        Service:     "",
        Floor:       "",
        Length:      DEFAULT_PRODUCT_API_LENGTH,
        Offset:      DEFAULT_API_OFFSET,
        Sort:        "",
        Keyword:     "",
    }
}

func (srv *ProductService) Execute() (*ProductResponse, error) {
    result, err := srv.ExecuteWeak()
    if err != nil {
        return nil, err
    }
    var raw ProductRawResponse
    if err = mapstructure.WeakDecode(result, &raw); err != nil {
        return nil, err
    }
    return &raw.Result, nil
}

func (srv *ProductService) ExecuteWeak() (interface{}, error) {
    reqUrl, err := srv.BuildRequestUrl()
    if err != nil {
        return nil, err
    }

    return RequestJson(reqUrl)
}

func (srv *ProductService) SetLength(length int64) *ProductService {
    srv.Length = length
    return srv
}

func (srv *ProductService) SetHits(length int64) *ProductService {
    srv.SetLength(length)
    return srv
}

func (srv *ProductService) SetOffset(offset int64) *ProductService {
    srv.Offset = offset
    return srv
}

func (srv *ProductService) SetKeyword(keyword string) *ProductService {
    srv.Keyword = TrimString(keyword)
    return srv
}

func (srv *ProductService) SetSort(sort string) *ProductService {
    srv.Sort = TrimString(sort)
    return srv
}

func (srv *ProductService) SetSite(site string) *ProductService {
    srv.Site = TrimString(site)
    return srv
}

func (srv *ProductService) SetService(service string) *ProductService {
    srv.Service = TrimString(service)
    return srv
}

func (srv *ProductService) SetFloor(floor string) *ProductService {
    srv.Floor = TrimString(floor)
    return srv
}

func (srv *ProductService) ValidateLength() bool {
    return ValidateRange(srv.Length, 1, DEFAULT_PRODUCT_MAX_LENGTH)
}

func (srv *ProductService) ValidateOffset() bool {
    return ValidateRange(srv.Offset, 1, DEFAULT_PRODUCT_MAX_OFFSET)
}

func (srv *ProductService) BuildRequestUrl() (string, error) {
    if srv.ApiId == "" {
        return "", fmt.Errorf("set invalid ApiId parameter.")
    }
    if !ValidateAffiliateId(srv.AffiliateId) {
        return "", fmt.Errorf("set invalid AffiliateId parameter.")
    }

    if !ValidateSite(srv.Site) {
        return "", fmt.Errorf("set invalid Site parameter.")
    }

    queries := url.Values{}
    queries.Set("api_id", srv.ApiId)
    queries.Set("affiliate_id", srv.AffiliateId)
    queries.Set("site", srv.Site)

    if srv.Length != 0 {
        if !srv.ValidateLength() {
            return "", fmt.Errorf("length out of range: %d", srv.Length)
        }
        queries.Set("hits", strconv.FormatInt(srv.Length, 10))
    }

    if srv.Offset != 0 {
        if !srv.ValidateOffset() {
            return "", fmt.Errorf("offset out of range: %d", srv.Offset)
        }
        queries.Set("offset", strconv.FormatInt(srv.Offset, 10))
    }

    if (srv.Service != "") {
        queries.Set("service", srv.Service)
    }
    if (srv.Floor != "") {
        queries.Set("floor", srv.Floor)
    }
    if (srv.Sort != "") {
        queries.Set("sort", srv.Sort)
    }
    if (srv.Keyword != "") {
        queries.Set("keyword", srv.Keyword)
    }
    return API_BASE_URL + "/ItemList?" + queries.Encode(), nil
}<|endoftext|>


<commit_before>// Copyright 2014 The Cayley Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package internal

import (
	"bytes"
	"compress/bzip2"
	"compress/gzip"
	"io"
	"strings"
	"testing"
)

var testDecompressor = []struct {
	message string
	input   io.Reader
	expect  []byte
	err     error
	readErr error
}{
	{
		message: "text input",
		input:   strings.NewReader("cayley data\n"),
		err:     nil,
		expect:  []byte("cayley data\n"),
		readErr: nil,
	},
	{
		message: "gzip input",
		input: bytes.NewReader([]byte{
			0x1f, 0x8b, 0x08, 0x00, 0x5c, 0xbc, 0xcd, 0x53, 0x00, 0x03, 0x4b, 0x4e, 0xac, 0xcc, 0x49, 0xad,
			0x54, 0x48, 0x49, 0x2c, 0x49, 0xe4, 0x02, 0x00, 0x03, 0xe1, 0xfc, 0xc3, 0x0c, 0x00, 0x00, 0x00,
		}),
		err:     nil,
		expect:  []byte("cayley data\n"),
		readErr: nil,
	},
	{
		message: "bzip2 input",
		input: bytes.NewReader([]byte{
			0x42, 0x5a, 0x68, 0x39, 0x31, 0x41, 0x59, 0x26, 0x53, 0x59, 0xb5, 0x4b, 0xe3, 0xc4, 0x00, 0x00,
			0x02, 0xd1, 0x80, 0x00, 0x10, 0x40, 0x00, 0x2e, 0x04, 0x04, 0x20, 0x20, 0x00, 0x31, 0x06, 0x4c,
			0x41, 0x4c, 0x1e, 0xa7, 0xa9, 0x2a, 0x18, 0x26, 0xb1, 0xc2, 0xee, 0x48, 0xa7, 0x0a, 0x12, 0x16,
			0xa9, 0x7c, 0x78, 0x80,
		}),
		err:     nil,
		expect:  []byte("cayley data\n"),
		readErr: nil,
	},
	{
		message: "bad gzip input",
		input:   strings.NewReader("\x1f\x8bcayley data\n"),
		err:     gzip.ErrHeader,
		expect:  nil,
		readErr: nil,
	},
	{
		message: "bad bzip2 input",
		input:   strings.NewReader("\x42\x5a\x68cayley data\n"),
		err:     nil,
		expect:  nil,
		readErr: bzip2.StructuralError("invalid compression level"),
	},
}

func TestDecompressor(t *testing.T) {
	for _, test := range testDecompressor {
		r, err := Decompressor(test.input)
		if err != test.err {
			t.Fatalf("Unexpected error for %s, got:%v expect:%v", test.message, err, test.err)
		}
		if err != nil {
			continue
		}
		p := make([]byte, len(test.expect)*2)
		n, err := r.Read(p)
		if err != test.readErr {
			t.Fatalf("Unexpected error for reading %s, got:%v expect:%v", test.message, err, test.err)
		}
		if bytes.Compare(p[:n], test.expect) != 0 {
			t.Errorf("Unexpected read result for %s, got:%q expect:%q", test.message, p[:n], test.expect)
		}
	}
}
<commit_msg>support passing EOF as an error<commit_after>// Copyright 2014 The Cayley Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package internal

import (
	"bytes"
	"compress/bzip2"
	"compress/gzip"
	"io"
	"strings"
	"testing"
)

var testDecompressor = []struct {
	message string
	input   io.Reader
	expect  []byte
	err     error
	readErr error
}{
	{
		message: "text input",
		input:   strings.NewReader("cayley data\n"),
		err:     nil,
		expect:  []byte("cayley data\n"),
		readErr: nil,
	},
	{
		message: "gzip input",
		input: bytes.NewReader([]byte{
			0x1f, 0x8b, 0x08, 0x00, 0x5c, 0xbc, 0xcd, 0x53, 0x00, 0x03, 0x4b, 0x4e, 0xac, 0xcc, 0x49, 0xad,
			0x54, 0x48, 0x49, 0x2c, 0x49, 0xe4, 0x02, 0x00, 0x03, 0xe1, 0xfc, 0xc3, 0x0c, 0x00, 0x00, 0x00,
		}),
		err:     nil,
		expect:  []byte("cayley data\n"),
		readErr: nil,
	},
	{
		message: "bzip2 input",
		input: bytes.NewReader([]byte{
			0x42, 0x5a, 0x68, 0x39, 0x31, 0x41, 0x59, 0x26, 0x53, 0x59, 0xb5, 0x4b, 0xe3, 0xc4, 0x00, 0x00,
			0x02, 0xd1, 0x80, 0x00, 0x10, 0x40, 0x00, 0x2e, 0x04, 0x04, 0x20, 0x20, 0x00, 0x31, 0x06, 0x4c,
			0x41, 0x4c, 0x1e, 0xa7, 0xa9, 0x2a, 0x18, 0x26, 0xb1, 0xc2, 0xee, 0x48, 0xa7, 0x0a, 0x12, 0x16,
			0xa9, 0x7c, 0x78, 0x80,
		}),
		err:     nil,
		expect:  []byte("cayley data\n"),
		readErr: nil,
	},
	{
		message: "bad gzip input",
		input:   strings.NewReader("\x1f\x8bcayley data\n"),
		err:     gzip.ErrHeader,
		expect:  nil,
		readErr: nil,
	},
	{
		message: "bad bzip2 input",
		input:   strings.NewReader("\x42\x5a\x68cayley data\n"),
		err:     nil,
		expect:  nil,
		readErr: bzip2.StructuralError("invalid compression level"),
	},
}

func TestDecompressor(t *testing.T) {
	for _, test := range testDecompressor {
		r, err := Decompressor(test.input)
		if err != test.err {
			t.Fatalf("Unexpected error for %s, got:%v expect:%v", test.message, err, test.err)
		}
		if err != nil {
			continue
		}
		p := make([]byte, len(test.expect)*2)
		n, err := r.Read(p)
		if err != test.readErr && err != io.EOF {
			t.Fatalf("Unexpected error for reading %s, got:%v expect:%v", test.message, err, test.err)
		}
		if bytes.Compare(p[:n], test.expect) != 0 {
			t.Errorf("Unexpected read result for %s, got:%q expect:%q", test.message, p[:n], test.expect)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package entry

import (
	"fmt"
	"strings"

	"github.com/issue9/mux/internal/syntax"
)

// 表示命名参数中的某个节点
type node struct {
	value    string // 当前节点的值。
	endByte  byte   // 下一个节点的起始字符
	isString bool   // 当前节点是否为一个字符串
	isLast   bool   // 是否为最后的节点
}

// 命名参数是正则表达式的简化版本，
// 在一个参数中未指定正则表达式，默认使用此类型，
// 性能会比用正则好一些。
type named struct {
	*base
	nodes []*node
}

// 声明一个新的 named 实例。
func newNamed(s *syntax.Syntax) *named {
	b := newBase(s)

	str := s.Patterns[len(s.Patterns)-1]
	if b.wildcard {
		str = str[:len(str)-2]
		if len(str) == 0 {
			s.Patterns = s.Patterns[:len(s.Patterns)-1]
		} else {
			s.Patterns[len(s.Patterns)-1] = str
		}
	}

	nodes := make([]*node, 0, len(s.Patterns))
	for index, str := range s.Patterns {
		last := index >= len(s.Patterns)-1
		if str[0] == syntax.Start {
			endByte := byte('/')
			if !last {
				endByte = s.Patterns[index+1][0]
			}
			nodes = append(nodes, &node{
				value:    str[1 : len(str)-1],
				isString: false,
				endByte:  endByte,
				isLast:   last,
			})
		} else {
			nodes = append(nodes, &node{
				value:    str,
				isString: true,
				isLast:   last,
			})
		}
	}

	return &named{
		base:  b,
		nodes: nodes,
	}
}

func (n *named) Priority() int {
	return syntax.TypeNamed
}

func (n *named) Match(path string) (bool, map[string]string) {
	rawPath := path
	for _, node := range n.nodes {
		if node.isString { // 普通字符串节点
			if !strings.HasPrefix(path, node.value) {
				return false, nil
			}

			if node.isLast {
				return (path == node.value), n.params(rawPath)
			}

			path = path[len(node.value):]
		} else { // 带命名的节点
			index := strings.IndexByte(path, node.endByte)
			if !node.isLast {
				if index == -1 {
					return false, nil
				}

				path = path[index:]
			} else { // 最后一个节点了
				if index == -1 {
					if n.wildcard {
						return false, nil
					}
					return true, n.params(rawPath)
				}

				if !n.wildcard {
					return false, nil
				}
				return true, n.params(rawPath)
			}
		} // end if
	} // end for

	return true, n.params(rawPath)
}

func (n *named) params(path string) map[string]string {
	// 由调用者 n.match() 保证 path 参数始终是与当前路由项匹配的。
	// 所以以下代码不再作是否匹配的检测工作。

	params := make(map[string]string, len(n.nodes))
	for _, node := range n.nodes {
		if node.isString { // 普通字符串节点
			if node.isLast {
				return params
			}

			path = path[len(node.value):]
		} else { // 带命名的节点
			index := strings.IndexByte(path, node.endByte)
			if !node.isLast {
				params[node.value] = path[:index]
				path = path[index:]
			} else { // 最后一个节点了
				if index == -1 {
					params[node.value] = path
					return params
				}

				params[node.value] = path[:index]
				return params
			}
		} // end if
	} // end for

	return params
}

func (n *named) URL(params map[string]string, path string) (string, error) {
	ret := make([]byte, 0, 500)

	for _, node := range n.nodes {
		if node.isString {
			ret = append(ret, node.value...)
			continue
		}

		if param, exists := params[node.value]; exists {
			ret = append(ret, param...)
		} else {
			return "", fmt.Errorf("参数 %v 未指定", node.value)
		}
	}

	if n.wildcard {
		ret = append(ret, '/')
		ret = append(ret, path...)
	}

	return string(ret), nil
}
<commit_msg>[internal/entry] named.Match 在返回 false 时，不再计算参数值<commit_after>// Copyright 2017 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package entry

import (
	"fmt"
	"strings"

	"github.com/issue9/mux/internal/syntax"
)

// 表示命名参数中的某个节点
type node struct {
	value    string // 当前节点的值。
	endByte  byte   // 下一个节点的起始字符
	isString bool   // 当前节点是否为一个字符串
	isLast   bool   // 是否为最后的节点
}

// 命名参数是正则表达式的简化版本，
// 在一个参数中未指定正则表达式，默认使用此类型，
// 性能会比用正则好一些。
type named struct {
	*base
	nodes []*node
}

// 声明一个新的 named 实例。
func newNamed(s *syntax.Syntax) *named {
	b := newBase(s)

	str := s.Patterns[len(s.Patterns)-1]
	if b.wildcard {
		str = str[:len(str)-2]
		if len(str) == 0 {
			s.Patterns = s.Patterns[:len(s.Patterns)-1]
		} else {
			s.Patterns[len(s.Patterns)-1] = str
		}
	}

	nodes := make([]*node, 0, len(s.Patterns))
	for index, str := range s.Patterns {
		last := index >= len(s.Patterns)-1
		if str[0] == syntax.Start {
			endByte := byte('/')
			if !last {
				endByte = s.Patterns[index+1][0]
			}
			nodes = append(nodes, &node{
				value:    str[1 : len(str)-1],
				isString: false,
				endByte:  endByte,
				isLast:   last,
			})
		} else {
			nodes = append(nodes, &node{
				value:    str,
				isString: true,
				isLast:   last,
			})
		}
	}

	return &named{
		base:  b,
		nodes: nodes,
	}
}

func (n *named) Priority() int {
	return syntax.TypeNamed
}

func (n *named) Match(path string) (bool, map[string]string) {
	rawPath := path
	for _, node := range n.nodes {
		if node.isString { // 普通字符串节点
			if !strings.HasPrefix(path, node.value) {
				return false, nil
			}
			path = path[len(node.value):]

			if node.isLast {
				if len(path) == 0 {
					return true, n.params(rawPath)
				}
				return false, nil
			}
		} else { // 带命名的节点
			index := strings.IndexByte(path, node.endByte)
			if !node.isLast {
				if index == -1 {
					return false, nil
				}

				path = path[index:]
			} else { // 最后一个节点了
				if index == -1 {
					if n.wildcard {
						return false, nil
					}
					return true, n.params(rawPath)
				}

				if !n.wildcard {
					return false, nil
				}
				return true, n.params(rawPath)
			}
		} // end if
	} // end for

	return true, n.params(rawPath)
}

func (n *named) params(path string) map[string]string {
	// 由调用者 n.match() 保证 path 参数始终是与当前路由项匹配的。
	// 所以以下代码不再作是否匹配的检测工作。

	params := make(map[string]string, len(n.nodes))
	for _, node := range n.nodes {
		if node.isString { // 普通字符串节点
			if node.isLast {
				return params
			}

			path = path[len(node.value):]
		} else { // 带命名的节点
			index := strings.IndexByte(path, node.endByte)
			if !node.isLast {
				params[node.value] = path[:index]
				path = path[index:]
			} else { // 最后一个节点了
				if index == -1 {
					params[node.value] = path
					return params
				}

				params[node.value] = path[:index]
				return params
			}
		} // end if
	} // end for

	return params
}

func (n *named) URL(params map[string]string, path string) (string, error) {
	ret := make([]byte, 0, 500)

	for _, node := range n.nodes {
		if node.isString {
			ret = append(ret, node.value...)
			continue
		}

		if param, exists := params[node.value]; exists {
			ret = append(ret, param...)
		} else {
			return "", fmt.Errorf("参数 %v 未指定", node.value)
		}
	}

	if n.wildcard {
		ret = append(ret, '/')
		ret = append(ret, path...)
	}

	return string(ret), nil
}
<|endoftext|>


<commit_before>package fs

import (
	"bytes"
	"io/ioutil"
	"os"
	"sort"
	"strings"
	"testing"
	"time"

	"github.com/google/go-cmp/cmp"
	"github.com/restic/restic/internal/test"
)

func verifyFileContentOpen(t testing.TB, fs FS, filename string, want []byte) {
	f, err := fs.Open(filename)
	if err != nil {
		t.Fatal(err)
	}

	buf, err := ioutil.ReadAll(f)
	if err != nil {
		t.Fatal(err)
	}

	err = f.Close()
	if err != nil {
		t.Fatal(err)
	}

	if !cmp.Equal(want, buf) {
		t.Error(cmp.Diff(want, buf))
	}
}

func verifyFileContentOpenFile(t testing.TB, fs FS, filename string, want []byte) {
	f, err := fs.OpenFile(filename, O_RDONLY, 0)
	if err != nil {
		t.Fatal(err)
	}

	buf, err := ioutil.ReadAll(f)
	if err != nil {
		t.Fatal(err)
	}

	err = f.Close()
	if err != nil {
		t.Fatal(err)
	}

	if !cmp.Equal(want, buf) {
		t.Error(cmp.Diff(want, buf))
	}
}

func verifyDirectoryContents(t testing.TB, fs FS, dir string, want []string) {
	f, err := fs.Open(dir)
	if err != nil {
		t.Fatal(err)
	}

	entries, err := f.Readdirnames(-1)
	if err != nil {
		t.Fatal(err)
	}

	err = f.Close()
	if err != nil {
		t.Fatal(err)
	}

	sort.Sort(sort.StringSlice(want))
	sort.Sort(sort.StringSlice(entries))

	if !cmp.Equal(want, entries) {
		t.Error(cmp.Diff(want, entries))
	}
}

type fiSlice []os.FileInfo

func (s fiSlice) Len() int {
	return len(s)
}

func (s fiSlice) Less(i, j int) bool {
	return s[i].Name() < s[j].Name()
}

func (s fiSlice) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func verifyDirectoryContentsFI(t testing.TB, fs FS, dir string, want []os.FileInfo) {
	f, err := fs.Open(dir)
	if err != nil {
		t.Fatal(err)
	}

	entries, err := f.Readdir(-1)
	if err != nil {
		t.Fatal(err)
	}

	err = f.Close()
	if err != nil {
		t.Fatal(err)
	}

	sort.Sort(fiSlice(want))
	sort.Sort(fiSlice(entries))

	if len(want) != len(entries) {
		t.Errorf("wrong number of entries returned, want %d, got %d", len(want), len(entries))
	}
	max := len(want)
	if len(entries) < max {
		max = len(entries)
	}

	for i := 0; i < max; i++ {
		fi1 := want[i]
		fi2 := entries[i]

		if fi1.Name() != fi2.Name() {
			t.Errorf("entry %d: wrong value for Name: want %q, got %q", i, fi1.Name(), fi2.Name())
		}

		if fi1.IsDir() != fi2.IsDir() {
			t.Errorf("entry %d: wrong value for IsDir: want %v, got %v", i, fi1.IsDir(), fi2.IsDir())
		}

		if fi1.Mode() != fi2.Mode() {
			t.Errorf("entry %d: wrong value for Mode: want %v, got %v", i, fi1.Mode(), fi2.Mode())
		}

		if fi1.ModTime() != fi2.ModTime() {
			t.Errorf("entry %d: wrong value for ModTime: want %v, got %v", i, fi1.ModTime(), fi2.ModTime())
		}

		if fi1.Size() != fi2.Size() {
			t.Errorf("entry %d: wrong value for Size: want %v, got %v", i, fi1.Size(), fi2.Size())
		}

		if fi1.Sys() != fi2.Sys() {
			t.Errorf("entry %d: wrong value for Sys: want %v, got %v", i, fi1.Sys(), fi2.Sys())
		}
	}
}

func checkFileInfo(t testing.TB, fi os.FileInfo, filename string, modtime time.Time, mode os.FileMode, isdir bool) {
	if fi.IsDir() {
		t.Errorf("IsDir returned true, want false")
	}

	if fi.Mode() != mode {
		t.Errorf("Mode() returned wrong value, want 0%o, got 0%o", mode, fi.Mode())
	}

	if !modtime.Equal(time.Time{}) && !fi.ModTime().Equal(modtime) {
		t.Errorf("ModTime() returned wrong value, want %v, got %v", modtime, fi.ModTime())
	}

	if fi.Name() != filename {
		t.Errorf("Name() returned wrong value, want %q, got %q", filename, fi.Name())
	}
}

func TestFSReader(t *testing.T) {
	data := test.Random(55, 1<<18+588)
	now := time.Now()
	filename := "foobar"

	var tests = []struct {
		name string
		f    func(t *testing.T, fs FS)
	}{
		{
			name: "Readdirnames-slash",
			f: func(t *testing.T, fs FS) {
				verifyDirectoryContents(t, fs, "/", []string{filename})
			},
		},
		{
			name: "Readdirnames-current",
			f: func(t *testing.T, fs FS) {
				verifyDirectoryContents(t, fs, ".", []string{filename})
			},
		},
		{
			name: "Readdir-slash",
			f: func(t *testing.T, fs FS) {
				fi := fakeFileInfo{
					mode:    0644,
					modtime: now,
					name:    filename,
					size:    int64(len(data)),
				}
				verifyDirectoryContentsFI(t, fs, "/", []os.FileInfo{fi})
			},
		},
		{
			name: "Readdir-current",
			f: func(t *testing.T, fs FS) {
				fi := fakeFileInfo{
					mode:    0644,
					modtime: now,
					name:    filename,
					size:    int64(len(data)),
				}
				verifyDirectoryContentsFI(t, fs, ".", []os.FileInfo{fi})
			},
		},
		{
			name: "file/Open",
			f: func(t *testing.T, fs FS) {
				verifyFileContentOpen(t, fs, filename, data)
			},
		},
		{
			name: "file/OpenFile",
			f: func(t *testing.T, fs FS) {
				verifyFileContentOpenFile(t, fs, filename, data)
			},
		},
		{
			name: "file/Lstat",
			f: func(t *testing.T, fs FS) {
				fi, err := fs.Lstat(filename)
				if err != nil {
					t.Fatal(err)
				}

				checkFileInfo(t, fi, filename, now, 0644, false)
			},
		},
		{
			name: "file/Stat",
			f: func(t *testing.T, fs FS) {
				f, err := fs.Open(filename)
				if err != nil {
					t.Fatal(err)
				}

				fi, err := f.Stat()
				if err != nil {
					t.Fatal(err)
				}

				err = f.Close()
				if err != nil {
					t.Fatal(err)
				}

				checkFileInfo(t, fi, filename, now, 0644, false)
			},
		},
		{
			name: "dir/Lstat-slash",
			f: func(t *testing.T, fs FS) {
				fi, err := fs.Lstat("/")
				if err != nil {
					t.Fatal(err)
				}

				checkFileInfo(t, fi, "/", time.Time{}, 0755, false)
			},
		},
		{
			name: "dir/Lstat-current",
			f: func(t *testing.T, fs FS) {
				fi, err := fs.Lstat(".")
				if err != nil {
					t.Fatal(err)
				}

				checkFileInfo(t, fi, ".", time.Time{}, 0755, false)
			},
		},
		{
			name: "dir/Open-slash",
			f: func(t *testing.T, fs FS) {
				fi, err := fs.Lstat("/")
				if err != nil {
					t.Fatal(err)
				}

				checkFileInfo(t, fi, "/", time.Time{}, 0755, false)
			},
		},
		{
			name: "dir/Open-current",
			f: func(t *testing.T, fs FS) {
				fi, err := fs.Lstat(".")
				if err != nil {
					t.Fatal(err)
				}

				checkFileInfo(t, fi, ".", time.Time{}, 0755, false)
			},
		},
	}

	for _, test := range tests {
		fs := &Reader{
			Name:       filename,
			ReadCloser: ioutil.NopCloser(bytes.NewReader(data)),

			Mode:    0644,
			Size:    int64(len(data)),
			ModTime: now,
		}

		t.Run(test.name, func(t *testing.T) {
			test.f(t, fs)
		})
	}
}

func TestFSReaderMinFileSize(t *testing.T) {
	var tests = []struct {
		name        string
		data        string
		allowEmpty  bool
		readMustErr bool
	}{
		{
			name: "regular",
			data: "foobar",
		},
		{
			name:        "empty",
			data:        "",
			allowEmpty:  false,
			readMustErr: true,
		},
		{
			name:        "empty2",
			data:        "",
			allowEmpty:  true,
			readMustErr: false,
		},
	}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			fs := &Reader{
				Name:           "testfile",
				ReadCloser:     ioutil.NopCloser(strings.NewReader(test.data)),
				Mode:           0644,
				ModTime:        time.Now(),
				AllowEmptyFile: test.allowEmpty,
			}

			f, err := fs.Open("testfile")
			if err != nil {
				t.Fatal(err)
			}

			buf, err := ioutil.ReadAll(f)
			if test.readMustErr {
				if err == nil {
					t.Fatal("expected error not found, got nil")
				}
			} else {
				if err != nil {
					t.Fatal(err)
				}
			}

			if string(buf) != test.data {
				t.Fatalf("wrong data returned, want %q, got %q", test.data, string(buf))
			}

			err = f.Close()
			if err != nil {
				t.Fatal(err)
			}
		})
	}
}
<commit_msg>fs: Update directory check in reader tests (#2063)<commit_after>package fs

import (
	"bytes"
	"io/ioutil"
	"os"
	"sort"
	"strings"
	"testing"
	"time"

	"github.com/google/go-cmp/cmp"
	"github.com/restic/restic/internal/test"
)

func verifyFileContentOpen(t testing.TB, fs FS, filename string, want []byte) {
	f, err := fs.Open(filename)
	if err != nil {
		t.Fatal(err)
	}

	buf, err := ioutil.ReadAll(f)
	if err != nil {
		t.Fatal(err)
	}

	err = f.Close()
	if err != nil {
		t.Fatal(err)
	}

	if !cmp.Equal(want, buf) {
		t.Error(cmp.Diff(want, buf))
	}
}

func verifyFileContentOpenFile(t testing.TB, fs FS, filename string, want []byte) {
	f, err := fs.OpenFile(filename, O_RDONLY, 0)
	if err != nil {
		t.Fatal(err)
	}

	buf, err := ioutil.ReadAll(f)
	if err != nil {
		t.Fatal(err)
	}

	err = f.Close()
	if err != nil {
		t.Fatal(err)
	}

	if !cmp.Equal(want, buf) {
		t.Error(cmp.Diff(want, buf))
	}
}

func verifyDirectoryContents(t testing.TB, fs FS, dir string, want []string) {
	f, err := fs.Open(dir)
	if err != nil {
		t.Fatal(err)
	}

	entries, err := f.Readdirnames(-1)
	if err != nil {
		t.Fatal(err)
	}

	err = f.Close()
	if err != nil {
		t.Fatal(err)
	}

	sort.Sort(sort.StringSlice(want))
	sort.Sort(sort.StringSlice(entries))

	if !cmp.Equal(want, entries) {
		t.Error(cmp.Diff(want, entries))
	}
}

type fiSlice []os.FileInfo

func (s fiSlice) Len() int {
	return len(s)
}

func (s fiSlice) Less(i, j int) bool {
	return s[i].Name() < s[j].Name()
}

func (s fiSlice) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func verifyDirectoryContentsFI(t testing.TB, fs FS, dir string, want []os.FileInfo) {
	f, err := fs.Open(dir)
	if err != nil {
		t.Fatal(err)
	}

	entries, err := f.Readdir(-1)
	if err != nil {
		t.Fatal(err)
	}

	err = f.Close()
	if err != nil {
		t.Fatal(err)
	}

	sort.Sort(fiSlice(want))
	sort.Sort(fiSlice(entries))

	if len(want) != len(entries) {
		t.Errorf("wrong number of entries returned, want %d, got %d", len(want), len(entries))
	}
	max := len(want)
	if len(entries) < max {
		max = len(entries)
	}

	for i := 0; i < max; i++ {
		fi1 := want[i]
		fi2 := entries[i]

		if fi1.Name() != fi2.Name() {
			t.Errorf("entry %d: wrong value for Name: want %q, got %q", i, fi1.Name(), fi2.Name())
		}

		if fi1.IsDir() != fi2.IsDir() {
			t.Errorf("entry %d: wrong value for IsDir: want %v, got %v", i, fi1.IsDir(), fi2.IsDir())
		}

		if fi1.Mode() != fi2.Mode() {
			t.Errorf("entry %d: wrong value for Mode: want %v, got %v", i, fi1.Mode(), fi2.Mode())
		}

		if fi1.ModTime() != fi2.ModTime() {
			t.Errorf("entry %d: wrong value for ModTime: want %v, got %v", i, fi1.ModTime(), fi2.ModTime())
		}

		if fi1.Size() != fi2.Size() {
			t.Errorf("entry %d: wrong value for Size: want %v, got %v", i, fi1.Size(), fi2.Size())
		}

		if fi1.Sys() != fi2.Sys() {
			t.Errorf("entry %d: wrong value for Sys: want %v, got %v", i, fi1.Sys(), fi2.Sys())
		}
	}
}

func checkFileInfo(t testing.TB, fi os.FileInfo, filename string, modtime time.Time, mode os.FileMode, isdir bool) {
	if fi.IsDir() != isdir {
		t.Errorf("IsDir returned %t, want %t", fi.IsDir(), isdir)
	}

	if fi.Mode() != mode {
		t.Errorf("Mode() returned wrong value, want 0%o, got 0%o", mode, fi.Mode())
	}

	if !modtime.Equal(time.Time{}) && !fi.ModTime().Equal(modtime) {
		t.Errorf("ModTime() returned wrong value, want %v, got %v", modtime, fi.ModTime())
	}

	if fi.Name() != filename {
		t.Errorf("Name() returned wrong value, want %q, got %q", filename, fi.Name())
	}
}

func TestFSReader(t *testing.T) {
	data := test.Random(55, 1<<18+588)
	now := time.Now()
	filename := "foobar"

	var tests = []struct {
		name string
		f    func(t *testing.T, fs FS)
	}{
		{
			name: "Readdirnames-slash",
			f: func(t *testing.T, fs FS) {
				verifyDirectoryContents(t, fs, "/", []string{filename})
			},
		},
		{
			name: "Readdirnames-current",
			f: func(t *testing.T, fs FS) {
				verifyDirectoryContents(t, fs, ".", []string{filename})
			},
		},
		{
			name: "Readdir-slash",
			f: func(t *testing.T, fs FS) {
				fi := fakeFileInfo{
					mode:    0644,
					modtime: now,
					name:    filename,
					size:    int64(len(data)),
				}
				verifyDirectoryContentsFI(t, fs, "/", []os.FileInfo{fi})
			},
		},
		{
			name: "Readdir-current",
			f: func(t *testing.T, fs FS) {
				fi := fakeFileInfo{
					mode:    0644,
					modtime: now,
					name:    filename,
					size:    int64(len(data)),
				}
				verifyDirectoryContentsFI(t, fs, ".", []os.FileInfo{fi})
			},
		},
		{
			name: "file/Open",
			f: func(t *testing.T, fs FS) {
				verifyFileContentOpen(t, fs, filename, data)
			},
		},
		{
			name: "file/OpenFile",
			f: func(t *testing.T, fs FS) {
				verifyFileContentOpenFile(t, fs, filename, data)
			},
		},
		{
			name: "file/Lstat",
			f: func(t *testing.T, fs FS) {
				fi, err := fs.Lstat(filename)
				if err != nil {
					t.Fatal(err)
				}

				checkFileInfo(t, fi, filename, now, 0644, false)
			},
		},
		{
			name: "file/Stat",
			f: func(t *testing.T, fs FS) {
				f, err := fs.Open(filename)
				if err != nil {
					t.Fatal(err)
				}

				fi, err := f.Stat()
				if err != nil {
					t.Fatal(err)
				}

				err = f.Close()
				if err != nil {
					t.Fatal(err)
				}

				checkFileInfo(t, fi, filename, now, 0644, false)
			},
		},
		{
			name: "dir/Lstat-slash",
			f: func(t *testing.T, fs FS) {
				fi, err := fs.Lstat("/")
				if err != nil {
					t.Fatal(err)
				}

				checkFileInfo(t, fi, "/", time.Time{}, 0755, false)
			},
		},
		{
			name: "dir/Lstat-current",
			f: func(t *testing.T, fs FS) {
				fi, err := fs.Lstat(".")
				if err != nil {
					t.Fatal(err)
				}

				checkFileInfo(t, fi, ".", time.Time{}, 0755, false)
			},
		},
		{
			name: "dir/Open-slash",
			f: func(t *testing.T, fs FS) {
				fi, err := fs.Lstat("/")
				if err != nil {
					t.Fatal(err)
				}

				checkFileInfo(t, fi, "/", time.Time{}, 0755, false)
			},
		},
		{
			name: "dir/Open-current",
			f: func(t *testing.T, fs FS) {
				fi, err := fs.Lstat(".")
				if err != nil {
					t.Fatal(err)
				}

				checkFileInfo(t, fi, ".", time.Time{}, 0755, false)
			},
		},
	}

	for _, test := range tests {
		fs := &Reader{
			Name:       filename,
			ReadCloser: ioutil.NopCloser(bytes.NewReader(data)),

			Mode:    0644,
			Size:    int64(len(data)),
			ModTime: now,
		}

		t.Run(test.name, func(t *testing.T) {
			test.f(t, fs)
		})
	}
}

func TestFSReaderMinFileSize(t *testing.T) {
	var tests = []struct {
		name        string
		data        string
		allowEmpty  bool
		readMustErr bool
	}{
		{
			name: "regular",
			data: "foobar",
		},
		{
			name:        "empty",
			data:        "",
			allowEmpty:  false,
			readMustErr: true,
		},
		{
			name:        "empty2",
			data:        "",
			allowEmpty:  true,
			readMustErr: false,
		},
	}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			fs := &Reader{
				Name:           "testfile",
				ReadCloser:     ioutil.NopCloser(strings.NewReader(test.data)),
				Mode:           0644,
				ModTime:        time.Now(),
				AllowEmptyFile: test.allowEmpty,
			}

			f, err := fs.Open("testfile")
			if err != nil {
				t.Fatal(err)
			}

			buf, err := ioutil.ReadAll(f)
			if test.readMustErr {
				if err == nil {
					t.Fatal("expected error not found, got nil")
				}
			} else {
				if err != nil {
					t.Fatal(err)
				}
			}

			if string(buf) != test.data {
				t.Fatalf("wrong data returned, want %q, got %q", test.data, string(buf))
			}

			err = f.Close()
			if err != nil {
				t.Fatal(err)
			}
		})
	}
}
<|endoftext|>