<commit_before>// Copyright 2014 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package host

import (
	"fmt"
	"net"
	"os"
	"os/exec"
	"runtime"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/control-center/serviced/servicedversion"
	"github.com/control-center/serviced/utils"
)

// IsLoopbackError is an error type for IP addresses that are loopback.
type IsLoopbackError string

func (err IsLoopbackError) Error() string {
	return fmt.Sprintf("IP %s is a loopback address", string(err))
}

// InvalidIPAddress is an error for Invalid IPs
type InvalidIPAddress string

func (err InvalidIPAddress) Error() string {
	return fmt.Sprintf("IP %s is not a valid address", string(err))
}

// currentHost creates a Host object of the reprsenting the host where this method is invoked. The passed in poolID is
// used as the resource pool in the result.
func currentHost(ip string, rpcPort int, poolID string) (host *Host, err error) {
	cpus := runtime.NumCPU()
	memory, err := utils.GetMemorySize()
	if err != nil {
		return nil, err
	}
	host = New()
	hostname, err := os.Hostname()
	if err != nil {
		return nil, err
	}
	host.Name = hostname
	hostidStr, err := utils.HostID()
	if err != nil {
		plog.WithError(err).WithFields(log.Fields{
			"ip": ip,
			"rpcport": rpcPort,
			"poolid": poolID,
		}).Debug("Unable to retrieve host ID")
		return nil, err
	}

	if ip != "" {
		if !ipExists(ip) {
			return nil, InvalidIPAddress(ip)
		}
		if isLoopBack(ip) {
			return nil, IsLoopbackError(ip)
		}

		host.IPAddr = ip
	} else {
		host.IPAddr, err = utils.GetIPAddress()
		if err != nil {
			return host, err
		}
	}
	host.RPCPort = rpcPort

	host.ID = hostidStr
	host.Cores = cpus
	host.Memory = memory

	// get embedded host information
	host.ServiceD.Version = servicedversion.Version
	host.ServiceD.Gitbranch = servicedversion.Gitbranch
	host.ServiceD.Gitcommit = servicedversion.Gitcommit
	host.ServiceD.Date = servicedversion.Date
	host.ServiceD.Buildtag = servicedversion.Buildtag
	host.ServiceD.Release = servicedversion.Release

	host.KernelVersion, host.KernelRelease, err = getOSKernelData()
	if err != nil {
		return nil, err
	}

	routes, err := utils.RouteCmd()
	if err != nil {
		plog.WithError(err).WithFields(log.Fields{
			"ip": ip,
			"rpcport": rpcPort,
			"poolid": poolID,
		}).Debug("Unable to get network routes")
		return nil, err
	}
	for _, route := range routes {
		if route.Iface == "docker0" {
			host.PrivateNetwork = route.Destination + "/" + route.Genmask
			break
		}
	}
	host.PoolID = poolID
	return host, err
}

func getOSKernelData() (string, string, error) {
	output, err := exec.Command("uname", "-r", "-v").Output()
	if err != nil {
		return "There was an error retrieving kernel data", "There was an error retrieving kernel data", err
	}

	kernelVersion, kernelRelease := parseOSKernelData(string(output))
	return kernelVersion, kernelRelease, err
}

func parseOSKernelData(data string) (string, string) {
	parts := strings.Split(data, " ")
	return parts[1], parts[0]
}

// getIPResources does the actual work of determining the IPs on the host. Parameters are the IPs to filter on
func getIPResources(hostID string, hostIP string, staticIPs ...string) ([]HostIPResource, error) {

	//make a map of all ipaddresses to interface
	ifacemap, err := getInterfaceMap()
	if err != nil {
		return nil, err
	}
	hostLogger := plog.WithFields(log.Fields{
		"hostid": hostID,
		"hostip": hostIP,
	})
	hostLogger.WithFields(log.Fields{
		"interfaces": ifacemap,
	}).Debug("Interfaces on this host")

	// Get a unique list of ips from staticIPs and hostIP.
	ips := func() []string {
		for _, ip := range staticIPs {
			if hostIP == ip {
				return staticIPs
			}
		}
		return append(staticIPs, hostIP)
	}()

	hostIPResources := make([]HostIPResource, len(ips))
	for i, ip := range ips {
		hostLogger.WithFields(log.Fields{
			"ip": ip,
		}).Debug("Checking IP")
		if iface, ok := ifacemap[ip]; ok {
			if isLoopBack(ip) {
				return nil, IsLoopbackError(ip)
			}
			hostIPResource := HostIPResource{
				HostID:        hostID,
				IPAddress:     ip,
				InterfaceName: iface.Name,
				MACAddress:    iface.HardwareAddr.String(),
			}
			hostIPResources[i] = hostIPResource
		} else {
			return nil, InvalidIPAddress(ip)
		}
	}

	return hostIPResources, nil
}

// getInterfaceMap returns a map of ip string to net.Interface
func getInterfaceMap() (map[string]net.Interface, error) {
	interfaces, err := net.Interfaces()
	if err != nil {
		plog.WithError(err).Debug("Unable to read network interfaces")
		return nil, err
	}
	//make a  of all ipaddresses to interface
	ips := make(map[string]net.Interface)
	for _, iface := range interfaces {
		addrs, err := iface.Addrs()
		if err != nil {
			plog.WithField("interface", iface.Name).WithError(err).Debug("Unable to read interface addresses")
			return nil, err
		}
		for _, ip := range addrs {
			normalIP := strings.SplitN(ip.String(), "/", 2)[0]
			normalIP = strings.Trim(strings.ToLower(normalIP), " ")

			ips[normalIP] = iface
		}
	}
	return ips, nil
}

func normalizeIP(ip string) string {
	return strings.Trim(strings.ToLower(ip), " ")
}

func ipExists(ip string) bool {
	interfaces, err := getInterfaceMap()
	if err != nil {
		plog.WithError(err).Debug("Unable to get network interface map")
		return false
	}
	normalIP := normalizeIP(ip)
	_, found := interfaces[normalIP]
	return found
}

func isLoopBack(ip string) bool {
	if strings.HasPrefix(ip, "127") {
		return true
	}
	return false
}
<commit_msg>Remove unecessary ipExists() check when adding a delegate<commit_after>// Copyright 2014 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package host

import (
	"fmt"
	"net"
	"os"
	"os/exec"
	"runtime"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/control-center/serviced/servicedversion"
	"github.com/control-center/serviced/utils"
)

// IsLoopbackError is an error type for IP addresses that are loopback.
type IsLoopbackError string

func (err IsLoopbackError) Error() string {
	return fmt.Sprintf("IP %s is a loopback address", string(err))
}

// InvalidIPAddress is an error for Invalid IPs
type InvalidIPAddress string

func (err InvalidIPAddress) Error() string {
	return fmt.Sprintf("IP %s is not a valid address", string(err))
}

// currentHost creates a Host object of the representing the host where this method is invoked. The passed in poolID is
// used as the resource pool in the result.
func currentHost(ip string, rpcPort int, poolID string) (host *Host, err error) {
	cpus := runtime.NumCPU()
	memory, err := utils.GetMemorySize()
	if err != nil {
		return nil, err
	}
	host = New()
	hostname, err := os.Hostname()
	if err != nil {
		return nil, err
	}
	host.Name = hostname
	hostidStr, err := utils.HostID()
	if err != nil {
		plog.WithError(err).WithFields(log.Fields{
			"ip": ip,
			"rpcport": rpcPort,
			"poolid": poolID,
		}).Debug("Unable to retrieve host ID")
		return nil, err
	}

	if ip != "" {
		if isLoopBack(ip) {
			return nil, IsLoopbackError(ip)
		}

		host.IPAddr = ip
	} else {
		host.IPAddr, err = utils.GetIPAddress()
		if err != nil {
			return host, err
		}
	}
	host.RPCPort = rpcPort

	host.ID = hostidStr
	host.Cores = cpus
	host.Memory = memory

	// get embedded host information
	host.ServiceD.Version = servicedversion.Version
	host.ServiceD.Gitbranch = servicedversion.Gitbranch
	host.ServiceD.Gitcommit = servicedversion.Gitcommit
	host.ServiceD.Date = servicedversion.Date
	host.ServiceD.Buildtag = servicedversion.Buildtag
	host.ServiceD.Release = servicedversion.Release

	host.KernelVersion, host.KernelRelease, err = getOSKernelData()
	if err != nil {
		return nil, err
	}

	routes, err := utils.RouteCmd()
	if err != nil {
		plog.WithError(err).WithFields(log.Fields{
			"ip": ip,
			"rpcport": rpcPort,
			"poolid": poolID,
		}).Debug("Unable to get network routes")
		return nil, err
	}
	for _, route := range routes {
		if route.Iface == "docker0" {
			host.PrivateNetwork = route.Destination + "/" + route.Genmask
			break
		}
	}
	host.PoolID = poolID
	return host, err
}

func getOSKernelData() (string, string, error) {
	output, err := exec.Command("uname", "-r", "-v").Output()
	if err != nil {
		return "There was an error retrieving kernel data", "There was an error retrieving kernel data", err
	}

	kernelVersion, kernelRelease := parseOSKernelData(string(output))
	return kernelVersion, kernelRelease, err
}

func parseOSKernelData(data string) (string, string) {
	parts := strings.Split(data, " ")
	return parts[1], parts[0]
}

// getIPResources does the actual work of determining the IPs on the host. Parameters are the IPs to filter on
func getIPResources(hostID string, hostIP string, staticIPs ...string) ([]HostIPResource, error) {

	//make a map of all ipaddresses to interface
	ifacemap, err := getInterfaceMap()
	if err != nil {
		return nil, err
	}
	hostLogger := plog.WithFields(log.Fields{
		"hostid": hostID,
		"hostip": hostIP,
	})
	hostLogger.WithFields(log.Fields{
		"interfaces": ifacemap,
	}).Debug("Interfaces on this host")

	// Get a unique list of ips from staticIPs and hostIP.
	ips := func() []string {
		for _, ip := range staticIPs {
			if hostIP == ip {
				return staticIPs
			}
		}
		return append(staticIPs, hostIP)
	}()

	hostIPResources := make([]HostIPResource, len(ips))
	for i, ip := range ips {
		hostLogger.WithFields(log.Fields{
			"ip": ip,
		}).Debug("Checking IP")
		if iface, ok := ifacemap[ip]; ok {
			if isLoopBack(ip) {
				return nil, IsLoopbackError(ip)
			}
			hostIPResource := HostIPResource{
				HostID:        hostID,
				IPAddress:     ip,
				InterfaceName: iface.Name,
				MACAddress:    iface.HardwareAddr.String(),
			}
			hostIPResources[i] = hostIPResource
		} else {
			return nil, InvalidIPAddress(ip)
		}
	}

	return hostIPResources, nil
}

// getInterfaceMap returns a map of ip string to net.Interface
func getInterfaceMap() (map[string]net.Interface, error) {
	interfaces, err := net.Interfaces()
	if err != nil {
		plog.WithError(err).Debug("Unable to read network interfaces")
		return nil, err
	}
	//make a  of all ipaddresses to interface
	ips := make(map[string]net.Interface)
	for _, iface := range interfaces {
		addrs, err := iface.Addrs()
		if err != nil {
			plog.WithField("interface", iface.Name).WithError(err).Debug("Unable to read interface addresses")
			return nil, err
		}
		for _, ip := range addrs {
			normalIP := strings.SplitN(ip.String(), "/", 2)[0]
			normalIP = strings.Trim(strings.ToLower(normalIP), " ")

			ips[normalIP] = iface
		}
	}
	return ips, nil
}

func isLoopBack(ip string) bool {
	if strings.HasPrefix(ip, "127") {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package dos

import (
	"os"
	"os/exec"
	"path/filepath"
	"testing"
)

func TestTruePath(t *testing.T) {
	wd, err := os.Getwd()
	if err != nil {
		t.Fatal(err)
		return
	}

	c := exec.Command("cmd", "/c", "mklink /J sub ..")
	if err := c.Run(); err != nil {
		t.Fatal(err)
		return
	}
	defer os.Remove("sub")

	result := TruePath(`sub`)
	expect := filepath.Dir(wd)
	if expect != result {
		t.Fatalf("Failed: TruePath(`sub`) -> %s (not %s)", result, expect)
		return
	}
}
<commit_msg>Append test pattern for dos.TruePath<commit_after>package dos

import (
	"os"
	"os/exec"
	"path/filepath"
	"testing"
)

func TestTruePath(t *testing.T) {
	wd, err := os.Getwd()
	if err != nil {
		t.Fatal(err)
		return
	}

	c := exec.Command("cmd", "/c", "mklink /J sub ..")
	if err := c.Run(); err != nil {
		t.Fatal(err)
		return
	}
	defer os.Remove("sub")

	result := TruePath(`sub`)
	expect := filepath.Dir(wd)
	if expect != result {
		t.Fatalf("Failed: TruePath(`sub`) -> %s (not %s)", result, expect)
		return
	}

	if err := os.Mkdir("sub/hoge",0777); err != nil {
		t.Fatalf("Failed: could not mkdir: %s", err.Error())
		return
	}
	defer os.Remove("sub/hoge")
	result = TruePath(`sub/hoge`)
	println(result)
}
<|endoftext|>


<commit_before>package koding

import (
	"errors"
	"fmt"
	"koding/db/models"
	"koding/kites/kloud/contexthelper/request"
	"koding/kites/kloud/machinestate"
	"time"

	"github.com/mitchellh/goamz/ec2"
	"golang.org/x/net/context"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

const (
	snapshotCollection = "jSnapshots"
)

func (m *Machine) DeleteSnapshot(ctx context.Context) error {
	req, ok := request.FromContext(ctx)
	if !ok {
		return errors.New("request context is not available")
	}

	var args struct {
		SnapshotId string
	}

	if err := req.Args.One().Unmarshal(&args); err != nil {
		return err
	}

	m.Log.Info("deleting snapshot from AWS %s", args.SnapshotId)
	if _, err := m.Session.AWSClient.Client.DeleteSnapshots([]string{args.SnapshotId}); err != nil {
		return err
	}

	m.Log.Debug("deleting snapshot data from MongoDB %s", args.SnapshotId)
	return m.deleteSnapshotData(args.SnapshotId)
}

func (m *Machine) CreateSnapshot(ctx context.Context) (err error) {
	req, ok := request.FromContext(ctx)
	if !ok {
		return errors.New("request context is not available")
	}

	// the user might send us a snapshot label
	var args struct {
		Label string
	}

	err = req.Args.One().Unmarshal(&args)
	if err != nil {
		return err
	}

	if err := m.UpdateState("Machine is creating snapshot", machinestate.Snapshotting); err != nil {
		return err
	}

	latestState := m.State()
	defer func() {
		if err != nil {
			m.UpdateState("Machine is marked as "+latestState.String(), latestState)
		}
	}()

	if err := m.Checker.SnapshotTotal(m.Id.Hex(), m.Username); err != nil {
		return err
	}

	a := m.Session.AWSClient

	m.push("Creating snapshot initialized", 10, machinestate.Snapshotting)
	instance, err := a.Instance()
	if err != nil {
		return err
	}

	if len(instance.BlockDevices) == 0 {
		return fmt.Errorf("createSnapshot: no block device available")
	}

	volumeId := instance.BlockDevices[0].VolumeId
	snapshotDesc := fmt.Sprintf("user-%s-%s", m.Username, m.Id.Hex())

	m.Log.Debug("Creating snapshot '%s'", snapshotDesc)
	m.push("Creating snapshot", 50, machinestate.Snapshotting)
	snapshot, err := a.CreateSnapshot(volumeId, snapshotDesc)
	if err != nil {
		return err
	}
	m.Log.Debug("Snapshot created successfully: %+v", snapshot)

	snapshotData := &models.Snapshot{
		Username:    m.Username,
		Region:      a.Client.Region.Name,
		SnapshotId:  snapshot.Id,
		MachineId:   m.Id,
		StorageSize: snapshot.VolumeSize,
		Label:       args.Label,
	}

	if err := m.addSnapshotData(snapshotData); err != nil {
		return err
	}

	tags := []ec2.Tag{
		{Key: "Name", Value: snapshotDesc},
		{Key: "koding-user", Value: m.Username},
		{Key: "koding-machineId", Value: m.Id.Hex()},
	}

	if _, err := a.Client.CreateTags([]string{snapshot.Id}, tags); err != nil {
		// don't return for a snapshot tag problem
		m.Log.Warning("Failed to tag the new snapshot: %v", err)
	}

	m.push("Snapshot creation finished successfully", 80, machinestate.Snapshotting)

	return m.Session.DB.Run("jMachines", func(c *mgo.Collection) error {
		return c.UpdateId(
			m.Id,
			bson.M{"$set": bson.M{
				"status.state":      machinestate.Running.String(),
				"status.modifiedAt": time.Now().UTC(),
				"status.reason":     "Machine is running",
			}},
		)
	})
}

func (m *Machine) addSnapshotData(doc *models.Snapshot) error {
	var account *models.Account
	if err := m.Session.DB.Run("jAccounts", func(c *mgo.Collection) error {
		return c.Find(bson.M{"profile.nickname": doc.Username}).One(&account)
	}); err != nil {
		m.Log.Error("Could not fetch account %v: err: %v", doc.Username, err)
		return errors.New("could not fetch account from DB")
	}

	// fill remaining fields
	doc.Id = bson.NewObjectId()
	doc.CreatedAt = time.Now().UTC()
	doc.OriginId = account.Id

	err := m.Session.DB.Run(snapshotCollection, func(c *mgo.Collection) error {
		return c.Insert(doc)
	})

	if err != nil {
		m.Log.Error("Could not add snapshot %v: err: %v", doc.MachineId.Hex(), doc, err)
		return errors.New("could not add snapshot to DB")
	}

	return nil
}

func (m *Machine) deleteSnapshotData(snapshotId string) error {
	err := m.Session.DB.Run(snapshotCollection, func(c *mgo.Collection) error {
		return c.Remove(bson.M{"snapshotId": snapshotId})
	})

	if err != nil {
		m.Log.Error("Could not delete %v: err: %v", snapshotId, err)
		return errors.New("could not delete snapshot from DB")
	}

	return nil
}

func (m *Machine) checkSnapshotExistence() (bool, error) {
	var exists bool
	var account *models.Account
	if err := m.Session.DB.Run("jAccounts", func(c *mgo.Collection) error {
		return c.Find(bson.M{"profile.nickname": m.Username}).One(&account)
	}); err != nil {
		m.Log.Error("Could not fetch account %v: err: %v", m.Username, err)
		return exists, errors.New("could not fetch account from DB")
	}

	var err error
	var count int

	err = m.Session.DB.Run(snapshotCollection, func(c *mgo.Collection) error {
		count, err = c.Find(bson.M{
			"originId":   account.Id,
			"snapshotId": m.Meta.SnapshotId,
		}).Count()
		return err
	})

	if err != nil {
		m.Log.Error("Could not fetch %v: err: %v", m.Meta.SnapshotId, err)
		return exists, errors.New("could not check Snapshot existency")
	}

	if count != 0 {
		exists = true
	}

	return exists, nil
}
<commit_msg>styleguide: Removed the exists var<commit_after>package koding

import (
	"errors"
	"fmt"
	"koding/db/models"
	"koding/kites/kloud/contexthelper/request"
	"koding/kites/kloud/machinestate"
	"time"

	"github.com/mitchellh/goamz/ec2"
	"golang.org/x/net/context"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

const (
	snapshotCollection = "jSnapshots"
)

func (m *Machine) DeleteSnapshot(ctx context.Context) error {
	req, ok := request.FromContext(ctx)
	if !ok {
		return errors.New("request context is not available")
	}

	var args struct {
		SnapshotId string
	}

	if err := req.Args.One().Unmarshal(&args); err != nil {
		return err
	}

	m.Log.Info("deleting snapshot from AWS %s", args.SnapshotId)
	if _, err := m.Session.AWSClient.Client.DeleteSnapshots([]string{args.SnapshotId}); err != nil {
		return err
	}

	m.Log.Debug("deleting snapshot data from MongoDB %s", args.SnapshotId)
	return m.deleteSnapshotData(args.SnapshotId)
}

func (m *Machine) CreateSnapshot(ctx context.Context) (err error) {
	req, ok := request.FromContext(ctx)
	if !ok {
		return errors.New("request context is not available")
	}

	// the user might send us a snapshot label
	var args struct {
		Label string
	}

	err = req.Args.One().Unmarshal(&args)
	if err != nil {
		return err
	}

	if err := m.UpdateState("Machine is creating snapshot", machinestate.Snapshotting); err != nil {
		return err
	}

	latestState := m.State()
	defer func() {
		if err != nil {
			m.UpdateState("Machine is marked as "+latestState.String(), latestState)
		}
	}()

	if err := m.Checker.SnapshotTotal(m.Id.Hex(), m.Username); err != nil {
		return err
	}

	a := m.Session.AWSClient

	m.push("Creating snapshot initialized", 10, machinestate.Snapshotting)
	instance, err := a.Instance()
	if err != nil {
		return err
	}

	if len(instance.BlockDevices) == 0 {
		return fmt.Errorf("createSnapshot: no block device available")
	}

	volumeId := instance.BlockDevices[0].VolumeId
	snapshotDesc := fmt.Sprintf("user-%s-%s", m.Username, m.Id.Hex())

	m.Log.Debug("Creating snapshot '%s'", snapshotDesc)
	m.push("Creating snapshot", 50, machinestate.Snapshotting)
	snapshot, err := a.CreateSnapshot(volumeId, snapshotDesc)
	if err != nil {
		return err
	}
	m.Log.Debug("Snapshot created successfully: %+v", snapshot)

	snapshotData := &models.Snapshot{
		Username:    m.Username,
		Region:      a.Client.Region.Name,
		SnapshotId:  snapshot.Id,
		MachineId:   m.Id,
		StorageSize: snapshot.VolumeSize,
		Label:       args.Label,
	}

	if err := m.addSnapshotData(snapshotData); err != nil {
		return err
	}

	tags := []ec2.Tag{
		{Key: "Name", Value: snapshotDesc},
		{Key: "koding-user", Value: m.Username},
		{Key: "koding-machineId", Value: m.Id.Hex()},
	}

	if _, err := a.Client.CreateTags([]string{snapshot.Id}, tags); err != nil {
		// don't return for a snapshot tag problem
		m.Log.Warning("Failed to tag the new snapshot: %v", err)
	}

	m.push("Snapshot creation finished successfully", 80, machinestate.Snapshotting)

	return m.Session.DB.Run("jMachines", func(c *mgo.Collection) error {
		return c.UpdateId(
			m.Id,
			bson.M{"$set": bson.M{
				"status.state":      machinestate.Running.String(),
				"status.modifiedAt": time.Now().UTC(),
				"status.reason":     "Machine is running",
			}},
		)
	})
}

func (m *Machine) addSnapshotData(doc *models.Snapshot) error {
	var account *models.Account
	if err := m.Session.DB.Run("jAccounts", func(c *mgo.Collection) error {
		return c.Find(bson.M{"profile.nickname": doc.Username}).One(&account)
	}); err != nil {
		m.Log.Error("Could not fetch account %v: err: %v", doc.Username, err)
		return errors.New("could not fetch account from DB")
	}

	// fill remaining fields
	doc.Id = bson.NewObjectId()
	doc.CreatedAt = time.Now().UTC()
	doc.OriginId = account.Id

	err := m.Session.DB.Run(snapshotCollection, func(c *mgo.Collection) error {
		return c.Insert(doc)
	})

	if err != nil {
		m.Log.Error("Could not add snapshot %v: err: %v", doc.MachineId.Hex(), doc, err)
		return errors.New("could not add snapshot to DB")
	}

	return nil
}

func (m *Machine) deleteSnapshotData(snapshotId string) error {
	err := m.Session.DB.Run(snapshotCollection, func(c *mgo.Collection) error {
		return c.Remove(bson.M{"snapshotId": snapshotId})
	})

	if err != nil {
		m.Log.Error("Could not delete %v: err: %v", snapshotId, err)
		return errors.New("could not delete snapshot from DB")
	}

	return nil
}

func (m *Machine) checkSnapshotExistence() (bool, error) {
	var account *models.Account
	if err := m.Session.DB.Run("jAccounts", func(c *mgo.Collection) error {
		return c.Find(bson.M{"profile.nickname": m.Username}).One(&account)
	}); err != nil {
		m.Log.Error("Could not fetch account %v: err: %v", m.Username, err)
		return false, errors.New("could not fetch account from DB")
	}

	var err error
	var count int

	err = m.Session.DB.Run(snapshotCollection, func(c *mgo.Collection) error {
		count, err = c.Find(bson.M{
			"originId":   account.Id,
			"snapshotId": m.Meta.SnapshotId,
		}).Count()
		return err
	})

	if err != nil {
		m.Log.Error("Could not fetch %v: err: %v", m.Meta.SnapshotId, err)
		return false, errors.New("could not check Snapshot existency")
	}

	return count != 0, nil
}
<|endoftext|>


<commit_before>package api

import (
	"encoding/json"
	"koding/db/mongodb/modelhelper"
	"socialapi/rest"
	"socialapi/workers/common/tests"
	"socialapi/workers/payment"
	"testing"

	. "github.com/smartystreets/goconvey/convey"
	"github.com/stripe/stripe-go"
	"github.com/stripe/stripe-go/customer"
	"github.com/stripe/stripe-go/invoice"
)

func TestCustomer(t *testing.T) {
	Convey("Given a user", t, func() {
		withTestServer(t, func(endpoint string) {
			withStubData(endpoint, func(username, groupName, sessionID string) {
				Convey("Then Group should have customer id", func() {
					group, err := modelhelper.GetGroup(groupName)
					tests.ResultedWithNoErrorCheck(group, err)

					So(group.Payment.Customer.ID, ShouldNotBeBlank)
					Convey("We should be able to get the customer", func() {
						getURL := endpoint + EndpointCustomerGet

						res, err := rest.DoRequestWithAuth("GET", getURL, nil, sessionID)
						So(err, ShouldBeNil)
						So(res, ShouldNotBeNil)

						v := &stripe.Customer{}
						err = json.Unmarshal(res, v)
						So(err, ShouldBeNil)

						So(v.Deleted, ShouldEqual, false)
						So(v.Desc, ShouldContainSubstring, groupName)
						So(len(v.Meta), ShouldBeGreaterThanOrEqualTo, 2)
						So(v.Meta["groupName"], ShouldEqual, groupName)
						So(v.Meta["username"], ShouldEqual, username)

						Convey("After adding credit card to the user", func() {
							addCreditCardToUserWithChecks(endpoint, sessionID)

							res, err = rest.DoRequestWithAuth("GET", getURL, nil, sessionID)
							So(err, ShouldBeNil)
							So(res, ShouldNotBeNil)

							Convey("Customer should have CC assigned", func() {
								v = &stripe.Customer{}
								err = json.Unmarshal(res, v)
								So(err, ShouldBeNil)

								So(v.DefaultSource, ShouldNotBeNil)
								So(v.DefaultSource.Deleted, ShouldBeFalse)
								So(v.DefaultSource.ID, ShouldNotBeEmpty)
							})
						})
					})
				})
			})
		})
	})
}

func TestCouponApply(t *testing.T) {
	Convey("Given a user", t, func() {
		withTestServer(t, func(endpoint string) {
			withStubData(endpoint, func(username, groupName, sessionID string) {
				withTestCoupon(func(couponID string) {
					Convey("After adding coupon to the user", func() {

						updateURL := endpoint + EndpointCustomerUpdate

						cp := &stripe.CustomerParams{
							Coupon: couponID,
						}

						req, err := json.Marshal(cp)
						So(err, ShouldBeNil)
						So(req, ShouldNotBeNil)

						res, err := rest.DoRequestWithAuth("POST", updateURL, req, sessionID)
						So(err, ShouldBeNil)
						So(res, ShouldNotBeNil)

						v := &stripe.Customer{}
						err = json.Unmarshal(res, v)
						So(err, ShouldBeNil)

						Convey("Customer should have coupon assigned", func() {
							So(v.Discount, ShouldNotBeNil)
							So(v.Discount.Coupon.ID, ShouldEqual, couponID)
							So(v.Discount.Coupon.Valid, ShouldBeTrue)
							So(v.Discount.Coupon.Deleted, ShouldBeFalse)
						})
					})
				})
			})
		})
	})
}

// TestBalanceApply does not test anything on our end. And i am not trying to be
// clever with testing stripe. This test is here only for making sure about the
// logic of Amount, Subtotal And the Total. This is the third time i am
// forgetting the logic and wanted to document it here with code.
func TestBalanceApply(t *testing.T) {
	Convey("Given a user who subscribed to a paid plan", t, func() {
		withTestServer(t, func(endpoint string) {
			withStubData(endpoint, func(username, groupName, sessionID string) {
				withNonFreeTestPlan(func(planID string) {
					addCreditCardToUserWithChecks(endpoint, sessionID)
					withSubscription(endpoint, groupName, sessionID, planID, func(subscriptionID string) {
						withTestCoupon(func(couponID string) {
							Convey("After adding balance to the user", func() {
								group, err := modelhelper.GetGroup(groupName)
								tests.ResultedWithNoErrorCheck(group, err)
								var subtotal int64 = 12345
								// A negative amount represents a credit that
								// decreases the amount due on an invoice; a
								// positive amount increases the amount due on
								// an invoice.
								var balance int64 = -150
								var coupon int64 = 100

								expectedAmount := subtotal - coupon - (-balance) // negate the balance
								cp := &stripe.CustomerParams{
									Balance: balance,
									Coupon:  couponID,
								}

								c, err := customer.Update(group.Payment.Customer.ID, cp)
								tests.ResultedWithNoErrorCheck(c, err)

								Convey("Customer should have discount", func() {
									So(c, ShouldNotBeNil)
									So(c.Balance, ShouldEqual, balance)

									Convey("Invoice should the discount", func() {
										i, err := invoice.GetNext(&stripe.InvoiceParams{Customer: c.ID})
										tests.ResultedWithNoErrorCheck(i, err)
										So(i.Subtotal, ShouldEqual, subtotal)
										So(i.Subtotal, ShouldBeGreaterThan, i.Total)
										So(i.Subtotal, ShouldEqual, i.Total+coupon) // dont forget to negate

										So(i.Total, ShouldEqual, subtotal-coupon)
										So(i.Total, ShouldBeGreaterThan, i.Amount)
										So(i.Total, ShouldEqual, i.Amount+(-balance))

										So(i.Amount, ShouldEqual, i.Total-(-balance))
										So(i.Amount, ShouldEqual, expectedAmount)
										// Subtotal = amount + coupon + balance
										// Total    = amount + coupon
										// Amount   = the final price that customer will pay.

										// Subtotal:     12345,
										// Total:        12245,
										// Amount:       12145,

										// Expected: '12245'
										// Actual:   '12445'
									})
								})
							})
						})
					})
				})
			})
		})
	})
}

func TestInfoPlan(t *testing.T) {
	Convey("Given a user", t, func() {
		withTestServer(t, func(endpoint string) {
			withStubData(endpoint, func(username, groupName, sessionID string) {
				withTestPlan(func(planID string) {
					addCreditCardToUserWithChecks(endpoint, sessionID)
					withSubscription(endpoint, groupName, sessionID, planID, func(subscriptionID string) {
						Convey("We should be able to get info", func() {
							infoURL := endpoint + EndpointInfo
							res, err := rest.DoRequestWithAuth("GET", infoURL, nil, sessionID)
							tests.ResultedWithNoErrorCheck(res, err)

							v := &payment.Usage{}
							err = json.Unmarshal(res, v)
							So(err, ShouldBeNil)

							So(v.ExpectedPlan.ID, ShouldEqual, planID)
						})
					})
				})
			})
		})
	})
}
<commit_msg>socialapi/payment: check against two plan name<commit_after>package api

import (
	"encoding/json"
	"koding/db/mongodb/modelhelper"
	"socialapi/rest"
	"socialapi/workers/common/tests"
	"socialapi/workers/payment"
	"testing"

	. "github.com/smartystreets/goconvey/convey"
	"github.com/stripe/stripe-go"
	"github.com/stripe/stripe-go/customer"
	"github.com/stripe/stripe-go/invoice"
)

func TestCustomer(t *testing.T) {
	Convey("Given a user", t, func() {
		withTestServer(t, func(endpoint string) {
			withStubData(endpoint, func(username, groupName, sessionID string) {
				Convey("Then Group should have customer id", func() {
					group, err := modelhelper.GetGroup(groupName)
					tests.ResultedWithNoErrorCheck(group, err)

					So(group.Payment.Customer.ID, ShouldNotBeBlank)
					Convey("We should be able to get the customer", func() {
						getURL := endpoint + EndpointCustomerGet

						res, err := rest.DoRequestWithAuth("GET", getURL, nil, sessionID)
						So(err, ShouldBeNil)
						So(res, ShouldNotBeNil)

						v := &stripe.Customer{}
						err = json.Unmarshal(res, v)
						So(err, ShouldBeNil)

						So(v.Deleted, ShouldEqual, false)
						So(v.Desc, ShouldContainSubstring, groupName)
						So(len(v.Meta), ShouldBeGreaterThanOrEqualTo, 2)
						So(v.Meta["groupName"], ShouldEqual, groupName)
						So(v.Meta["username"], ShouldEqual, username)

						Convey("After adding credit card to the user", func() {
							addCreditCardToUserWithChecks(endpoint, sessionID)

							res, err = rest.DoRequestWithAuth("GET", getURL, nil, sessionID)
							So(err, ShouldBeNil)
							So(res, ShouldNotBeNil)

							Convey("Customer should have CC assigned", func() {
								v = &stripe.Customer{}
								err = json.Unmarshal(res, v)
								So(err, ShouldBeNil)

								So(v.DefaultSource, ShouldNotBeNil)
								So(v.DefaultSource.Deleted, ShouldBeFalse)
								So(v.DefaultSource.ID, ShouldNotBeEmpty)
							})
						})
					})
				})
			})
		})
	})
}

func TestCouponApply(t *testing.T) {
	Convey("Given a user", t, func() {
		withTestServer(t, func(endpoint string) {
			withStubData(endpoint, func(username, groupName, sessionID string) {
				withTestCoupon(func(couponID string) {
					Convey("After adding coupon to the user", func() {

						updateURL := endpoint + EndpointCustomerUpdate

						cp := &stripe.CustomerParams{
							Coupon: couponID,
						}

						req, err := json.Marshal(cp)
						So(err, ShouldBeNil)
						So(req, ShouldNotBeNil)

						res, err := rest.DoRequestWithAuth("POST", updateURL, req, sessionID)
						So(err, ShouldBeNil)
						So(res, ShouldNotBeNil)

						v := &stripe.Customer{}
						err = json.Unmarshal(res, v)
						So(err, ShouldBeNil)

						Convey("Customer should have coupon assigned", func() {
							So(v.Discount, ShouldNotBeNil)
							So(v.Discount.Coupon.ID, ShouldEqual, couponID)
							So(v.Discount.Coupon.Valid, ShouldBeTrue)
							So(v.Discount.Coupon.Deleted, ShouldBeFalse)
						})
					})
				})
			})
		})
	})
}

// TestBalanceApply does not test anything on our end. And i am not trying to be
// clever with testing stripe. This test is here only for making sure about the
// logic of Amount, Subtotal And the Total. This is the third time i am
// forgetting the logic and wanted to document it here with code.
func TestBalanceApply(t *testing.T) {
	Convey("Given a user who subscribed to a paid plan", t, func() {
		withTestServer(t, func(endpoint string) {
			withStubData(endpoint, func(username, groupName, sessionID string) {
				withNonFreeTestPlan(func(planID string) {
					addCreditCardToUserWithChecks(endpoint, sessionID)
					withSubscription(endpoint, groupName, sessionID, planID, func(subscriptionID string) {
						withTestCoupon(func(couponID string) {
							Convey("After adding balance to the user", func() {
								group, err := modelhelper.GetGroup(groupName)
								tests.ResultedWithNoErrorCheck(group, err)
								var subtotal int64 = 12345
								// A negative amount represents a credit that
								// decreases the amount due on an invoice; a
								// positive amount increases the amount due on
								// an invoice.
								var balance int64 = -150
								var coupon int64 = 100

								expectedAmount := subtotal - coupon - (-balance) // negate the balance
								cp := &stripe.CustomerParams{
									Balance: balance,
									Coupon:  couponID,
								}

								c, err := customer.Update(group.Payment.Customer.ID, cp)
								tests.ResultedWithNoErrorCheck(c, err)

								Convey("Customer should have discount", func() {
									So(c, ShouldNotBeNil)
									So(c.Balance, ShouldEqual, balance)

									Convey("Invoice should the discount", func() {
										i, err := invoice.GetNext(&stripe.InvoiceParams{Customer: c.ID})
										tests.ResultedWithNoErrorCheck(i, err)
										So(i.Subtotal, ShouldEqual, subtotal)
										So(i.Subtotal, ShouldBeGreaterThan, i.Total)
										So(i.Subtotal, ShouldEqual, i.Total+coupon) // dont forget to negate

										So(i.Total, ShouldEqual, subtotal-coupon)
										So(i.Total, ShouldBeGreaterThan, i.Amount)
										So(i.Total, ShouldEqual, i.Amount+(-balance))

										So(i.Amount, ShouldEqual, i.Total-(-balance))
										So(i.Amount, ShouldEqual, expectedAmount)
										// Subtotal = amount + coupon + balance
										// Total    = amount + coupon
										// Amount   = the final price that customer will pay.

										// Subtotal:     12345,
										// Total:        12245,
										// Amount:       12145,

										// Expected: '12245'
										// Actual:   '12445'
									})
								})
							})
						})
					})
				})
			})
		})
	})
}

func TestInfoPlan(t *testing.T) {
	Convey("Given a user", t, func() {
		withTestServer(t, func(endpoint string) {
			withStubData(endpoint, func(username, groupName, sessionID string) {
				withTestPlan(func(planID string) {
					addCreditCardToUserWithChecks(endpoint, sessionID)
					withSubscription(endpoint, groupName, sessionID, planID, func(subscriptionID string) {
						Convey("We should be able to get info", func() {
							infoURL := endpoint + EndpointInfo
							res, err := rest.DoRequestWithAuth("GET", infoURL, nil, sessionID)
							tests.ResultedWithNoErrorCheck(res, err)

							v := &payment.Usage{}
							err = json.Unmarshal(res, v)
							So(err, ShouldBeNil)

							// we use "in" here because presence request is not synchronous and might create issues
							So(v.ExpectedPlan.ID, ShouldBeIn, []string{payment.Free, payment.Solo})
						})
					})
				})
			})
		})
	})
}
<|endoftext|>


<commit_before>package download

import (
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/subutai-io/base/agent/log"
	"github.com/subutai-io/gorjun/config"
	"github.com/subutai-io/gorjun/db"
)

type listItem struct {
	Id   string `json:"id"`
	Name string `json:"name"`
}

type AptItem struct {
	ID           string            `json:"id"`
	Size         string            `json:"size"`
	Name         string            `json:"name,omitempty"`
	Owner        []string          `json:"owner,omitempty"`
	Version      string            `json:"version,omitempty"`
	Filename     string            `json:"filename,omitempty"`
	Signature    map[string]string `json:"signature,omitempty"`
	Description  string            `json:"description,omitempty"`
	Architecture string            `json:"architecture,omitempty"`
}

type RawItem struct {
	ID        string            `json:"id"`
	Size      int64             `json:"size"`
	Name      string            `json:"name,omitempty"`
	Owner     []string          `json:"owner,omitempty"`
	Version   string            `json:"version,omitempty"`
	Signature map[string]string `json:"signature,omitempty"`
}

type ListItem struct {
	ID           string            `json:"id"`
	Size         int64             `json:"size"`
	Name         string            `json:"name"`
	Filename     string            `json:"filename"`
	Parent       string            `json:"parent"`
	Version      string            `json:"version"`
	Owner        []string          `json:"owner,omitempty"`
	Architecture string            `json:"architecture"`
	Signature    map[string]string `json:"signature,omitempty"`
}

func Handler(repo string, w http.ResponseWriter, r *http.Request) {
	hash := r.URL.Query().Get("hash")
	name := r.URL.Query().Get("name")
	if len(r.URL.Query().Get("id")) > 0 {
		hash = r.URL.Query().Get("id")
		if tmp := strings.Split(hash, "."); len(tmp) > 1 {
			hash = tmp[1]
		}
	}
	if len(hash) == 0 && len(name) == 0 {
		io.WriteString(w, "Please specify hash or name")
		return
	} else if len(name) != 0 {
		hash = db.LastHash(name, repo)
	}

	if len(db.Read(hash)) > 0 && !db.Public(hash) && !db.CheckShare(hash, db.CheckToken(r.URL.Query().Get("token"))) {
		w.WriteHeader(http.StatusNotFound)
		w.Write([]byte("Not found"))
		return
	}

	// if len(db.Read(hash)) == 0 && repo == "template" && !torrent.IsDownloaded(hash) {
	// 	torrent.AddTorrent(hash)
	// 	w.WriteHeader(http.StatusAccepted)
	// 	w.Write([]byte(torrent.Info(hash)))
	// 	return
	// }

	f, err := os.Open(config.Storage.Path + hash)
	defer f.Close()

	if log.Check(log.WarnLevel, "Opening file "+config.Storage.Path+hash, err) || len(hash) == 0 {
		if len(config.CDN.Node) > 0 {
			client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
			resp, err := client.Get(config.CDN.Node + r.URL.RequestURI())
			if !log.Check(log.WarnLevel, "Getting file from CDN", err) {
				w.Header().Set("Content-Length", resp.Header.Get("Content-Length"))
				w.Header().Set("Content-Type", resp.Header.Get("Content-Type"))
				w.Header().Set("Last-Modified", resp.Header.Get("Last-Modified"))
				w.Header().Set("Content-Disposition", resp.Header.Get("Content-Disposition"))

				io.Copy(w, resp.Body)
				resp.Body.Close()
				return
			}
		}

		w.WriteHeader(http.StatusNotFound)
		io.WriteString(w, "File not found")
		return
	}
	fi, _ := f.Stat()

	if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && fi.ModTime().Unix() <= t.Unix() {
		w.WriteHeader(http.StatusNotModified)
		return
	}

	w.Header().Set("Content-Length", fmt.Sprint(fi.Size()))
	w.Header().Set("Content-Type", r.Header.Get("Content-Type"))
	w.Header().Set("Last-Modified", fi.ModTime().Format(http.TimeFormat))

	if name = db.Read(hash); len(name) == 0 && len(config.CDN.Node) > 0 {
		httpclient := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
		resp, err := httpclient.Get(config.CDN.Node + "/kurjun/rest/template/info?id=" + hash)
		if !log.Check(log.WarnLevel, "Getting info from CDN", err) {
			var info ListItem
			rsp, err := ioutil.ReadAll(resp.Body)
			if log.Check(log.WarnLevel, "Reading from CDN response", err) {
				w.WriteHeader(http.StatusNotFound)
				io.WriteString(w, "File not found")
				return
			}
			if !log.Check(log.WarnLevel, "Decrypting request", json.Unmarshal([]byte(rsp), &info)) {
				w.Header().Set("Content-Disposition", "attachment; filename=\""+info.Filename+"\"")
			}
			resp.Body.Close()
		}
	} else {
		w.Header().Set("Content-Disposition", "attachment; filename=\""+db.Read(hash)+"\"")
	}

	io.Copy(w, f)
}

func Info(repo string, r *http.Request) []byte {
	var item, js []byte
	var info map[string]string
	p := []int{0, 1000}

	id := r.URL.Query().Get("id")
	name := r.URL.Query().Get("name")
	page := r.URL.Query().Get("page")
	owner := r.URL.Query().Get("owner")
	version := r.URL.Query().Get("version")

	list := db.Search(name)
	if len(id) > 0 {
		list = append(list[:0], id)
	}

	pstr := strings.Split(page, ",")
	p[0], _ = strconv.Atoi(pstr[0])
	if len(pstr) == 2 {
		p[1], _ = strconv.Atoi(pstr[1])
	}

	counter := 0
	for _, k := range list {
		if (!db.Public(k) && !db.CheckShare(k, db.CheckToken(r.URL.Query().Get("token")))) || (len(owner) != 0 && !db.CheckOwner(owner, k)) {
			// log.Warn("File " + k + " is not shared with " + db.CheckToken(r.URL.Query().Get("token")))
			continue
		}

		if name == "management" && repo == "template" {
			info = db.LatestTmpl(name, version)
		} else {
			info = db.Info(k)
		}

		if info["type"] == repo {
			size, _ := strconv.ParseInt(info["size"], 10, 64)

			switch repo {
			case "template":
				item, _ = json.Marshal(ListItem{
					ID:           k,
					Size:         size,
					Name:         strings.Split(info["name"], "-subutai-template")[0],
					Filename:     info["name"],
					Parent:       info["parent"],
					Version:      info["version"],
					Architecture: strings.ToUpper(info["arch"]),
					// Owner:        db.FileSignatures(k),
					Owner:     db.FileOwner(k),
					Signature: db.FileSignatures(k, name),
				})
			case "apt":
				item, _ = json.Marshal(AptItem{
					ID:           info["MD5sum"],
					Name:         info["name"],
					Description:  info["Description"],
					Architecture: info["Architecture"],
					Version:      info["Version"],
					Size:         info["Size"],
					// Owner:        db.FileSignatures(k),
					Owner:     db.FileOwner(k),
					Signature: db.FileSignatures(k, name),
				})
			case "raw":
				item, _ = json.Marshal(RawItem{
					ID:      k,
					Size:    size,
					Name:    info["name"],
					Version: info["version"],
					// Owner:   db.FileSignatures(k),
					Owner:     db.FileOwner(k),
					Signature: db.FileSignatures(k, name),
				})
			}

			if strings.HasPrefix(info["name"], name+"-subutai-template") || name == info["name"] {
				if (len(version) == 0 || strings.Contains(info["version"], version)) && k == db.LastHash(info["name"], info["type"]) {
					return item
				}
				continue
			}

			counter++
			if counter < (p[0]-1)*p[1]+1 {
				continue
			}

			if counter > (p[0]-1)*p[1]+1 {
				js = append(js, []byte(",")...)
			}
			js = append(js, item...)

			if counter == p[0]*p[1] {
				break
			}
		}
	}
	if counter > 1 {
		js = append([]byte("["), js...)
		js = append(js, []byte("]")...)
	}
	return js
}

// ProxyList retrieves list of artifacts from main CDN nodes if no data found in local database
// It creates simple JSON list of artifacts to provide it to Subutai Social.
func ProxyList(t string) []byte {
	if len(config.CDN.Node) == 0 {
		return nil
	}
	list := make([]listItem, 0)

	client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
	resp, err := client.Get(config.CDN.Node + "/kurjun/rest/" + t + "/list")
	defer resp.Body.Close()
	if log.Check(log.WarnLevel, "Getting list from CDN", err) {
		return nil
	}

	rsp, err := ioutil.ReadAll(resp.Body)
	if log.Check(log.WarnLevel, "Reading from CDN response", err) {
		return nil
	}

	if log.Check(log.WarnLevel, "Decrypting request", json.Unmarshal([]byte(rsp), &list)) {
		return nil
	}

	output, err := json.Marshal(list)
	if log.Check(log.WarnLevel, "Marshaling list", err) {
		return nil
	}
	return output
}

// ProxyInfo retrieves information from main CDN nodes if no data found in local database
// It creates simple info JSON to provide it to Subutai Social.
func ProxyInfo(uri string) []byte {
	if len(config.CDN.Node) == 0 {
		return nil
	}
	client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
	resp, err := client.Get(config.CDN.Node + uri)
	defer resp.Body.Close()
	if log.Check(log.WarnLevel, "Getting list of templates from CDN", err) {
		return nil
	}

	rsp, err := ioutil.ReadAll(resp.Body)
	if log.Check(log.WarnLevel, "Reading from CDN response", err) {
		return nil
	}
	return rsp
}
<commit_msg>Counter condition fix<commit_after>package download

import (
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/subutai-io/base/agent/log"
	"github.com/subutai-io/gorjun/config"
	"github.com/subutai-io/gorjun/db"
)

type listItem struct {
	Id   string `json:"id"`
	Name string `json:"name"`
}

type AptItem struct {
	ID           string            `json:"id"`
	Size         string            `json:"size"`
	Name         string            `json:"name,omitempty"`
	Owner        []string          `json:"owner,omitempty"`
	Version      string            `json:"version,omitempty"`
	Filename     string            `json:"filename,omitempty"`
	Signature    map[string]string `json:"signature,omitempty"`
	Description  string            `json:"description,omitempty"`
	Architecture string            `json:"architecture,omitempty"`
}

type RawItem struct {
	ID        string            `json:"id"`
	Size      int64             `json:"size"`
	Name      string            `json:"name,omitempty"`
	Owner     []string          `json:"owner,omitempty"`
	Version   string            `json:"version,omitempty"`
	Signature map[string]string `json:"signature,omitempty"`
}

type ListItem struct {
	ID           string            `json:"id"`
	Size         int64             `json:"size"`
	Name         string            `json:"name"`
	Filename     string            `json:"filename"`
	Parent       string            `json:"parent"`
	Version      string            `json:"version"`
	Owner        []string          `json:"owner,omitempty"`
	Architecture string            `json:"architecture"`
	Signature    map[string]string `json:"signature,omitempty"`
}

func Handler(repo string, w http.ResponseWriter, r *http.Request) {
	hash := r.URL.Query().Get("hash")
	name := r.URL.Query().Get("name")
	if len(r.URL.Query().Get("id")) > 0 {
		hash = r.URL.Query().Get("id")
		if tmp := strings.Split(hash, "."); len(tmp) > 1 {
			hash = tmp[1]
		}
	}
	if len(hash) == 0 && len(name) == 0 {
		io.WriteString(w, "Please specify hash or name")
		return
	} else if len(name) != 0 {
		hash = db.LastHash(name, repo)
	}

	if len(db.Read(hash)) > 0 && !db.Public(hash) && !db.CheckShare(hash, db.CheckToken(r.URL.Query().Get("token"))) {
		w.WriteHeader(http.StatusNotFound)
		w.Write([]byte("Not found"))
		return
	}

	// if len(db.Read(hash)) == 0 && repo == "template" && !torrent.IsDownloaded(hash) {
	// 	torrent.AddTorrent(hash)
	// 	w.WriteHeader(http.StatusAccepted)
	// 	w.Write([]byte(torrent.Info(hash)))
	// 	return
	// }

	f, err := os.Open(config.Storage.Path + hash)
	defer f.Close()

	if log.Check(log.WarnLevel, "Opening file "+config.Storage.Path+hash, err) || len(hash) == 0 {
		if len(config.CDN.Node) > 0 {
			client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
			resp, err := client.Get(config.CDN.Node + r.URL.RequestURI())
			if !log.Check(log.WarnLevel, "Getting file from CDN", err) {
				w.Header().Set("Content-Length", resp.Header.Get("Content-Length"))
				w.Header().Set("Content-Type", resp.Header.Get("Content-Type"))
				w.Header().Set("Last-Modified", resp.Header.Get("Last-Modified"))
				w.Header().Set("Content-Disposition", resp.Header.Get("Content-Disposition"))

				io.Copy(w, resp.Body)
				resp.Body.Close()
				return
			}
		}

		w.WriteHeader(http.StatusNotFound)
		io.WriteString(w, "File not found")
		return
	}
	fi, _ := f.Stat()

	if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && fi.ModTime().Unix() <= t.Unix() {
		w.WriteHeader(http.StatusNotModified)
		return
	}

	w.Header().Set("Content-Length", fmt.Sprint(fi.Size()))
	w.Header().Set("Content-Type", r.Header.Get("Content-Type"))
	w.Header().Set("Last-Modified", fi.ModTime().Format(http.TimeFormat))

	if name = db.Read(hash); len(name) == 0 && len(config.CDN.Node) > 0 {
		httpclient := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
		resp, err := httpclient.Get(config.CDN.Node + "/kurjun/rest/template/info?id=" + hash)
		if !log.Check(log.WarnLevel, "Getting info from CDN", err) {
			var info ListItem
			rsp, err := ioutil.ReadAll(resp.Body)
			if log.Check(log.WarnLevel, "Reading from CDN response", err) {
				w.WriteHeader(http.StatusNotFound)
				io.WriteString(w, "File not found")
				return
			}
			if !log.Check(log.WarnLevel, "Decrypting request", json.Unmarshal([]byte(rsp), &info)) {
				w.Header().Set("Content-Disposition", "attachment; filename=\""+info.Filename+"\"")
			}
			resp.Body.Close()
		}
	} else {
		w.Header().Set("Content-Disposition", "attachment; filename=\""+db.Read(hash)+"\"")
	}

	io.Copy(w, f)
}

func Info(repo string, r *http.Request) []byte {
	var item, js []byte
	var info map[string]string
	p := []int{0, 1000}

	id := r.URL.Query().Get("id")
	name := r.URL.Query().Get("name")
	page := r.URL.Query().Get("page")
	owner := r.URL.Query().Get("owner")
	version := r.URL.Query().Get("version")

	list := db.Search(name)
	if len(id) > 0 {
		list = append(list[:0], id)
	}

	pstr := strings.Split(page, ",")
	p[0], _ = strconv.Atoi(pstr[0])
	if len(pstr) == 2 {
		p[1], _ = strconv.Atoi(pstr[1])
	}

	counter := 0
	for _, k := range list {
		if (!db.Public(k) && !db.CheckShare(k, db.CheckToken(r.URL.Query().Get("token")))) || (len(owner) != 0 && !db.CheckOwner(owner, k)) {
			// log.Warn("File " + k + " is not shared with " + db.CheckToken(r.URL.Query().Get("token")))
			continue
		}

		if name == "management" && repo == "template" {
			info = db.LatestTmpl(name, version)
		} else {
			info = db.Info(k)
		}

		if info["type"] == repo {
			size, _ := strconv.ParseInt(info["size"], 10, 64)

			switch repo {
			case "template":
				item, _ = json.Marshal(ListItem{
					ID:           k,
					Size:         size,
					Name:         strings.Split(info["name"], "-subutai-template")[0],
					Filename:     info["name"],
					Parent:       info["parent"],
					Version:      info["version"],
					Architecture: strings.ToUpper(info["arch"]),
					// Owner:        db.FileSignatures(k),
					Owner:     db.FileOwner(k),
					Signature: db.FileSignatures(k, name),
				})
			case "apt":
				item, _ = json.Marshal(AptItem{
					ID:           info["MD5sum"],
					Name:         info["name"],
					Description:  info["Description"],
					Architecture: info["Architecture"],
					Version:      info["Version"],
					Size:         info["Size"],
					// Owner:        db.FileSignatures(k),
					Owner:     db.FileOwner(k),
					Signature: db.FileSignatures(k, name),
				})
			case "raw":
				item, _ = json.Marshal(RawItem{
					ID:      k,
					Size:    size,
					Name:    info["name"],
					Version: info["version"],
					// Owner:   db.FileSignatures(k),
					Owner:     db.FileOwner(k),
					Signature: db.FileSignatures(k, name),
				})
			}

			if strings.HasPrefix(info["name"], name+"-subutai-template") || name == info["name"] {
				if (len(version) == 0 || strings.Contains(info["version"], version)) && k == db.LastHash(info["name"], info["type"]) {
					return item
				}
				continue
			}

			counter++
			if counter < (p[0]-1)*p[1]+1 {
				continue
			}

			if counter > 1 && counter > (p[0]-1)*p[1]+1 {
				js = append(js, []byte(",")...)
			}
			js = append(js, item...)

			if counter == p[0]*p[1] {
				break
			}
		}
	}
	if counter > 1 {
		js = append([]byte("["), js...)
		js = append(js, []byte("]")...)
	}
	return js
}

// ProxyList retrieves list of artifacts from main CDN nodes if no data found in local database
// It creates simple JSON list of artifacts to provide it to Subutai Social.
func ProxyList(t string) []byte {
	if len(config.CDN.Node) == 0 {
		return nil
	}
	list := make([]listItem, 0)

	client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
	resp, err := client.Get(config.CDN.Node + "/kurjun/rest/" + t + "/list")
	defer resp.Body.Close()
	if log.Check(log.WarnLevel, "Getting list from CDN", err) {
		return nil
	}

	rsp, err := ioutil.ReadAll(resp.Body)
	if log.Check(log.WarnLevel, "Reading from CDN response", err) {
		return nil
	}

	if log.Check(log.WarnLevel, "Decrypting request", json.Unmarshal([]byte(rsp), &list)) {
		return nil
	}

	output, err := json.Marshal(list)
	if log.Check(log.WarnLevel, "Marshaling list", err) {
		return nil
	}
	return output
}

// ProxyInfo retrieves information from main CDN nodes if no data found in local database
// It creates simple info JSON to provide it to Subutai Social.
func ProxyInfo(uri string) []byte {
	if len(config.CDN.Node) == 0 {
		return nil
	}
	client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
	resp, err := client.Get(config.CDN.Node + uri)
	defer resp.Body.Close()
	if log.Check(log.WarnLevel, "Getting list of templates from CDN", err) {
		return nil
	}

	rsp, err := ioutil.ReadAll(resp.Body)
	if log.Check(log.WarnLevel, "Reading from CDN response", err) {
		return nil
	}
	return rsp
}
<|endoftext|>


<commit_before>package gwent

type CardScorch struct {
	BasicCard
}

func (c *CardScorch) Play(p *Player, target Card) {
	p.Grave = append(p.Grave, c)
	p.Scorch()
}

func (c *CardScorch) Type() CardType {
	return TypeScorch
}
<commit_msg>++docs<commit_after>package gwent

// CardScorch destroys cards with most power
type CardScorch struct {
	BasicCard
}

// Play put Scorch card to grave and plays Scorch effect
func (c *CardScorch) Play(p *Player, target Card) {
	p.Grave = append(p.Grave, c)
	p.Scorch()
}

// Type reports that this is a scorch card
func (c *CardScorch) Type() CardType {
	return TypeScorch
}
<|endoftext|>


<commit_before>package builder

import (
	"github.com/appc/spec/aci"
	"github.com/appc/spec/schema"
	"github.com/blablacar/cnt/config"
	"github.com/blablacar/cnt/log"
	"github.com/blablacar/cnt/utils"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
)

func (cnt *Img) Push() {
	if config.GetConfig().Push.Type == "" {
		log.Get().Panic("Can't push, push is not configured in cnt global configuration file")
	}

	cnt.CheckBuilt()
	cnt.tarAci(true)

	im := extractManifestFromAci(cnt.target + PATH_IMAGE_ACI_ZIP)
	val, _ := im.Labels.Get("version")
	if err := utils.ExecCmd("curl", "-i",
		"-F", "r=releases",
		"-F", "hasPom=false",
		"-F", "e=aci",
		"-F", "g=com.blablacar.aci.linux.amd64",
		"-F", "p=aci",
		"-F", "v="+val,
		"-F", "a="+ShortNameId(im.Name),
		"-F", "file=@"+cnt.target+PATH_IMAGE_ACI_ZIP,
		"-u", config.GetConfig().Push.Username+":"+config.GetConfig().Push.Password,
		config.GetConfig().Push.Url+"/service/local/artifact/maven/content"); err != nil {
		log.Get().Panic("Cannot push aci", err)
	}
}

func extractManifestFromAci(aciPath string) schema.ImageManifest {
	input, err := os.Open(aciPath)
	if err != nil {
		log.Get().Panic("cat-manifest: Cannot open %s: %v", aciPath, err)
	}
	defer input.Close()

	tr, err := aci.NewCompressedTarReader(input)
	if err != nil {
		log.Get().Panic("cat-manifest: Cannot open tar %s: %v", aciPath, err)
	}

	im := schema.ImageManifest{}

Tar:
	for {
		hdr, err := tr.Next()
		switch err {
		case io.EOF:
			break Tar
		case nil:
			if filepath.Clean(hdr.Name) == aci.ManifestFile {
				bytes, err := ioutil.ReadAll(tr)
				if err != nil {
					log.Get().Panic(err)
				}

				err = im.UnmarshalJSON(bytes)
				if err != nil {
					log.Get().Panic(err)
				}
				return im
			}
		default:
			log.Get().Panic("error reading tarball: %v", err)
		}
	}
	log.Get().Panic("Cannot found manifest if aci")
	return im
}
<commit_msg>tell curl to fail when pushing to nexus if status > 400<commit_after>package builder

import (
	"github.com/appc/spec/aci"
	"github.com/appc/spec/schema"
	"github.com/blablacar/cnt/config"
	"github.com/blablacar/cnt/log"
	"github.com/blablacar/cnt/utils"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
)

func (cnt *Img) Push() {
	if config.GetConfig().Push.Type == "" {
		log.Get().Panic("Can't push, push is not configured in cnt global configuration file")
	}

	cnt.CheckBuilt()
	cnt.tarAci(true)

	im := extractManifestFromAci(cnt.target + PATH_IMAGE_ACI_ZIP)
	val, _ := im.Labels.Get("version")
	if err := utils.ExecCmd("curl", "-f", "-i",
		"-F", "r=releases",
		"-F", "hasPom=false",
		"-F", "e=aci",
		"-F", "g=com.blablacar.aci.linux.amd64",
		"-F", "p=aci",
		"-F", "v="+val,
		"-F", "a="+ShortNameId(im.Name),
		"-F", "file=@"+cnt.target+PATH_IMAGE_ACI_ZIP,
		"-u", config.GetConfig().Push.Username+":"+config.GetConfig().Push.Password,
		config.GetConfig().Push.Url+"/service/local/artifact/maven/content"); err != nil {
		log.Get().Panic("Cannot push aci", err)
	}
}

func extractManifestFromAci(aciPath string) schema.ImageManifest {
	input, err := os.Open(aciPath)
	if err != nil {
		log.Get().Panic("cat-manifest: Cannot open %s: %v", aciPath, err)
	}
	defer input.Close()

	tr, err := aci.NewCompressedTarReader(input)
	if err != nil {
		log.Get().Panic("cat-manifest: Cannot open tar %s: %v", aciPath, err)
	}

	im := schema.ImageManifest{}

Tar:
	for {
		hdr, err := tr.Next()
		switch err {
		case io.EOF:
			break Tar
		case nil:
			if filepath.Clean(hdr.Name) == aci.ManifestFile {
				bytes, err := ioutil.ReadAll(tr)
				if err != nil {
					log.Get().Panic(err)
				}

				err = im.UnmarshalJSON(bytes)
				if err != nil {
					log.Get().Panic(err)
				}
				return im
			}
		default:
			log.Get().Panic("error reading tarball: %v", err)
		}
	}
	log.Get().Panic("Cannot found manifest if aci")
	return im
}
<|endoftext|>


<commit_before>// Copyright 2016 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package public

import (
	"log"
	"net/http"
	"path"
	"path/filepath"
	"strings"

	"code.gitea.io/gitea/modules/httpcache"
	"code.gitea.io/gitea/modules/setting"
)

// Options represents the available options to configure the handler.
type Options struct {
	Directory   string
	IndexFile   string
	SkipLogging bool
	FileSystem  http.FileSystem
	Prefix      string
}

// KnownPublicEntries list all direct children in the `public` directory
var KnownPublicEntries = []string{
	"css",
	"img",
	"js",
	"serviceworker.js",
	"vendor",
	"favicon.ico",
}

// Custom implements the static handler for serving custom assets.
func Custom(opts *Options) func(next http.Handler) http.Handler {
	return opts.staticHandler(path.Join(setting.CustomPath, "public"))
}

// staticFileSystem implements http.FileSystem interface.
type staticFileSystem struct {
	dir *http.Dir
}

func newStaticFileSystem(directory string) staticFileSystem {
	if !filepath.IsAbs(directory) {
		directory = filepath.Join(setting.AppWorkPath, directory)
	}
	dir := http.Dir(directory)
	return staticFileSystem{&dir}
}

func (fs staticFileSystem) Open(name string) (http.File, error) {
	return fs.dir.Open(name)
}

// StaticHandler sets up a new middleware for serving static files in the
func StaticHandler(dir string, opts *Options) func(next http.Handler) http.Handler {
	return opts.staticHandler(dir)
}

func (opts *Options) staticHandler(dir string) func(next http.Handler) http.Handler {
	return func(next http.Handler) http.Handler {
		// Defaults
		if len(opts.IndexFile) == 0 {
			opts.IndexFile = "index.html"
		}
		// Normalize the prefix if provided
		if opts.Prefix != "" {
			// Ensure we have a leading '/'
			if opts.Prefix[0] != '/' {
				opts.Prefix = "/" + opts.Prefix
			}
			// Remove any trailing '/'
			opts.Prefix = strings.TrimRight(opts.Prefix, "/")
		}
		if opts.FileSystem == nil {
			opts.FileSystem = newStaticFileSystem(dir)
		}

		return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
			if !opts.handle(w, req, opts) {
				next.ServeHTTP(w, req)
			}
		})
	}
}

// parseAcceptEncoding parse Accept-Encoding: deflate, gzip;q=1.0, *;q=0.5 as compress methods
func parseAcceptEncoding(val string) map[string]bool {
	parts := strings.Split(val, ";")
	var types = make(map[string]bool)
	for _, v := range strings.Split(parts[0], ",") {
		types[strings.TrimSpace(v)] = true
	}
	return types
}

func (opts *Options) handle(w http.ResponseWriter, req *http.Request, opt *Options) bool {
	if req.Method != "GET" && req.Method != "HEAD" {
		return false
	}

	file := req.URL.Path
	// if we have a prefix, filter requests by stripping the prefix
	if opt.Prefix != "" {
		if !strings.HasPrefix(file, opt.Prefix) {
			return false
		}
		file = file[len(opt.Prefix):]
		if file != "" && file[0] != '/' {
			return false
		}
	}

	f, err := opt.FileSystem.Open(file)
	if err != nil {
		// 404 requests to any known entries in `public`
		if path.Base(opts.Directory) == "public" {
			parts := strings.Split(file, "/")
			if len(parts) < 2 {
				return false
			}
			for _, entry := range KnownPublicEntries {
				if entry == parts[1] {
					w.WriteHeader(404)
					return true
				}
			}
		}
		return false
	}
	defer f.Close()

	fi, err := f.Stat()
	if err != nil {
		log.Printf("[Static] %q exists, but fails to open: %v", file, err)
		return true
	}

	// Try to serve index file
	if fi.IsDir() {
		// Redirect if missing trailing slash.
		if !strings.HasSuffix(req.URL.Path, "/") {
			http.Redirect(w, req, path.Clean(req.URL.Path+"/"), http.StatusFound)
			return true
		}

		f, err = opt.FileSystem.Open(file)
		if err != nil {
			return false // Discard error.
		}
		defer f.Close()

		fi, err = f.Stat()
		if err != nil || fi.IsDir() {
			return false
		}
	}

	if !opt.SkipLogging {
		log.Println("[Static] Serving " + file)
	}

	if httpcache.HandleEtagCache(req, w, fi) {
		return true
	}

	ServeContent(w, req, fi, fi.ModTime(), f)
	return true
}
<commit_msg>add 'fonts' into 'KnownPublicEntries' (#15188)<commit_after>// Copyright 2016 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package public

import (
	"log"
	"net/http"
	"path"
	"path/filepath"
	"strings"

	"code.gitea.io/gitea/modules/httpcache"
	"code.gitea.io/gitea/modules/setting"
)

// Options represents the available options to configure the handler.
type Options struct {
	Directory   string
	IndexFile   string
	SkipLogging bool
	FileSystem  http.FileSystem
	Prefix      string
}

// KnownPublicEntries list all direct children in the `public` directory
var KnownPublicEntries = []string{
	"css",
	"fonts",
	"img",
	"js",
	"serviceworker.js",
	"vendor",
	"favicon.ico",
}

// Custom implements the static handler for serving custom assets.
func Custom(opts *Options) func(next http.Handler) http.Handler {
	return opts.staticHandler(path.Join(setting.CustomPath, "public"))
}

// staticFileSystem implements http.FileSystem interface.
type staticFileSystem struct {
	dir *http.Dir
}

func newStaticFileSystem(directory string) staticFileSystem {
	if !filepath.IsAbs(directory) {
		directory = filepath.Join(setting.AppWorkPath, directory)
	}
	dir := http.Dir(directory)
	return staticFileSystem{&dir}
}

func (fs staticFileSystem) Open(name string) (http.File, error) {
	return fs.dir.Open(name)
}

// StaticHandler sets up a new middleware for serving static files in the
func StaticHandler(dir string, opts *Options) func(next http.Handler) http.Handler {
	return opts.staticHandler(dir)
}

func (opts *Options) staticHandler(dir string) func(next http.Handler) http.Handler {
	return func(next http.Handler) http.Handler {
		// Defaults
		if len(opts.IndexFile) == 0 {
			opts.IndexFile = "index.html"
		}
		// Normalize the prefix if provided
		if opts.Prefix != "" {
			// Ensure we have a leading '/'
			if opts.Prefix[0] != '/' {
				opts.Prefix = "/" + opts.Prefix
			}
			// Remove any trailing '/'
			opts.Prefix = strings.TrimRight(opts.Prefix, "/")
		}
		if opts.FileSystem == nil {
			opts.FileSystem = newStaticFileSystem(dir)
		}

		return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
			if !opts.handle(w, req, opts) {
				next.ServeHTTP(w, req)
			}
		})
	}
}

// parseAcceptEncoding parse Accept-Encoding: deflate, gzip;q=1.0, *;q=0.5 as compress methods
func parseAcceptEncoding(val string) map[string]bool {
	parts := strings.Split(val, ";")
	var types = make(map[string]bool)
	for _, v := range strings.Split(parts[0], ",") {
		types[strings.TrimSpace(v)] = true
	}
	return types
}

func (opts *Options) handle(w http.ResponseWriter, req *http.Request, opt *Options) bool {
	if req.Method != "GET" && req.Method != "HEAD" {
		return false
	}

	file := req.URL.Path
	// if we have a prefix, filter requests by stripping the prefix
	if opt.Prefix != "" {
		if !strings.HasPrefix(file, opt.Prefix) {
			return false
		}
		file = file[len(opt.Prefix):]
		if file != "" && file[0] != '/' {
			return false
		}
	}

	f, err := opt.FileSystem.Open(file)
	if err != nil {
		// 404 requests to any known entries in `public`
		if path.Base(opts.Directory) == "public" {
			parts := strings.Split(file, "/")
			if len(parts) < 2 {
				return false
			}
			for _, entry := range KnownPublicEntries {
				if entry == parts[1] {
					w.WriteHeader(404)
					return true
				}
			}
		}
		return false
	}
	defer f.Close()

	fi, err := f.Stat()
	if err != nil {
		log.Printf("[Static] %q exists, but fails to open: %v", file, err)
		return true
	}

	// Try to serve index file
	if fi.IsDir() {
		// Redirect if missing trailing slash.
		if !strings.HasSuffix(req.URL.Path, "/") {
			http.Redirect(w, req, path.Clean(req.URL.Path+"/"), http.StatusFound)
			return true
		}

		f, err = opt.FileSystem.Open(file)
		if err != nil {
			return false // Discard error.
		}
		defer f.Close()

		fi, err = f.Stat()
		if err != nil || fi.IsDir() {
			return false
		}
	}

	if !opt.SkipLogging {
		log.Println("[Static] Serving " + file)
	}

	if httpcache.HandleEtagCache(req, w, fi) {
		return true
	}

	ServeContent(w, req, fi, fi.ModTime(), f)
	return true
}
<|endoftext|>


<commit_before>package renter

import (
	"errors"
	"io"
	"os"
	"sync"
	"sync/atomic"

	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

const (
	defaultDuration     = 6000 // Duration that hosts will hold onto the file
	defaultDataPieces   = 2    // Data pieces per erasure-coded chunk
	defaultParityPieces = 10   // Parity pieces per erasure-coded chunk

	// piece sizes
	// NOTE: The encryption overhead is subtracted so that encrypted piece
	// will always be a multiple of 64 (i.e. crypto.SegmentSize). Without this
	// property, revisions break the file's Merkle root.
	defaultPieceSize = 1<<22 - crypto.TwofishOverhead // 4 MiB
	smallPieceSize   = 1<<16 - crypto.TwofishOverhead // 64 KiB
)

type uploadPiece struct {
	data       []byte
	chunkIndex uint64
	pieceIndex uint64
}

// An uploader uploads pieces to a host. This interface exists to facilitate
// easy testing.
type uploader interface {
	// addPiece uploads a piece to the uploader.
	addPiece(uploadPiece) error

	// fileContract returns the fileContract containing the metadata of all
	// previously added pieces.
	fileContract() fileContract
}

// upload reads chunks from r and uploads them to hosts. It spawns a worker
// for each host, and instructs them to upload pieces of each chunk.
func (f *file) upload(r io.Reader, hosts []uploader) error {
	// encode and upload each chunk
	var wg sync.WaitGroup
	for i := uint64(0); ; i++ {
		// read next chunk
		chunk := make([]byte, f.chunkSize())
		_, err := io.ReadFull(r, chunk)
		if err == io.EOF {
			break
		} else if err != nil && err != io.ErrUnexpectedEOF {
			return err
		}
		// encode
		pieces, err := f.erasureCode.Encode(chunk)
		if err != nil {
			return err
		}
		// upload pieces, split evenly among hosts
		wg.Add(len(pieces))
		for j, data := range pieces {
			go func(j int, data []byte) {
				err := hosts[j%len(hosts)].addPiece(uploadPiece{data, i, uint64(j)})
				if err == nil {
					atomic.AddUint64(&f.bytesUploaded, uint64(len(data)))
				}
				wg.Done()
			}(j, data)
		}
		wg.Wait()
		atomic.AddUint64(&f.chunksUploaded, 1)

		// update contracts
		for _, h := range hosts {
			contract := h.fileContract()
			f.contracts[contract.IP] = contract
		}
	}

	return nil
}

// checkWalletBalance looks at an upload and determines if there is enough
// money in the wallet to support such an upload. An error is returned if it is
// determined that there is not enough money.
func (r *Renter) checkWalletBalance(up modules.FileUploadParams) error {
	// Get the size of the file.
	fileInfo, err := os.Stat(up.Filename)
	if err != nil {
		return err
	}
	curSize := types.NewCurrency64(uint64(fileInfo.Size()))

	var averagePrice types.Currency
	sampleSize := up.ErasureCode.NumPieces() * 3 / 2
	hosts := r.hostDB.RandomHosts(sampleSize)
	for _, host := range hosts {
		averagePrice = averagePrice.Add(host.Price)
	}
	if len(hosts) == 0 {
		return errors.New("no hosts!")
	}
	averagePrice = averagePrice.Div(types.NewCurrency64(uint64(len(hosts))))
	estimatedCost := averagePrice.Mul(types.NewCurrency64(uint64(up.Duration))).Mul(curSize)
	bufferedCost := estimatedCost.Mul(types.NewCurrency64(2))

	siacoinBalance, _, _ := r.wallet.ConfirmedBalance()
	if bufferedCost.Cmp(siacoinBalance) > 0 {
		return errors.New("insufficient balance for upload")
	}
	return nil
}

// Upload takes an upload parameters, which contain a file to upload, and then
// creates a redundant copy of the file on the Sia network.
func (r *Renter) Upload(up modules.FileUploadParams) error {
	// Open the file.
	handle, err := os.Open(up.Filename)
	if err != nil {
		return err
	}
	defer handle.Close()

	// Check for a nickname conflict.
	lockID := r.mu.RLock()
	_, exists := r.files[up.Nickname]
	r.mu.RUnlock(lockID)
	if exists {
		return errors.New("file with that nickname already exists")
	}

	// Check that the file is less than 5 GiB.
	fileInfo, err := handle.Stat()
	if err != nil {
		return err
	}
	// NOTE: The upload max of 5 GiB is temporary and therefore does not have
	// a constant. This should be removed once micropayments + upload resuming
	// are in place. 5 GiB is chosen to prevent confusion - on anybody's
	// machine any file appearing to be under 5 GB will be below the hard
	// limit.
	if fileInfo.Size() > 5*1024*1024*1024 {
		return errors.New("cannot upload a file larger than 5 GB")
	}

	// Fill in any missing upload params with sensible defaults.
	if up.Duration == 0 {
		up.Duration = defaultDuration
	}
	if up.ErasureCode == nil {
		up.ErasureCode, _ = NewRSCode(defaultDataPieces, defaultParityPieces)
	}
	if up.PieceSize == 0 {
		if fileInfo.Size() > defaultPieceSize {
			up.PieceSize = defaultPieceSize
		} else {
			up.PieceSize = smallPieceSize
		}
	}

	// Check that we have enough money to finance the upload.
	err = r.checkWalletBalance(up)
	if err != nil {
		return err
	}

	// Create file object.
	f := newFile(up.Nickname, up.ErasureCode, up.PieceSize, uint64(fileInfo.Size()))
	f.mode = uint32(fileInfo.Mode())

	// Select and connect to hosts.
	totalsize := up.PieceSize * uint64(up.ErasureCode.NumPieces()) * f.numChunks()
	var hosts []uploader
	randHosts := r.hostDB.RandomHosts(up.ErasureCode.NumPieces())
	for i := range randHosts {
		hostUploader, err := r.newHostUploader(randHosts[i], totalsize, up.Duration, f.masterKey)
		if err != nil {
			continue
		}
		defer hostUploader.Close()
		hosts = append(hosts, hostUploader)
	}
	if len(hosts) < up.ErasureCode.MinPieces() {
		return errors.New("not enough hosts to support upload")
	}

	// Add file to renter.
	lockID = r.mu.Lock()
	r.files[up.Nickname] = f
	r.save()
	r.mu.Unlock(lockID)

	// Upload in parallel.
	err = f.upload(handle, hosts)
	if err != nil {
		// Upload failed; remove the file object.
		lockID = r.mu.Lock()
		delete(r.files, up.Nickname)
		r.save()
		r.mu.Unlock(lockID)
		return errors.New("failed to upload any file pieces")
	}

	// Add file to repair set.
	lockID = r.mu.Lock()
	r.repairing[up.Nickname] = up.Filename
	r.mu.Unlock(lockID)

	// Save the .sia file to the renter directory.
	err = r.saveFile(f)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>save after adding to repair set<commit_after>package renter

import (
	"errors"
	"io"
	"os"
	"sync"
	"sync/atomic"

	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

const (
	defaultDuration     = 6000 // Duration that hosts will hold onto the file
	defaultDataPieces   = 2    // Data pieces per erasure-coded chunk
	defaultParityPieces = 10   // Parity pieces per erasure-coded chunk

	// piece sizes
	// NOTE: The encryption overhead is subtracted so that encrypted piece
	// will always be a multiple of 64 (i.e. crypto.SegmentSize). Without this
	// property, revisions break the file's Merkle root.
	defaultPieceSize = 1<<22 - crypto.TwofishOverhead // 4 MiB
	smallPieceSize   = 1<<16 - crypto.TwofishOverhead // 64 KiB
)

type uploadPiece struct {
	data       []byte
	chunkIndex uint64
	pieceIndex uint64
}

// An uploader uploads pieces to a host. This interface exists to facilitate
// easy testing.
type uploader interface {
	// addPiece uploads a piece to the uploader.
	addPiece(uploadPiece) error

	// fileContract returns the fileContract containing the metadata of all
	// previously added pieces.
	fileContract() fileContract
}

// upload reads chunks from r and uploads them to hosts. It spawns a worker
// for each host, and instructs them to upload pieces of each chunk.
func (f *file) upload(r io.Reader, hosts []uploader) error {
	// encode and upload each chunk
	var wg sync.WaitGroup
	for i := uint64(0); ; i++ {
		// read next chunk
		chunk := make([]byte, f.chunkSize())
		_, err := io.ReadFull(r, chunk)
		if err == io.EOF {
			break
		} else if err != nil && err != io.ErrUnexpectedEOF {
			return err
		}
		// encode
		pieces, err := f.erasureCode.Encode(chunk)
		if err != nil {
			return err
		}
		// upload pieces, split evenly among hosts
		wg.Add(len(pieces))
		for j, data := range pieces {
			go func(j int, data []byte) {
				err := hosts[j%len(hosts)].addPiece(uploadPiece{data, i, uint64(j)})
				if err == nil {
					atomic.AddUint64(&f.bytesUploaded, uint64(len(data)))
				}
				wg.Done()
			}(j, data)
		}
		wg.Wait()
		atomic.AddUint64(&f.chunksUploaded, 1)

		// update contracts
		for _, h := range hosts {
			contract := h.fileContract()
			f.contracts[contract.IP] = contract
		}
	}

	return nil
}

// checkWalletBalance looks at an upload and determines if there is enough
// money in the wallet to support such an upload. An error is returned if it is
// determined that there is not enough money.
func (r *Renter) checkWalletBalance(up modules.FileUploadParams) error {
	// Get the size of the file.
	fileInfo, err := os.Stat(up.Filename)
	if err != nil {
		return err
	}
	curSize := types.NewCurrency64(uint64(fileInfo.Size()))

	var averagePrice types.Currency
	sampleSize := up.ErasureCode.NumPieces() * 3 / 2
	hosts := r.hostDB.RandomHosts(sampleSize)
	for _, host := range hosts {
		averagePrice = averagePrice.Add(host.Price)
	}
	if len(hosts) == 0 {
		return errors.New("no hosts!")
	}
	averagePrice = averagePrice.Div(types.NewCurrency64(uint64(len(hosts))))
	estimatedCost := averagePrice.Mul(types.NewCurrency64(uint64(up.Duration))).Mul(curSize)
	bufferedCost := estimatedCost.Mul(types.NewCurrency64(2))

	siacoinBalance, _, _ := r.wallet.ConfirmedBalance()
	if bufferedCost.Cmp(siacoinBalance) > 0 {
		return errors.New("insufficient balance for upload")
	}
	return nil
}

// Upload takes an upload parameters, which contain a file to upload, and then
// creates a redundant copy of the file on the Sia network.
func (r *Renter) Upload(up modules.FileUploadParams) error {
	// Open the file.
	handle, err := os.Open(up.Filename)
	if err != nil {
		return err
	}
	defer handle.Close()

	// Check for a nickname conflict.
	lockID := r.mu.RLock()
	_, exists := r.files[up.Nickname]
	r.mu.RUnlock(lockID)
	if exists {
		return errors.New("file with that nickname already exists")
	}

	// Check that the file is less than 5 GiB.
	fileInfo, err := handle.Stat()
	if err != nil {
		return err
	}
	// NOTE: The upload max of 5 GiB is temporary and therefore does not have
	// a constant. This should be removed once micropayments + upload resuming
	// are in place. 5 GiB is chosen to prevent confusion - on anybody's
	// machine any file appearing to be under 5 GB will be below the hard
	// limit.
	if fileInfo.Size() > 5*1024*1024*1024 {
		return errors.New("cannot upload a file larger than 5 GB")
	}

	// Fill in any missing upload params with sensible defaults.
	if up.Duration == 0 {
		up.Duration = defaultDuration
	}
	if up.ErasureCode == nil {
		up.ErasureCode, _ = NewRSCode(defaultDataPieces, defaultParityPieces)
	}
	if up.PieceSize == 0 {
		if fileInfo.Size() > defaultPieceSize {
			up.PieceSize = defaultPieceSize
		} else {
			up.PieceSize = smallPieceSize
		}
	}

	// Check that we have enough money to finance the upload.
	err = r.checkWalletBalance(up)
	if err != nil {
		return err
	}

	// Create file object.
	f := newFile(up.Nickname, up.ErasureCode, up.PieceSize, uint64(fileInfo.Size()))
	f.mode = uint32(fileInfo.Mode())

	// Select and connect to hosts.
	totalsize := up.PieceSize * uint64(up.ErasureCode.NumPieces()) * f.numChunks()
	var hosts []uploader
	randHosts := r.hostDB.RandomHosts(up.ErasureCode.NumPieces())
	for i := range randHosts {
		hostUploader, err := r.newHostUploader(randHosts[i], totalsize, up.Duration, f.masterKey)
		if err != nil {
			continue
		}
		defer hostUploader.Close()
		hosts = append(hosts, hostUploader)
	}
	if len(hosts) < up.ErasureCode.MinPieces() {
		return errors.New("not enough hosts to support upload")
	}

	// Add file to renter.
	lockID = r.mu.Lock()
	r.files[up.Nickname] = f
	r.save()
	r.mu.Unlock(lockID)

	// Upload in parallel.
	err = f.upload(handle, hosts)
	if err != nil {
		// Upload failed; remove the file object.
		lockID = r.mu.Lock()
		delete(r.files, up.Nickname)
		r.save()
		r.mu.Unlock(lockID)
		return errors.New("failed to upload any file pieces")
	}

	// Add file to repair set.
	lockID = r.mu.Lock()
	r.repairing[up.Nickname] = up.Filename
	r.save()
	r.mu.Unlock(lockID)

	// Save the .sia file to the renter directory.
	err = r.saveFile(f)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package cache

import (
	"bytes"
	"crypto/rand"
	"io/ioutil"
	"os"
	"testing"
	"time"
)

func TestCacheMetrics(t *testing.T) {
	m1 := CacheMetrics{}
	m2 := CacheMetrics{1, 1, 1, 1, 1, 1, 1, 1, 1}
	m1.Add(m2)
	if m1 != m2 {
		t.Fatal("not equal", m1, m2)
	}
}

func TestShardSetGet(t *testing.T) {
	dir, err := ioutil.TempDir("", "test_cachedata")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir)
	s, err := LoadCacheShard(dir, &ShardOptions{Size: 1024, TTL: 1})
	if err != nil {
		t.Fatal(err)
	}
	b := make([]byte, 300)
	rand.Read(b)

	_, err = s.Get("k1")
	if err != ErrNotFound {
		t.Fatal("should not found")
	}
	if err := s.Set(Item{Key: "k1", Value: b}); err != nil {
		t.Fatal(err)
	}
	ci, err := s.Get("k1")
	if err != nil {
		t.Fatal(err)
	}
	if ci.Key != "k1" {
		t.Fatal("key not equal")
	}
	if !bytes.Equal(ci.Value, b) {
		t.Fatal("set get not equal")
	}

	var st gcstat

	if testing.Short() {
		goto EndOfTest
	}

	s.Set(Item{Key: "k2", Value: b})
	s.Set(Item{Key: "k3", Value: b})
	s.Set(Item{Key: "k4", Value: b})

	st.LastKey = ""
	s.scanKeysForGC(100, &st)

	time.Sleep(1020 * time.Millisecond)

	st.LastKey = ""
	s.scanKeysForGC(100, &st)

	for _, key := range []string{"k1", "k2", "k3", "k4"} {
		if _, err := s.Get(key); err != ErrNotFound {
			t.Fatal("should not found")
		}
	}
	{
		m1 := s.GetMetrics()
		m2 := CacheMetrics{6, 1, 5, 0, 4, 0, 3, 1, 0}
		if m1 != m2 {
			t.Logf("\nget %+v\nexpect %+v", m1, m2)
			t.Fatal("metrics err")
		}
	}

EndOfTest:
	if err := s.Close(); err != nil {
		t.Fatal(err)
	}
}
<commit_msg>cache: test: cleanup files<commit_after>package cache

import (
	"bytes"
	"crypto/rand"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"
	"time"
)

func TestCacheMetrics(t *testing.T) {
	m1 := CacheMetrics{}
	m2 := CacheMetrics{1, 1, 1, 1, 1, 1, 1, 1, 1}
	m1.Add(m2)
	if m1 != m2 {
		t.Fatal("not equal", m1, m2)
	}
}

func TestShardSetGet(t *testing.T) {
	dir, err := ioutil.TempDir("", "test_cachedata")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir)
	s, err := LoadCacheShard(filepath.Join(dir, "shard"), &ShardOptions{Size: 1024, TTL: 1})
	if err != nil {
		t.Fatal(err)
	}
	b := make([]byte, 300)
	rand.Read(b)

	_, err = s.Get("k1")
	if err != ErrNotFound {
		t.Fatal("should not found")
	}
	if err := s.Set(Item{Key: "k1", Value: b}); err != nil {
		t.Fatal(err)
	}
	ci, err := s.Get("k1")
	if err != nil {
		t.Fatal(err)
	}
	if ci.Key != "k1" {
		t.Fatal("key not equal")
	}
	if !bytes.Equal(ci.Value, b) {
		t.Fatal("set get not equal")
	}

	var st gcstat

	if testing.Short() {
		goto EndOfTest
	}

	s.Set(Item{Key: "k2", Value: b})
	s.Set(Item{Key: "k3", Value: b})
	s.Set(Item{Key: "k4", Value: b})

	st.LastKey = ""
	s.scanKeysForGC(100, &st)

	time.Sleep(1020 * time.Millisecond)

	st.LastKey = ""
	s.scanKeysForGC(100, &st)

	for _, key := range []string{"k1", "k2", "k3", "k4"} {
		if _, err := s.Get(key); err != ErrNotFound {
			t.Fatal("should not found")
		}
	}
	{
		m1 := s.GetMetrics()
		m2 := CacheMetrics{6, 1, 5, 0, 4, 0, 3, 1, 0}
		if m1 != m2 {
			t.Logf("\nget %+v\nexpect %+v", m1, m2)
			t.Fatal("metrics err")
		}
	}

EndOfTest:
	if err := s.Close(); err != nil {
		t.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package govaluate

// sanitizedParameters is a wrapper for Parameters that does sanitization as
// parameters are accessed.
type sanitizedParameters struct {
	orig Parameters
}

func (p sanitizedParameters) Get(key string) (interface{}, error) {
	value, err := p.orig.Get(key)
	if err != nil {
		return nil, err
	}

	// should be converted to fixed point?
	return castToFloat64(value), nil
}

func isFixedPoint(value interface{}) bool {

	switch value.(type) {
	case uint8:
		return true
	case uint16:
		return true
	case uint32:
		return true
	case uint64:
		return true
	case int8:
		return true
	case int16:
		return true
	case int32:
		return true
	case int64:
		return true
	case int:
		return true
	}
	return false
}

func castToFloat64(value interface{}) interface{} {
	switch value.(type) {
	case uint8:
		return float64(value.(uint8))
	case uint16:
		return float64(value.(uint16))
	case uint32:
		return float64(value.(uint32))
	case uint64:
		return float64(value.(uint64))
	case int8:
		return float64(value.(int8))
	case int16:
		return float64(value.(int16))
	case int32:
		return float64(value.(int32))
	case int64:
		return float64(value.(int64))
	case int:
		return float64(value.(int))
	case float32:
		return float64(value.(float32))
	}

	return value
}
<commit_msg>Removed unused isFixedPoint().<commit_after>package govaluate

// sanitizedParameters is a wrapper for Parameters that does sanitization as
// parameters are accessed.
type sanitizedParameters struct {
	orig Parameters
}

func (p sanitizedParameters) Get(key string) (interface{}, error) {
	value, err := p.orig.Get(key)
	if err != nil {
		return nil, err
	}

	return castToFloat64(value), nil
}

func castToFloat64(value interface{}) interface{} {
	switch value.(type) {
	case uint8:
		return float64(value.(uint8))
	case uint16:
		return float64(value.(uint16))
	case uint32:
		return float64(value.(uint32))
	case uint64:
		return float64(value.(uint64))
	case int8:
		return float64(value.(int8))
	case int16:
		return float64(value.(int16))
	case int32:
		return float64(value.(int32))
	case int64:
		return float64(value.(int64))
	case int:
		return float64(value.(int))
	case float32:
		return float64(value.(float32))
	}

	return value
}
<|endoftext|>