<commit_before>package slb

import (
	"fmt"
	"time"

	"github.com/denverdino/aliyungo/common"
	"github.com/denverdino/aliyungo/util"
)

type AddressType string

const (
	InternetAddressType = AddressType("internet")
	IntranetAddressType = AddressType("intranet")
)

type InternetChargeType string

const (
	PayByBandwidth = InternetChargeType("paybybandwidth")
	PayByTraffic   = InternetChargeType("paybytraffic")
)

type AddressIPVersionType string

const (
	IPv4 = AddressIPVersionType("ipv4")
	IPv6 = AddressIPVersionType("ipv6")
)

type LoadBalancerSpecType string

const (
	S1Small  = "slb.s1.small"
	S2Small  = "slb.s2.small"
	S2Medium = "slb.s2.medium"
	S3Small  = "slb.s3.small"
	S3Medium = "slb.s3.medium"
	S3Large  = "slb.s3.large"
)

type ModificationProtectionType string

const (
	NonProtection     = ModificationProtectionType("NonProtection")
	ConsoleProtection = ModificationProtectionType("ConsoleProtection")
)

type CreateLoadBalancerArgs struct {
	RegionId                     common.Region
	LoadBalancerName             string
	AddressType                  AddressType
	VSwitchId                    string
	InternetChargeType           InternetChargeType
	Bandwidth                    int
	ClientToken                  string
	MasterZoneId                 string
	SlaveZoneId                  string
	LoadBalancerSpec             LoadBalancerSpecType
	AddressIPVersion             AddressIPVersionType
	DeleteProtection             FlagType
	ModificationProtectionStatus ModificationProtectionType
	ModificationProtectionReason string
	ResourceGroupId              string
}

type CreateLoadBalancerResponse struct {
	common.Response
	LoadBalancerId   string
	Address          string
	NetworkType      string
	VpcId            string
	VSwitchId        string
	LoadBalancerName string
	MasterZoneId     string
	SlaveZoneId      string
	AddressIPVersion AddressIPVersionType
}

// CreateLoadBalancer create loadbalancer
//
// You can read doc at http://docs.aliyun.com/#/pub/slb/api-reference/api-related-loadbalancer&CreateLoadBalancer
func (client *Client) CreateLoadBalancer(args *CreateLoadBalancerArgs) (response *CreateLoadBalancerResponse, err error) {
	response = &CreateLoadBalancerResponse{}
	err = client.Invoke("CreateLoadBalancer", args, response)
	if err != nil {
		return nil, err
	}
	return response, err
}

type DeleteLoadBalancerArgs struct {
	LoadBalancerId string
}

type DeleteLoadBalancerResponse struct {
	common.Response
}

// DeleteLoadBalancer delete loadbalancer
//
// You can read doc at http://docs.aliyun.com/#/pub/slb/api-reference/api-related-loadbalancer&DeleteLoadBalancer
func (client *Client) DeleteLoadBalancer(loadBalancerId string) (err error) {
	args := &DeleteLoadBalancerArgs{
		LoadBalancerId: loadBalancerId,
	}
	response := &DeleteLoadBalancerResponse{}
	err = client.Invoke("DeleteLoadBalancer", args, response)
	return err
}

type ModifyLoadBalancerInternetSpecArgs struct {
	LoadBalancerId     string
	InternetChargeType InternetChargeType
	Bandwidth          int
}

type ModifyLoadBalancerInternetSpecResponse struct {
	common.Response
}

// ModifyLoadBalancerInternetSpec Modify loadbalancer internet spec
//
// You can read doc at http://docs.aliyun.com/#/pub/slb/api-reference/api-related-loadbalancer&ModifyLoadBalancerInternetSpec

func (client *Client) ModifyLoadBalancerInternetSpec(args *ModifyLoadBalancerInternetSpecArgs) (err error) {
	response := &ModifyLoadBalancerInternetSpecResponse{}
	err = client.Invoke("ModifyLoadBalancerInternetSpec", args, response)
	return err
}

type ModifyLoadBalancerInstanceSpecArgs struct {
	RegionId         common.Region
	LoadBalancerId   string
	LoadBalancerSpec LoadBalancerSpecType
}

// ModifyLoadBalancerInstanceSpec Modify loadbalancer instance spec
//
// You can read doc at https://help.aliyun.com/document_detail/53360.html

func (client *Client) ModifyLoadBalancerInstanceSpec(args *ModifyLoadBalancerInstanceSpecArgs) (err error) {
	response := &common.Response{}
	err = client.Invoke("ModifyLoadBalancerInstanceSpec", args, response)
	return err
}

type Status string

const InactiveStatus = Status("inactive")
const ActiveStatus = Status("active")
const LockedStatus = Status("locked")

type SetLoadBalancerStatusArgs struct {
	LoadBalancerId     string
	LoadBalancerStatus Status
}

type SetLoadBalancerStatusResponse struct {
	common.Response
}

// SetLoadBalancerStatus Set loadbalancer status
//
// You can read doc at http://docs.aliyun.com/#/pub/slb/api-reference/api-related-loadbalancer&SetLoadBalancerStatus

func (client *Client) SetLoadBalancerStatus(loadBalancerId string, status Status) (err error) {
	args := &SetLoadBalancerStatusArgs{
		LoadBalancerId:     loadBalancerId,
		LoadBalancerStatus: status,
	}
	response := &SetLoadBalancerStatusResponse{}
	err = client.Invoke("SetLoadBalancerStatus", args, response)
	return err
}

type SetLoadBalancerNameArgs struct {
	LoadBalancerId   string
	LoadBalancerName string
}

type SetLoadBalancerNameResponse struct {
	common.Response
}

// SetLoadBalancerName Set loadbalancer name
//
// You can read doc at http://docs.aliyun.com/#/pub/slb/api-reference/api-related-loadbalancer&SetLoadBalancerName

func (client *Client) SetLoadBalancerName(loadBalancerId string, name string) (err error) {
	args := &SetLoadBalancerNameArgs{
		LoadBalancerId:   loadBalancerId,
		LoadBalancerName: name,
	}
	response := &SetLoadBalancerNameResponse{}
	err = client.Invoke("SetLoadBalancerName", args, response)
	return err
}

type DescribeLoadBalancersArgs struct {
	RegionId           common.Region
	LoadBalancerId     string
	LoadBalancerName   string
	AddressType        AddressType
	NetworkType        string
	VpcId              string
	VSwitchId          string
	Address            string
	InternetChargeType InternetChargeType
	ServerId           string
	Tags               string
}

type ListenerPortAndProtocolType struct {
	ListenerPort     int
	ListenerProtocol string
	Description      string
}

type BackendServerType struct {
	ServerId string
	Weight   int
	Type     string
}

type LoadBalancerType struct {
	LoadBalancerId               string
	LoadBalancerName             string
	LoadBalancerStatus           string
	Address                      string
	RegionId                     common.Region
	RegionIdAlias                string
	AddressType                  AddressType
	VSwitchId                    string
	VpcId                        string
	NetworkType                  string
	Bandwidth                    int
	InternetChargeType           InternetChargeType
	CreateTime                   string //Why not ISO 6801
	CreateTimeStamp              util.ISO6801Time
	DeleteProtection             FlagType
	ModificationProtectionStatus ModificationProtectionType
	ModificationProtectionReason string
	ListenerPorts                struct {
		ListenerPort []int
	}
	ListenerPortsAndProtocol struct {
		ListenerPortAndProtocol []ListenerPortAndProtocolType
	}
	BackendServers struct {
		BackendServer []BackendServerType
	}
	LoadBalancerSpec LoadBalancerSpecType
	MasterZoneId     string
	SlaveZoneId      string
	AddressIPVersion AddressIPVersionType
}

type DescribeLoadBalancersResponse struct {
	common.Response
	LoadBalancers struct {
		LoadBalancer []LoadBalancerType
	}
}

// DescribeLoadBalancers Describe loadbalancers
//
// You can read doc at http://docs.aliyun.com/#/pub/slb/api-reference/api-related-loadbalancer&DescribeLoadBalancers

func (client *Client) DescribeLoadBalancers(args *DescribeLoadBalancersArgs) (loadBalancers []LoadBalancerType, err error) {
	response := &DescribeLoadBalancersResponse{}
	err = client.Invoke("DescribeLoadBalancers", args, response)
	if err != nil {
		return nil, err
	}
	return response.LoadBalancers.LoadBalancer, err
}

type DescribeLoadBalancerAttributeArgs struct {
	LoadBalancerId string
}

type DescribeLoadBalancerAttributeResponse struct {
	common.Response
	LoadBalancerType
}

// DescribeLoadBalancerAttribute Describe loadbalancer attribute
//
// You can read doc at http://docs.aliyun.com/#/pub/slb/api-reference/api-related-loadbalancer&DescribeLoadBalancerAttribute

func (client *Client) DescribeLoadBalancerAttribute(loadBalancerId string) (loadBalancer *LoadBalancerType, err error) {
	args := &DescribeLoadBalancersArgs{
		LoadBalancerId: loadBalancerId,
	}
	response := &DescribeLoadBalancerAttributeResponse{}
	err = client.Invoke("DescribeLoadBalancerAttribute", args, response)
	if err != nil {
		return nil, err
	}
	return &response.LoadBalancerType, err
}

type NewDescribeLoadBalancerAttributeArgs struct {
	LoadBalancerId string
	RegionId       common.Region
	MasterZoneId   string
	SlaveZoneId    string
}

// New DescribeLoadBalancerAttribute to describe loadbalancer attribute using regionId avoiding to get not found error
//
// You can read doc at https://www.alibabacloud.com/help/doc-detail/27583.htm

func (client *Client) NewDescribeLoadBalancerAttribute(args *NewDescribeLoadBalancerAttributeArgs) (loadBalancer *LoadBalancerType, err error) {

	response := &DescribeLoadBalancerAttributeResponse{}
	err = client.Invoke("DescribeLoadBalancerAttribute", args, response)
	if err != nil {
		return nil, err
	}
	return &response.LoadBalancerType, err
}

// WaitForListener waits for listener to given status
func (client *Client) WaitForLoadBalancerAsyn(loadBalancerId string, status Status, timeout int) error {
	if timeout <= 0 {
		timeout = DefaultTimeout
	}

	for {
		lb, err := client.DescribeLoadBalancerAttribute(loadBalancerId)

		e, _ := err.(*common.Error)
		if e != nil {
			if e.StatusCode == 404 || e.Code == "InvalidLoadBalancerId.NotFound" {
				continue
			}
			return err
		} else if lb != nil && Status(lb.LoadBalancerStatus) == status {
			//TODO
			break
		}
		timeout = timeout - DefaultWaitForInterval
		if timeout <= 0 {
			return common.GetClientErrorFromString(fmt.Sprintf("Timeout waitting for load balacner %#v", status))
		}
		time.Sleep(DefaultWaitForInterval * time.Second)
	}
	return nil
}

type SetLoadBalancerDeleteProtectionArgs struct {
	LoadBalancerId   string
	DeleteProtection FlagType
	RegionId         common.Region
}

type SetLoadBalancerDeleteProtectionResponse struct {
	common.Response
}

// SetLoadBalancerDeleteProtection loadbalancer delete protection
//
// You can read doc at https://help.aliyun.com/document_detail/122674.html?spm=a2c4g.11186623.6.720.694f4265hwOdXQ
func (client *Client) SetLoadBalancerDeleteProtection(args *SetLoadBalancerDeleteProtectionArgs) (err error) {
	response := &SetLoadBalancerDeleteProtectionResponse{}
	err = client.Invoke("SetLoadBalancerDeleteProtection", args, response)
	return err
}

type SetLoadBalancerModificationProtectionArgs struct {
	RegionId                     common.Region
	LoadBalancerId               string
	ModificationProtectionStatus ModificationProtectionType
	ModificationProtectionReason string
}

type SetLoadBalancerModificationProtectionResponse struct {
	common.Response
}

func (client *Client) SetLoadBalancerModificationProtection(args *SetLoadBalancerModificationProtectionArgs) (err error) {
	response := &SetLoadBalancerModificationProtectionResponse{}
	err = client.Invoke("SetLoadBalancerModificationProtection", args, response)
	return err
}
<commit_msg>add attr ResourceGroupId for struct LoadBalancerType<commit_after>package slb

import (
	"fmt"
	"time"

	"github.com/denverdino/aliyungo/common"
	"github.com/denverdino/aliyungo/util"
)

type AddressType string

const (
	InternetAddressType = AddressType("internet")
	IntranetAddressType = AddressType("intranet")
)

type InternetChargeType string

const (
	PayByBandwidth = InternetChargeType("paybybandwidth")
	PayByTraffic   = InternetChargeType("paybytraffic")
)

type AddressIPVersionType string

const (
	IPv4 = AddressIPVersionType("ipv4")
	IPv6 = AddressIPVersionType("ipv6")
)

type LoadBalancerSpecType string

const (
	S1Small  = "slb.s1.small"
	S2Small  = "slb.s2.small"
	S2Medium = "slb.s2.medium"
	S3Small  = "slb.s3.small"
	S3Medium = "slb.s3.medium"
	S3Large  = "slb.s3.large"
)

type ModificationProtectionType string

const (
	NonProtection     = ModificationProtectionType("NonProtection")
	ConsoleProtection = ModificationProtectionType("ConsoleProtection")
)

type CreateLoadBalancerArgs struct {
	RegionId                     common.Region
	LoadBalancerName             string
	AddressType                  AddressType
	VSwitchId                    string
	InternetChargeType           InternetChargeType
	Bandwidth                    int
	ClientToken                  string
	MasterZoneId                 string
	SlaveZoneId                  string
	LoadBalancerSpec             LoadBalancerSpecType
	AddressIPVersion             AddressIPVersionType
	DeleteProtection             FlagType
	ModificationProtectionStatus ModificationProtectionType
	ModificationProtectionReason string
	ResourceGroupId              string
}

type CreateLoadBalancerResponse struct {
	common.Response
	LoadBalancerId   string
	Address          string
	NetworkType      string
	VpcId            string
	VSwitchId        string
	LoadBalancerName string
	MasterZoneId     string
	SlaveZoneId      string
	AddressIPVersion AddressIPVersionType
}

// CreateLoadBalancer create loadbalancer
//
// You can read doc at http://docs.aliyun.com/#/pub/slb/api-reference/api-related-loadbalancer&CreateLoadBalancer
func (client *Client) CreateLoadBalancer(args *CreateLoadBalancerArgs) (response *CreateLoadBalancerResponse, err error) {
	response = &CreateLoadBalancerResponse{}
	err = client.Invoke("CreateLoadBalancer", args, response)
	if err != nil {
		return nil, err
	}
	return response, err
}

type DeleteLoadBalancerArgs struct {
	LoadBalancerId string
}

type DeleteLoadBalancerResponse struct {
	common.Response
}

// DeleteLoadBalancer delete loadbalancer
//
// You can read doc at http://docs.aliyun.com/#/pub/slb/api-reference/api-related-loadbalancer&DeleteLoadBalancer
func (client *Client) DeleteLoadBalancer(loadBalancerId string) (err error) {
	args := &DeleteLoadBalancerArgs{
		LoadBalancerId: loadBalancerId,
	}
	response := &DeleteLoadBalancerResponse{}
	err = client.Invoke("DeleteLoadBalancer", args, response)
	return err
}

type ModifyLoadBalancerInternetSpecArgs struct {
	LoadBalancerId     string
	InternetChargeType InternetChargeType
	Bandwidth          int
}

type ModifyLoadBalancerInternetSpecResponse struct {
	common.Response
}

// ModifyLoadBalancerInternetSpec Modify loadbalancer internet spec
//
// You can read doc at http://docs.aliyun.com/#/pub/slb/api-reference/api-related-loadbalancer&ModifyLoadBalancerInternetSpec

func (client *Client) ModifyLoadBalancerInternetSpec(args *ModifyLoadBalancerInternetSpecArgs) (err error) {
	response := &ModifyLoadBalancerInternetSpecResponse{}
	err = client.Invoke("ModifyLoadBalancerInternetSpec", args, response)
	return err
}

type ModifyLoadBalancerInstanceSpecArgs struct {
	RegionId         common.Region
	LoadBalancerId   string
	LoadBalancerSpec LoadBalancerSpecType
}

// ModifyLoadBalancerInstanceSpec Modify loadbalancer instance spec
//
// You can read doc at https://help.aliyun.com/document_detail/53360.html

func (client *Client) ModifyLoadBalancerInstanceSpec(args *ModifyLoadBalancerInstanceSpecArgs) (err error) {
	response := &common.Response{}
	err = client.Invoke("ModifyLoadBalancerInstanceSpec", args, response)
	return err
}

type Status string

const InactiveStatus = Status("inactive")
const ActiveStatus = Status("active")
const LockedStatus = Status("locked")

type SetLoadBalancerStatusArgs struct {
	LoadBalancerId     string
	LoadBalancerStatus Status
}

type SetLoadBalancerStatusResponse struct {
	common.Response
}

// SetLoadBalancerStatus Set loadbalancer status
//
// You can read doc at http://docs.aliyun.com/#/pub/slb/api-reference/api-related-loadbalancer&SetLoadBalancerStatus

func (client *Client) SetLoadBalancerStatus(loadBalancerId string, status Status) (err error) {
	args := &SetLoadBalancerStatusArgs{
		LoadBalancerId:     loadBalancerId,
		LoadBalancerStatus: status,
	}
	response := &SetLoadBalancerStatusResponse{}
	err = client.Invoke("SetLoadBalancerStatus", args, response)
	return err
}

type SetLoadBalancerNameArgs struct {
	LoadBalancerId   string
	LoadBalancerName string
}

type SetLoadBalancerNameResponse struct {
	common.Response
}

// SetLoadBalancerName Set loadbalancer name
//
// You can read doc at http://docs.aliyun.com/#/pub/slb/api-reference/api-related-loadbalancer&SetLoadBalancerName

func (client *Client) SetLoadBalancerName(loadBalancerId string, name string) (err error) {
	args := &SetLoadBalancerNameArgs{
		LoadBalancerId:   loadBalancerId,
		LoadBalancerName: name,
	}
	response := &SetLoadBalancerNameResponse{}
	err = client.Invoke("SetLoadBalancerName", args, response)
	return err
}

type DescribeLoadBalancersArgs struct {
	RegionId           common.Region
	LoadBalancerId     string
	LoadBalancerName   string
	AddressType        AddressType
	NetworkType        string
	VpcId              string
	VSwitchId          string
	Address            string
	InternetChargeType InternetChargeType
	ServerId           string
	Tags               string
}

type ListenerPortAndProtocolType struct {
	ListenerPort     int
	ListenerProtocol string
	Description      string
}

type BackendServerType struct {
	ServerId string
	Weight   int
	Type     string
}

type LoadBalancerType struct {
	LoadBalancerId               string
	ResourceGroupId              string
	LoadBalancerName             string
	LoadBalancerStatus           string
	Address                      string
	RegionId                     common.Region
	RegionIdAlias                string
	AddressType                  AddressType
	VSwitchId                    string
	VpcId                        string
	NetworkType                  string
	Bandwidth                    int
	InternetChargeType           InternetChargeType
	CreateTime                   string //Why not ISO 6801
	CreateTimeStamp              util.ISO6801Time
	DeleteProtection             FlagType
	ModificationProtectionStatus ModificationProtectionType
	ModificationProtectionReason string
	ListenerPorts                struct {
		ListenerPort []int
	}
	ListenerPortsAndProtocol struct {
		ListenerPortAndProtocol []ListenerPortAndProtocolType
	}
	BackendServers struct {
		BackendServer []BackendServerType
	}
	LoadBalancerSpec LoadBalancerSpecType
	MasterZoneId     string
	SlaveZoneId      string
	AddressIPVersion AddressIPVersionType
}

type DescribeLoadBalancersResponse struct {
	common.Response
	LoadBalancers struct {
		LoadBalancer []LoadBalancerType
	}
}

// DescribeLoadBalancers Describe loadbalancers
//
// You can read doc at http://docs.aliyun.com/#/pub/slb/api-reference/api-related-loadbalancer&DescribeLoadBalancers

func (client *Client) DescribeLoadBalancers(args *DescribeLoadBalancersArgs) (loadBalancers []LoadBalancerType, err error) {
	response := &DescribeLoadBalancersResponse{}
	err = client.Invoke("DescribeLoadBalancers", args, response)
	if err != nil {
		return nil, err
	}
	return response.LoadBalancers.LoadBalancer, err
}

type DescribeLoadBalancerAttributeArgs struct {
	LoadBalancerId string
}

type DescribeLoadBalancerAttributeResponse struct {
	common.Response
	LoadBalancerType
}

// DescribeLoadBalancerAttribute Describe loadbalancer attribute
//
// You can read doc at http://docs.aliyun.com/#/pub/slb/api-reference/api-related-loadbalancer&DescribeLoadBalancerAttribute

func (client *Client) DescribeLoadBalancerAttribute(loadBalancerId string) (loadBalancer *LoadBalancerType, err error) {
	args := &DescribeLoadBalancersArgs{
		LoadBalancerId: loadBalancerId,
	}
	response := &DescribeLoadBalancerAttributeResponse{}
	err = client.Invoke("DescribeLoadBalancerAttribute", args, response)
	if err != nil {
		return nil, err
	}
	return &response.LoadBalancerType, err
}

type NewDescribeLoadBalancerAttributeArgs struct {
	LoadBalancerId string
	RegionId       common.Region
	MasterZoneId   string
	SlaveZoneId    string
}

// New DescribeLoadBalancerAttribute to describe loadbalancer attribute using regionId avoiding to get not found error
//
// You can read doc at https://www.alibabacloud.com/help/doc-detail/27583.htm

func (client *Client) NewDescribeLoadBalancerAttribute(args *NewDescribeLoadBalancerAttributeArgs) (loadBalancer *LoadBalancerType, err error) {

	response := &DescribeLoadBalancerAttributeResponse{}
	err = client.Invoke("DescribeLoadBalancerAttribute", args, response)
	if err != nil {
		return nil, err
	}
	return &response.LoadBalancerType, err
}

// WaitForListener waits for listener to given status
func (client *Client) WaitForLoadBalancerAsyn(loadBalancerId string, status Status, timeout int) error {
	if timeout <= 0 {
		timeout = DefaultTimeout
	}

	for {
		lb, err := client.DescribeLoadBalancerAttribute(loadBalancerId)

		e, _ := err.(*common.Error)
		if e != nil {
			if e.StatusCode == 404 || e.Code == "InvalidLoadBalancerId.NotFound" {
				continue
			}
			return err
		} else if lb != nil && Status(lb.LoadBalancerStatus) == status {
			//TODO
			break
		}
		timeout = timeout - DefaultWaitForInterval
		if timeout <= 0 {
			return common.GetClientErrorFromString(fmt.Sprintf("Timeout waitting for load balacner %#v", status))
		}
		time.Sleep(DefaultWaitForInterval * time.Second)
	}
	return nil
}

type SetLoadBalancerDeleteProtectionArgs struct {
	LoadBalancerId   string
	DeleteProtection FlagType
	RegionId         common.Region
}

type SetLoadBalancerDeleteProtectionResponse struct {
	common.Response
}

// SetLoadBalancerDeleteProtection loadbalancer delete protection
//
// You can read doc at https://help.aliyun.com/document_detail/122674.html?spm=a2c4g.11186623.6.720.694f4265hwOdXQ
func (client *Client) SetLoadBalancerDeleteProtection(args *SetLoadBalancerDeleteProtectionArgs) (err error) {
	response := &SetLoadBalancerDeleteProtectionResponse{}
	err = client.Invoke("SetLoadBalancerDeleteProtection", args, response)
	return err
}

type SetLoadBalancerModificationProtectionArgs struct {
	RegionId                     common.Region
	LoadBalancerId               string
	ModificationProtectionStatus ModificationProtectionType
	ModificationProtectionReason string
}

type SetLoadBalancerModificationProtectionResponse struct {
	common.Response
}

func (client *Client) SetLoadBalancerModificationProtection(args *SetLoadBalancerModificationProtectionArgs) (err error) {
	response := &SetLoadBalancerModificationProtectionResponse{}
	err = client.Invoke("SetLoadBalancerModificationProtection", args, response)
	return err
}
<|endoftext|>


<commit_before>package storageengines

import (
	"bytes"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"

	"github.com/BurntSushi/toml"
	"github.com/gillesdemey/npm-registry/model"
)

type FSStorage struct {
	Folder string
}

func NewFSStorage(folder string) *FSStorage {
	engine := &FSStorage{Folder: folder}
	engine.initialize()
	return engine
}

func (s *FSStorage) initialize() {
	os.MkdirAll(s.Folder, os.ModePerm)
	return
}

func (s *FSStorage) StoreTarball(pkg string, filename string, reader io.Reader) error {
	tarballPath := filepath.Join(s.Folder, "tarballs", pkg)
	tarballLocation := filepath.Join(tarballPath, filename)

	if _, err := os.Stat(tarballLocation); err == nil {
		log.WithFields(log.Fields{
			"tarball": filename,
		}).Info("Tarball already exists, skipping")
		return nil
	}

	// create subdirectories we need, ignore errors
	os.MkdirAll(tarballPath, os.ModePerm)

	tarball, err := os.Create(tarballLocation)
	if err != nil {
		return err
	}
	defer tarball.Close()

	io.Copy(tarball, reader)
	return nil
}

func (s *FSStorage) RetrieveTarball(pkg string, filename string, writer io.Writer) error {
	tarballLocation := filepath.Join(s.Folder, "tarballs", pkg, filename)

	tarball, err := os.Open(tarballLocation)
	if err != nil {
		return err
	}
	defer tarball.Close()

	io.Copy(writer, tarball)
	return nil
}

func (s *FSStorage) RetrieveUsernameFromToken(token string) (string, error) {
	tokenEntries := make(map[string]model.Token)
	tokenFile := filepath.Join(s.Folder, "tokens.toml")

	if _, err := toml.DecodeFile(tokenFile, &tokenEntries); err != nil {
		return "", err
	}
	tokenEntry := tokenEntries[token]

	return tokenEntry.Username, nil
}

func (s *FSStorage) StoreUser(pkg string) error {
	return nil
}

func (s *FSStorage) RetrieveUser(string, io.Writer) error {
	return nil
}

func (s *FSStorage) StoreUserToken(token string, username string) error {
	tokenEntry := make(map[string]model.Token, 1)
	tokenEntry[token] = model.Token{
		Username:  username,
		Timestamp: time.Now(),
	}

	entry := new(bytes.Buffer)
	if err := toml.NewEncoder(entry).Encode(tokenEntry); err != nil {
		return err
	}

	tokensFile := filepath.Join(s.Folder, "tokens.toml")

	if err := ioutil.WriteFile(tokensFile, entry.Bytes(), 0666); err != nil {
		return err
	}

	return nil
}

func (s *FSStorage) RetrieveMetadata(pkg string, writer io.Writer) error {
	metaFileName := fmt.Sprintf("%s.json", pkg)
	metaFileLocation := filepath.Join(s.Folder, "meta", metaFileName)

	metaFile, err := os.Open(metaFileLocation)
	if err != nil {
		return err
	}
	defer metaFile.Close()
	io.Copy(writer, metaFile)

	return nil
}

func (s *FSStorage) StoreMetadata(pkg string, data io.Reader) error {
	metaFilePath := filepath.Join(s.Folder, "meta")
	metaFileLocation := filepath.Join(metaFilePath, pkg+".json")

	// create subdirectories we need, ignore errors
	os.MkdirAll(metaFilePath, os.ModePerm)

	metaFile, err := os.Create(metaFileLocation)
	if err != nil {
		return err
	}
	defer metaFile.Close()
	io.Copy(metaFile, data)

	return nil
}
<commit_msg>Fixes error that prevented already cached packages from not properly responding with the package data<commit_after>package storageengines

import (
	"bytes"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"

	"github.com/BurntSushi/toml"
	"github.com/gillesdemey/npm-registry/model"
)

type FSStorage struct {
	Folder string
}

func NewFSStorage(folder string) *FSStorage {
	engine := &FSStorage{Folder: folder}
	engine.initialize()
	return engine
}

func (s *FSStorage) initialize() {
	os.MkdirAll(s.Folder, os.ModePerm)
	return
}

func (s *FSStorage) StoreTarball(pkg string, filename string, reader io.Reader) error {
	tarballPath := filepath.Join(s.Folder, "tarballs", pkg)
	tarballLocation := filepath.Join(tarballPath, filename)

	if _, err := os.Stat(tarballLocation); err == nil {
		log.WithFields(log.Fields{
			"tarball": filename,
		}).Info("Tarball already exists, skipping")
		// for some reason teeReader doesn't function properly if we don't consume
		// the reader, so send everything to /dev/null
		io.Copy(ioutil.Discard, reader)
		return nil
	}

	// create subdirectories we need, ignore errors
	os.MkdirAll(tarballPath, os.ModePerm)

	tarball, err := os.Create(tarballLocation)
	if err != nil {
		return err
	}
	defer tarball.Close()

	io.Copy(tarball, reader)
	return nil
}

func (s *FSStorage) RetrieveTarball(pkg string, filename string, writer io.Writer) error {
	tarballLocation := filepath.Join(s.Folder, "tarballs", pkg, filename)

	tarball, err := os.Open(tarballLocation)
	if err != nil {
		return err
	}
	defer tarball.Close()

	io.Copy(writer, tarball)
	return nil
}

func (s *FSStorage) RetrieveUsernameFromToken(token string) (string, error) {
	tokenEntries := make(map[string]model.Token)
	tokenFile := filepath.Join(s.Folder, "tokens.toml")

	if _, err := toml.DecodeFile(tokenFile, &tokenEntries); err != nil {
		return "", err
	}
	tokenEntry := tokenEntries[token]

	return tokenEntry.Username, nil
}

func (s *FSStorage) StoreUser(pkg string) error {
	return nil
}

func (s *FSStorage) RetrieveUser(string, io.Writer) error {
	return nil
}

func (s *FSStorage) StoreUserToken(token string, username string) error {
	tokenEntry := make(map[string]model.Token, 1)
	tokenEntry[token] = model.Token{
		Username:  username,
		Timestamp: time.Now(),
	}

	entry := new(bytes.Buffer)
	if err := toml.NewEncoder(entry).Encode(tokenEntry); err != nil {
		return err
	}

	tokensFile := filepath.Join(s.Folder, "tokens.toml")

	if err := ioutil.WriteFile(tokensFile, entry.Bytes(), 0666); err != nil {
		return err
	}

	return nil
}

func (s *FSStorage) RetrieveMetadata(pkg string, writer io.Writer) error {
	metaFileName := fmt.Sprintf("%s.json", pkg)
	metaFileLocation := filepath.Join(s.Folder, "meta", metaFileName)

	metaFile, err := os.Open(metaFileLocation)
	if err != nil {
		return err
	}
	defer metaFile.Close()
	io.Copy(writer, metaFile)

	return nil
}

func (s *FSStorage) StoreMetadata(pkg string, data io.Reader) error {
	metaFilePath := filepath.Join(s.Folder, "meta")
	metaFileLocation := filepath.Join(metaFilePath, pkg+".json")

	// create subdirectories we need, ignore errors
	os.MkdirAll(metaFilePath, os.ModePerm)

	metaFile, err := os.Create(metaFileLocation)
	if err != nil {
		return err
	}
	defer metaFile.Close()
	io.Copy(metaFile, data)

	return nil
}
<|endoftext|>


<commit_before>// storage/compressed.go
// Copyright(c) 2017 Matt Pharr
// BSD licensed; see LICENSE for details.

package storage

import (
	"bytes"
	"compress/gzip"
	u "github.com/mmp/bk/util"
	"io"
	"os"
	"sync"
	"time"
)

///////////////////////////////////////////////////////////////////////////
// compressed

// compressed implements the Backend interface. It applies gzip compression
// to the provided data before passing it along to another backend for
// storage.
type compressed struct {
	backend                            Backend
	bytesSaved, bytesProcessed         int64
	compressedBlobs, uncompressedBlobs int
}

// NewCompressed returns a new storage.Backend that applies gzip compression
// to the contents of blobs stored in the provided underlying backend.
// Note: the contents of metadata files are not compressed.
func NewCompressed(backend Backend) Backend {
	return &compressed{backend: backend}
}

func (c *compressed) String() string {
	return "gzip compressed " + c.backend.String()
}

func (c *compressed) LogStats() {
	tot := c.compressedBlobs + c.uncompressedBlobs
	if tot > 0 {
		log.Print("compressed %d / %d blobs (%2.f%%)",
			c.compressedBlobs, tot, 100.*float64(c.compressedBlobs)/float64(tot))
		log.Print("passed through %s / %s input bytes (%.2f%%)",
			u.FmtBytes(c.bytesSaved), u.FmtBytes(c.bytesProcessed),
			100.*float64(c.bytesSaved)/float64(c.bytesProcessed))
	}
	c.backend.LogStats()
}

func (c *compressed) Fsck() {
	c.backend.Fsck()
}

// Reusing gzip writers gives a huge benefit; an almost 40% reduction in
// overall runtime thanks to much less GC.
var writerPool = sync.Pool{
	New: func() interface{} {
		return gzip.NewWriter(os.Stderr)
	},
}

func (c *compressed) Write(data []byte) Hash {
	// Compress the input to a buffer.
	var compressed bytes.Buffer

	//w := gzip.NewWriter(&compressed)
	w := writerPool.Get().(*gzip.Writer)
	w.Reset(&compressed)
	defer writerPool.Put(w)

	_, err := w.Write(data)
	log.CheckError(err)
	err = w.Close()
	log.CheckError(err)

	c.bytesProcessed += int64(len(data))

	// Is the compressed buffer smaller than the input?
	var stored []byte
	if compressed.Len() < len(data) {
		// Yes; write out a 1 byte to indicate that the rest of the blob is
		// indeed compressed and then save the compressed bytes.
		stored = append([]byte{1}, compressed.Bytes()...)
		c.compressedBlobs++
	} else {
		// No; write a 0 to indicate that the data is uncompressed before
		// storing the original data.
		stored = append([]byte{0}, data...)
		c.uncompressedBlobs++
	}
	c.bytesSaved += int64(len(stored))
	return c.backend.Write(stored)
}

func (c *compressed) SyncWrites() {
	c.backend.SyncWrites()
}

func (c *compressed) HashExists(hash Hash) bool {
	return c.backend.HashExists(hash)
}

func (c *compressed) Hashes() map[Hash]struct{} {
	return c.backend.Hashes()
}

// Reusing readers gives a smaller benefit than writers, but still ~15%.
var readerPool = sync.Pool{
	New: func() interface{} {
		// "foo", gzip compressed, to give us a valid initial reader
		// without an error being issued.  (Its state will be reset
		// immediately after it's fetched from the pool.)
		foo := []byte{0x1f, 0x8b, 0x8, 0x0, 0x0, 0x9, 0x6e, 0x88, 0x0, 0xff}
		r, err := gzip.NewReader(bytes.NewReader(foo))
		log.CheckError(err)
		return r
	},
}

func (c *compressed) Read(hash Hash) (io.ReadCloser, error) {
	r, err := c.backend.Read(hash)
	if err != nil {
		return r, err
	}

	// Read the first byte to see if it's compressed or not.
	var b [1]byte
	n, err := r.Read(b[:])
	if err != nil {
		return nil, err
	}
	log.Check(n == 1)

	if b[0] == 1 {
		// Compressed: make a gzip reader.
		gzr := readerPool.Get().(*gzip.Reader)
		err = gzr.Reset(r)
		if err != nil {
			return nil, err
		}
		return &zipReaderAndCloser{gzr, r}, nil
	} else {
		// Otherwise just read the rest of the data normally.
		return r, nil
	}
}

type zipReaderAndCloser struct {
	gzr *gzip.Reader
	c   io.Closer
}

func (z *zipReaderAndCloser) Read(b []byte) (int, error) {
	return z.gzr.Read(b)
}

func (z *zipReaderAndCloser) Close() error {
	readerPool.Put(z.gzr)
	return z.c.Close()
}

func (c *compressed) WriteMetadata(name string, data []byte) {
	// Metadata contents aren't compressed, so just pass this along
	// directly.
	c.backend.WriteMetadata(name, data)
}

func (c *compressed) ReadMetadata(name string) []byte {
	return c.backend.ReadMetadata(name)
}

func (c *compressed) MetadataExists(name string) bool {
	return c.backend.MetadataExists(name)
}

func (c *compressed) ListMetadata() map[string]time.Time {
	return c.backend.ListMetadata()
}
<commit_msg>Naming consistency: blob vs chunk<commit_after>// storage/compressed.go
// Copyright(c) 2017 Matt Pharr
// BSD licensed; see LICENSE for details.

package storage

import (
	"bytes"
	"compress/gzip"
	u "github.com/mmp/bk/util"
	"io"
	"os"
	"sync"
	"time"
)

///////////////////////////////////////////////////////////////////////////
// compressed

// compressed implements the Backend interface. It applies gzip compression
// to the provided data before passing it along to another backend for
// storage.
type compressed struct {
	backend                              Backend
	bytesSaved, bytesProcessed           int64
	compressedChunks, uncompressedChunks int
}

// NewCompressed returns a new storage.Backend that applies gzip compression
// to the contents of chunks stored in the provided underlying backend.
// Note: the contents of metadata files are not compressed.
func NewCompressed(backend Backend) Backend {
	return &compressed{backend: backend}
}

func (c *compressed) String() string {
	return "gzip compressed " + c.backend.String()
}

func (c *compressed) LogStats() {
	tot := c.compressedChunks + c.uncompressedChunks
	if tot > 0 {
		log.Print("compressed %d / %d chunks (%2.f%%)",
			c.compressedChunks, tot, 100.*float64(c.compressedChunks)/float64(tot))
		log.Print("passed through %s / %s input bytes (%.2f%%)",
			u.FmtBytes(c.bytesSaved), u.FmtBytes(c.bytesProcessed),
			100.*float64(c.bytesSaved)/float64(c.bytesProcessed))
	}
	c.backend.LogStats()
}

func (c *compressed) Fsck() {
	c.backend.Fsck()
}

// Reusing gzip writers gives a huge benefit; an almost 40% reduction in
// overall runtime thanks to much less GC.
var writerPool = sync.Pool{
	New: func() interface{} {
		return gzip.NewWriter(os.Stderr)
	},
}

func (c *compressed) Write(chunk []byte) Hash {
	// Compress the input to a buffer.
	var compressed bytes.Buffer

	//w := gzip.NewWriter(&compressed)
	w := writerPool.Get().(*gzip.Writer)
	w.Reset(&compressed)
	defer writerPool.Put(w)

	_, err := w.Write(chunk)
	log.CheckError(err)
	err = w.Close()
	log.CheckError(err)

	c.bytesProcessed += int64(len(chunk))

	// Is the compressed buffer smaller than the input?
	var stored []byte
	if compressed.Len() < len(chunk) {
		// Yes; write out a 1 byte to indicate that the rest of the chunk
		// is indeed compressed and then save the compressed bytes.
		stored = append([]byte{1}, compressed.Bytes()...)
		c.compressedChunks++
	} else {
		// No; write a 0 to indicate that the data is uncompressed before
		// storing the original chunk.
		stored = append([]byte{0}, chunk...)
		c.uncompressedChunks++
	}
	c.bytesSaved += int64(len(stored))
	return c.backend.Write(stored)
}

func (c *compressed) SyncWrites() {
	c.backend.SyncWrites()
}

func (c *compressed) HashExists(hash Hash) bool {
	return c.backend.HashExists(hash)
}

func (c *compressed) Hashes() map[Hash]struct{} {
	return c.backend.Hashes()
}

// Reusing readers gives a smaller benefit than writers, but still ~15%.
var readerPool = sync.Pool{
	New: func() interface{} {
		// "foo", gzip compressed, to give us a valid initial reader
		// without an error being issued.  (Its state will be reset
		// immediately after it's fetched from the pool.)
		foo := []byte{0x1f, 0x8b, 0x8, 0x0, 0x0, 0x9, 0x6e, 0x88, 0x0, 0xff}
		r, err := gzip.NewReader(bytes.NewReader(foo))
		log.CheckError(err)
		return r
	},
}

func (c *compressed) Read(hash Hash) (io.ReadCloser, error) {
	r, err := c.backend.Read(hash)
	if err != nil {
		return r, err
	}

	// Read the first byte to see if it's compressed or not.
	var b [1]byte
	n, err := r.Read(b[:])
	if err != nil {
		return nil, err
	}
	log.Check(n == 1)

	if b[0] == 1 {
		// Compressed: make a gzip reader.
		gzr := readerPool.Get().(*gzip.Reader)
		err = gzr.Reset(r)
		if err != nil {
			return nil, err
		}
		return &zipReaderAndCloser{gzr, r}, nil
	} else {
		// Otherwise just read the rest of the data normally.
		return r, nil
	}
}

type zipReaderAndCloser struct {
	gzr *gzip.Reader
	c   io.Closer
}

func (z *zipReaderAndCloser) Read(b []byte) (int, error) {
	return z.gzr.Read(b)
}

func (z *zipReaderAndCloser) Close() error {
	readerPool.Put(z.gzr)
	return z.c.Close()
}

func (c *compressed) WriteMetadata(name string, data []byte) {
	// Metadata contents aren't compressed, so just pass this along
	// directly.
	c.backend.WriteMetadata(name, data)
}

func (c *compressed) ReadMetadata(name string) []byte {
	return c.backend.ReadMetadata(name)
}

func (c *compressed) MetadataExists(name string) bool {
	return c.backend.MetadataExists(name)
}

func (c *compressed) ListMetadata() map[string]time.Time {
	return c.backend.ListMetadata()
}
<|endoftext|>


<commit_before>package gs

import (
	"bytes"
	"context"
	"crypto/md5"
	"errors"
	"fmt"
	"hash/crc32"
	"io"
	"io/ioutil"
	"net/url"

	"cloud.google.com/go/storage"
	"github.com/viant/toolbox"
	tstorage "github.com/viant/toolbox/storage"
	"google.golang.org/api/option"
	"strings"
	"time"
)

type service struct {
	projectID string
	options   []option.ClientOption
}

func (s *service) NewClient() (*storage.Client, context.Context, error) {
	if s.projectID == "" {
		return nil, nil, fmt.Errorf("project ID was empty, consider setting GOOGLE_CLOUD_PROJECT")
	}
	ctx := context.Background()
	deadline, _ := ctx.Deadline()
	deadline.Add(time.Minute * 30)
	client, err := storage.NewClient(ctx, s.options...)
	if err != nil {
		err = fmt.Errorf("failed to create google storage client:%v", err)
	}
	return client, ctx, err
}

//List returns a list of object for supplied url
func (s *service) List(URL string) ([]tstorage.Object, error) {
	parsedUrl, err := url.Parse(URL)
	if err != nil {
		return nil, err
	}
	client, ctx, err := s.NewClient()
	if err != nil {
		return nil, err
	}
	defer client.Close()

	var query = &storage.Query{
		Delimiter: "/",
	}
	if len(parsedUrl.Path) > 0 {
		query.Prefix = parsedUrl.Path[1:]
	}

	responseIterator := client.Bucket(parsedUrl.Host).Objects(ctx, query)
	var result = make([]tstorage.Object, 0)
	for obj, err := responseIterator.Next(); err == nil; obj, err = responseIterator.Next() {
		objectURL := "gs://" + parsedUrl.Host + "/" + obj.Prefix + obj.Name
		var fileMode, _ = tstorage.NewFileMode("-rw-rw-rw-")
		if obj.Prefix != "" {
			fileMode, _ = tstorage.NewFileMode("drw-rw-rw-")
		}
		var _, name = toolbox.URLSplit(objectURL)
		var fileInfo = tstorage.NewFileInfo(name, obj.Size, fileMode, obj.Updated, fileMode.IsDir())
		var object = newStorageObject(objectURL, obj, fileInfo)
		result = append(result, object)
	}
	return result, err
}

func (s *service) Exists(URL string) (bool, error) {
	objects, err := s.List(URL)
	if err != nil {
		return false, err
	}
	return len(objects) > 0, nil
}

func (s *service) StorageObject(URL string) (tstorage.Object, error) {
	objects, err := s.List(URL)
	if err != nil {
		return nil, err
	}
	if len(objects) == 0 {
		return nil, fmt.Errorf("Not found %v", URL)
	}
	return objects[0], nil
}

//Download returns reader for downloaded storage object
func (s *service) Download(object tstorage.Object) (io.ReadCloser, error) {
	client, ctx, err := s.NewClient()
	if err != nil {
		return nil, err
	}
	defer client.Close()

	objectInfo := &storage.ObjectAttrs{}
	err = object.Unwrap(&objectInfo)
	if err != nil {
		return nil, err
	}
	return client.Bucket(objectInfo.Bucket).
		Object(objectInfo.Name).
		NewReader(ctx)
}

//Upload uploads provided reader content for supplied url.
func (s *service) Upload(URL string, reader io.Reader) error {
	parserURL, err := url.Parse(URL)
	if err != nil {
		return fmt.Errorf("failed to parse URL for uploading: %v, %v", URL, err)
	}
	client, ctx, err := s.NewClient()
	if err != nil {
		return err
	}
	defer client.Close()
	name := parserURL.Path
	if len(parserURL.Path) > 0 {
		name = parserURL.Path[1:]
	}

	err = s.uploadContent(ctx, client, parserURL, name, reader)
	if toolbox.IsNotFoundError(err) {
		err := client.Bucket(parserURL.Host).Create(ctx, s.projectID, &storage.BucketAttrs{})
		if err != nil {
			return fmt.Errorf("failed to create bucket: %v, %v", parserURL.Host, err)
		}
		//_, _ = client.Bucket(parserURL.Host).DefaultObjectACL().List(ctx)
		return s.uploadContent(ctx, client, parserURL, name, reader)
	}
	if err != nil {
		return fmt.Errorf("unable upload: %v", err)
	}
	return nil
}

func (s *service) uploadContent(ctx context.Context, client *storage.Client, parserURL *url.URL, name string, reader io.Reader) error {
	writer := client.Bucket(parserURL.Host).
		Object(name).
		NewWriter(ctx)

	expiry := parserURL.Query().Get("expiry")
	if expiry != "" {
		writer.Metadata = map[string]string{
			"Cache-Control": "private, max-age=" + expiry,
		}
	}

	var err error
	bufferReader, ok := reader.(*bytes.Buffer)
	if !ok {
		content, err := ioutil.ReadAll(reader)
		if err != nil {
			return fmt.Errorf("failed to read all during upload:%v", err)
		}
		bufferReader = bytes.NewBuffer(content)
	}

	if parserURL.Query().Get("disableMD5") == "" {
		hashReader := bytes.NewBuffer(bufferReader.Bytes())
		h := md5.New()
		_, _ = io.Copy(h, hashReader)
		writer.MD5 = h.Sum(nil)
		hashReader.Reset()
	}

	if parserURL.Query().Get("disableCRC32") == "" {
		crc32HashReader := bytes.NewBuffer(bufferReader.Bytes())
		crc32Hash := crc32.New(crc32.MakeTable(crc32.Castagnoli))
		_, _ = io.Copy(crc32Hash, crc32HashReader)
		writer.CRC32C = crc32Hash.Sum32()
		crc32HashReader.Reset()
	}

	if _, err = io.Copy(writer, bufferReader); err != nil {
		return fmt.Errorf("failed to copy to writer during upload:%v", err)
	}
	if err = writer.Close(); err != nil {
		return toolbox.ReclassifyNotFoundIfMatched(err, parserURL.String())
	}
	return nil
}

func (s *service) Register(schema string, service tstorage.Service) error {
	return errors.New("unsupported")
}

func (s *service) Close() error {
	return nil
}

func (s *service) listAll(URL string, result *[]tstorage.Object) error {
	if !strings.HasSuffix(URL, "/") {
		URL += "/"
	}
	objects, err := s.List(URL)
	if err != nil {
		return err
	}
	for _, object := range objects {
		if !object.IsFolder() {
			*result = append(*result, object)
			continue
		}
		if err = s.listAll(object.URL(), result); err != nil {
			return err
		}
	}
	return nil
}

//Delete removes passed in storage object
func (s *service) Delete(object tstorage.Object) error {
	client, ctx, err := s.NewClient()
	if err != nil {
		return err
	}
	defer client.Close()
	objectInfo := &storage.ObjectAttrs{}
	err = object.Unwrap(&objectInfo)
	if err != nil {
		return err
	}
	if object.IsFolder() {
		var objects = []tstorage.Object{}
		err := s.listAll(object.URL(), &objects)
		if err != nil {
			return err
		}
		for _, object := range objects {
			if err := s.Delete(object); err != nil {
				return err
			}
		}
		return nil
	}
	return client.Bucket(objectInfo.Bucket).
		Object(objectInfo.Name).Delete(ctx)
}

//DownloadWithURL downloads content for passed in object URL
func (s *service) DownloadWithURL(URL string) (io.ReadCloser, error) {
	object, err := s.StorageObject(URL)
	if err != nil {
		return nil, err
	}
	return s.Download(object)
}

//NewService create a new gc storage service
func NewService(projectId string, options ...option.ClientOption) *service {
	return &service{
		projectID: projectId,
		options:   options,
	}
}
<commit_msg>added disableChecksum flag<commit_after>package gs

import (
	"bytes"
	"context"
	"crypto/md5"
	"errors"
	"fmt"
	"hash/crc32"
	"io"
	"io/ioutil"
	"net/url"

	"cloud.google.com/go/storage"
	"github.com/viant/toolbox"
	tstorage "github.com/viant/toolbox/storage"
	"google.golang.org/api/option"
	"strings"
	"time"
)

type service struct {
	projectID string
	options   []option.ClientOption
}

func (s *service) NewClient() (*storage.Client, context.Context, error) {
	if s.projectID == "" {
		return nil, nil, fmt.Errorf("project ID was empty, consider setting GOOGLE_CLOUD_PROJECT")
	}
	ctx := context.Background()
	deadline, _ := ctx.Deadline()
	deadline.Add(time.Minute * 30)
	client, err := storage.NewClient(ctx, s.options...)
	if err != nil {
		err = fmt.Errorf("failed to create google storage client:%v", err)
	}
	return client, ctx, err
}

//List returns a list of object for supplied url
func (s *service) List(URL string) ([]tstorage.Object, error) {
	parsedUrl, err := url.Parse(URL)
	if err != nil {
		return nil, err
	}
	client, ctx, err := s.NewClient()
	if err != nil {
		return nil, err
	}
	defer client.Close()

	var query = &storage.Query{
		Delimiter: "/",
	}
	if len(parsedUrl.Path) > 0 {
		query.Prefix = parsedUrl.Path[1:]
	}

	responseIterator := client.Bucket(parsedUrl.Host).Objects(ctx, query)
	var result = make([]tstorage.Object, 0)
	for obj, err := responseIterator.Next(); err == nil; obj, err = responseIterator.Next() {
		objectURL := "gs://" + parsedUrl.Host + "/" + obj.Prefix + obj.Name
		var fileMode, _ = tstorage.NewFileMode("-rw-rw-rw-")
		if obj.Prefix != "" {
			fileMode, _ = tstorage.NewFileMode("drw-rw-rw-")
		}
		var _, name = toolbox.URLSplit(objectURL)
		var fileInfo = tstorage.NewFileInfo(name, obj.Size, fileMode, obj.Updated, fileMode.IsDir())
		var object = newStorageObject(objectURL, obj, fileInfo)
		result = append(result, object)
	}
	return result, err
}

func (s *service) Exists(URL string) (bool, error) {
	objects, err := s.List(URL)
	if err != nil {
		return false, err
	}
	return len(objects) > 0, nil
}

func (s *service) StorageObject(URL string) (tstorage.Object, error) {
	objects, err := s.List(URL)
	if err != nil {
		return nil, err
	}
	if len(objects) == 0 {
		return nil, fmt.Errorf("Not found %v", URL)
	}
	return objects[0], nil
}

//Download returns reader for downloaded storage object
func (s *service) Download(object tstorage.Object) (io.ReadCloser, error) {
	client, ctx, err := s.NewClient()
	if err != nil {
		return nil, err
	}
	defer client.Close()

	objectInfo := &storage.ObjectAttrs{}
	err = object.Unwrap(&objectInfo)
	if err != nil {
		return nil, err
	}
	return client.Bucket(objectInfo.Bucket).
		Object(objectInfo.Name).
		NewReader(ctx)
}

//Upload uploads provided reader content for supplied url.
func (s *service) Upload(URL string, reader io.Reader) error {
	parserURL, err := url.Parse(URL)
	if err != nil {
		return fmt.Errorf("failed to parse URL for uploading: %v, %v", URL, err)
	}
	client, ctx, err := s.NewClient()
	if err != nil {
		return err
	}
	defer client.Close()
	name := parserURL.Path
	if len(parserURL.Path) > 0 {
		name = parserURL.Path[1:]
	}

	err = s.uploadContent(ctx, client, parserURL, name, reader)
	if toolbox.IsNotFoundError(err) {
		err := client.Bucket(parserURL.Host).Create(ctx, s.projectID, &storage.BucketAttrs{})
		if err != nil {
			return fmt.Errorf("failed to create bucket: %v, %v", parserURL.Host, err)
		}
		//_, _ = client.Bucket(parserURL.Host).DefaultObjectACL().List(ctx)
		return s.uploadContent(ctx, client, parserURL, name, reader)
	}
	if err != nil {
		return fmt.Errorf("unable upload: %v", err)
	}
	return nil
}

func (s *service) uploadContent(ctx context.Context, client *storage.Client, parserURL *url.URL, name string, reader io.Reader) error {
	writer := client.Bucket(parserURL.Host).
		Object(name).
		NewWriter(ctx)
	expiry := parserURL.Query().Get("expiry")
	if expiry != "" {
		writer.Metadata = map[string]string{
			"Cache-Control": "private, max-age=" + expiry,
		}
	}
	reader, err:= updateUploadChecksum(parserURL, reader, writer)
	if _, err = io.Copy(writer, reader); err != nil {
		return fmt.Errorf("failed to copy to writer during upload:%v", err)
	}
	if err = writer.Close(); err != nil {
		return toolbox.ReclassifyNotFoundIfMatched(err, parserURL.String())
	}
	return nil
}

func updateUploadChecksum(parserURL *url.URL, reader io.Reader, writer *storage.Writer) (io.Reader, error) {
	checksumDisabled := parserURL.Query().Get("disableChecksum") != ""
	updateMD5 := parserURL.Query().Get("disableMD5") == ""
	updateCRC := parserURL.Query().Get("disableCRC32") == ""
	if !(updateCRC || updateMD5) || checksumDisabled {
		return reader, nil
	}

	var err error
	bufferReader, ok := reader.(*bytes.Buffer)
	if !ok {
		content, err := ioutil.ReadAll(reader)
		if err != nil {
			return nil, fmt.Errorf("failed to read all during upload:%v", err)
		}
		bufferReader = bytes.NewBuffer(content)
	}
	if parserURL.Query().Get("disableMD5") == "" {
		hashReader := bytes.NewBuffer(bufferReader.Bytes())
		h := md5.New()
		_, _ = io.Copy(h, hashReader)
		writer.MD5 = h.Sum(nil)
		hashReader.Reset()
	} else if parserURL.Query().Get("disableCRC32") == "" {
		crc32HashReader := bytes.NewBuffer(bufferReader.Bytes())
		crc32Hash := crc32.New(crc32.MakeTable(crc32.Castagnoli))
		_, _ = io.Copy(crc32Hash, crc32HashReader)
		writer.CRC32C = crc32Hash.Sum32()
		crc32HashReader.Reset()
	}
	return bufferReader, err
}

func (s *service) Register(schema string, service tstorage.Service) error {
	return errors.New("unsupported")
}

func (s *service) Close() error {
	return nil
}

func (s *service) listAll(URL string, result *[]tstorage.Object) error {
	if !strings.HasSuffix(URL, "/") {
		URL += "/"
	}
	objects, err := s.List(URL)
	if err != nil {
		return err
	}
	for _, object := range objects {
		if !object.IsFolder() {
			*result = append(*result, object)
			continue
		}
		if err = s.listAll(object.URL(), result); err != nil {
			return err
		}
	}
	return nil
}

//Delete removes passed in storage object
func (s *service) Delete(object tstorage.Object) error {
	client, ctx, err := s.NewClient()
	if err != nil {
		return err
	}
	defer client.Close()
	objectInfo := &storage.ObjectAttrs{}
	err = object.Unwrap(&objectInfo)
	if err != nil {
		return err
	}
	if object.IsFolder() {
		var objects = []tstorage.Object{}
		err := s.listAll(object.URL(), &objects)
		if err != nil {
			return err
		}
		for _, object := range objects {
			if err := s.Delete(object); err != nil {
				return err
			}
		}
		return nil
	}
	return client.Bucket(objectInfo.Bucket).
		Object(objectInfo.Name).Delete(ctx)
}

//DownloadWithURL downloads content for passed in object URL
func (s *service) DownloadWithURL(URL string) (io.ReadCloser, error) {
	object, err := s.StorageObject(URL)
	if err != nil {
		return nil, err
	}
	return s.Download(object)
}

//NewService create a new gc storage service
func NewService(projectId string, options ...option.ClientOption) *service {
	return &service{
		projectID: projectId,
		options:   options,
	}
}
<|endoftext|>


<commit_before>package storage

import (
	"bytes"
	"database/sql"
	"fmt"
	"html/template"
	"time"

	"github.com/VagabondDataNinjas/gizlinebot/domain"
	"github.com/go-sql-driver/mysql"

	_ "github.com/go-sql-driver/mysql"
)

type Sql struct {
	Db *sql.DB
}

func NewSql(conDsn string) (s *Sql, err error) {
	db, err := sql.Open("mysql", conDsn)
	if err != nil {
		return s, err
	}

	return &Sql{
		Db: db,
	}, nil
}

func (s *Sql) Close() error {
	return s.Db.Close()
}

func (s *Sql) AddRawLineEvent(eventType, rawevent string) error {
	stmt, err := s.Db.Prepare("INSERT INTO linebot_raw_events(eventtype, rawevent, timestamp) VALUES(?, ?, ?)")
	if err != nil {
		return err
	}
	defer stmt.Close()
	_, err = stmt.Exec(eventType, rawevent, int32(time.Now().UTC().Unix()))
	if err != nil {
		return err
	}

	return nil
}

// AddUserProfile adds a user profile
// if the user already exists in the table this method does nothing
func (s *Sql) AddUserProfile(userID, displayName string) error {
	stmt, err := s.Db.Prepare("INSERT INTO user_profiles(userId, displayName, timestamp) VALUES(?, ?, ?)")
	if err != nil {
		return err
	}
	defer stmt.Close()
	_, err = stmt.Exec(userID, displayName, int32(time.Now().UTC().Unix()))

	if err != nil {
		if mysqlErr := err.(*mysql.MySQLError); mysqlErr.Number == 1062 {
			// ignore duplicate entry errors for profiles
			return nil
		}
		return err
	}

	return nil
}

func (s *Sql) MarkProfileBotSurveyInited(userId string) error {
	stmt, err := s.Db.Prepare("UPDATE user_profiles SET bot_survey_inited = 1 WHERE userId = ?")
	if err != nil {
		return err
	}
	defer stmt.Close()
	_, err = stmt.Exec(userId)

	if err != nil {
		return err
	}

	return nil
}

func (s *Sql) GetUsersWithoutAnswers(delaySecs int64) (userIds []string, err error) {
	var (
		userId string
	)

	tsCompare := time.Now().UTC().Unix() - delaySecs
	rows, err := s.Db.Query(`SELECT p.userId FROM user_profiles p
		LEFT JOIN answers a ON a.userId = p.userId
		WHERE a.userId IS NULL AND p.bot_survey_inited = 0 AND p.timestamp < ?`, tsCompare)
	if err != nil {
		return userIds, err
	}
	defer rows.Close()

	userIds = make([]string, 0)
	for rows.Next() {
		err := rows.Scan(&userId)
		if err != nil {
			return userIds, err
		}
		userIds = append(userIds, userId)
	}

	err = rows.Err()
	if err != nil {
		return userIds, err
	}

	return userIds, nil
}

func (s *Sql) GetUserProfile(userId string) (profile domain.UserProfile, err error) {
	var (
		displayName string
		timestamp   int
	)
	err = s.Db.QueryRow(`SELECT displayName, timestamp
		FROM user_profiles where userId = ?`, userId).Scan(&displayName, &timestamp)
	if err != nil {
		if err == sql.ErrNoRows {
			return profile, nil
		}
		return profile, err
	}

	return domain.UserProfile{
		UserId:      userId,
		DisplayName: displayName,
		Timestamp:   timestamp,
	}, nil
}

func (s *Sql) UserHasAnswers(userId string) (bool, error) {
	var hasAnswers int
	err := s.Db.QueryRow(`SELECT count(id) FROM answers
		WHERE userId = ?`, userId).Scan(&hasAnswers)
	if err != nil {
		return false, err
	}

	if hasAnswers > 0 {
		return true, nil
	}
	return false, nil
}

func (s *Sql) UserGetLastAnswer(uid string) (domain.Answer, error) {
	var id uint
	var userId string
	var questionId string
	var answer string
	var timestamp int64
	err := s.Db.QueryRow(`SELECT id, userId, questionId, answer, timestamp FROM answers
		WHERE userId = ? AND answer != ""
		ORDER BY timestamp DESC
		LIMIT 0,1
		`, uid).Scan(&id, &userId, &questionId, &answer, &timestamp)
	if err != nil {
		var emptyAnswer domain.Answer
		return emptyAnswer, err
	}

	return domain.Answer{
		Id:         id,
		UserId:     userId,
		QuestionId: questionId,
		Answer:     answer,
		Timestamp:  time.Unix(timestamp, 0),
	}, nil
}

func (s *Sql) GetQuestions() (qs *domain.Questions, err error) {
	var (
		id           string
		questionText string
		weight       int
		channel      string
	)
	rows, err := s.Db.Query(`SELECT id, question, weight, channel FROM questions ORDER BY weight ASC`)
	if err != nil {
		return qs, err
	}
	defer rows.Close()

	qs = domain.NewQuestions()
	for rows.Next() {
		err := rows.Scan(&id, &questionText, &weight, &channel)
		if err != nil {
			return qs, err
		}
		err = qs.Add(id, questionText, weight, channel)
		if err != nil {
			return qs, err
		}
	}

	err = rows.Err()
	if err != nil {
		return qs, err
	}

	return qs, nil
}

type WelcomeMsgTplVars struct {
	UserId   string
	Hostname string
}

func (s *Sql) GetWelcomeMsgs(tplVars *WelcomeMsgTplVars) (msgs []string, err error) {
	var (
		msgRaw string
	)
	rows, err := s.Db.Query(`SELECT msg FROM welcome_msgs WHERE channel IN ("line", "both") ORDER BY weight ASC`)
	if err != nil {
		return msgs, err
	}
	defer rows.Close()

	msgs = make([]string, 0)
	for rows.Next() {
		err := rows.Scan(&msgRaw)
		if err != nil {
			return msgs, err
		}
		msg, err := s.applyWelcomeTpl(msgRaw, tplVars)
		if err != nil {
			return msgs, err
		}
		msgs = append(msgs, msg)
	}

	err = rows.Err()
	if err != nil {
		return msgs, err
	}

	return msgs, nil
}

type UserAnswerData struct {
	// @TODO embed domain.Answer
	// domain.Answer
	Id         uint
	UserId     string
	QuestionId string
	Answer     string
	Channel    string
	Timestamp  int
}

type UserGpsAnswerData struct {
	Id        uint
	UserId    string
	Address   string
	Lat       float64
	Lon       float64
	Timestamp int
	Channel   string
}

func (s *Sql) GetGpsAnswerData() (answerGpsData []UserGpsAnswerData, err error) {
	rows, err := s.Db.Query(`SELECT p.id, p.userId, IFNULL(a.address, ""), IFNULL(a.lat, "") AS lat, IFNULL(a.lon, "") AS lon, IFNULL(a.channel, "") AS channel, IFNULL(a.timestamp, 0) AS timestamp FROM user_profiles p
		LEFT JOIN answers_gps a ON a.userId = p.userId
		ORDER BY a.timestamp ASC
		`)
	if err != nil {
		return answerGpsData, err
	}
	defer rows.Close()

	answerGpsData = make([]UserGpsAnswerData, 0)
	for rows.Next() {
		a := UserGpsAnswerData{}
		err := rows.Scan(&a.Id, &a.UserId, &a.Address, &a.Lat, &a.Lon, &a.Channel, &a.Timestamp)
		if err != nil {
			return answerGpsData, err
		}
		answerGpsData = append(answerGpsData, a)
	}

	err = rows.Err()
	if err != nil {
		return answerGpsData, err
	}

	return answerGpsData, nil

}
func (s *Sql) GetUserAnswerData() (answerData []UserAnswerData, err error) {
	var (
		userId     string
		questionId string
		answer     string
		channel    string
		answerTime int
	)
	rows, err := s.Db.Query(`SELECT p.userId, a.questionId, a.answer, a.channel, a.timestamp as answerTime FROM user_profiles p
		LEFT JOIN answers a ON a.userId = p.userId
		ORDER BY a.timestamp ASC
		`)
	if err != nil {
		return answerData, err
	}
	defer rows.Close()

	answerData = make([]UserAnswerData, 0)
	for rows.Next() {
		err := rows.Scan(&userId, &questionId, &answer, &channel, &answerTime)
		if err != nil {
			return answerData, err
		}
		answerData = append(answerData, UserAnswerData{
			UserId:     userId,
			QuestionId: questionId,
			Answer:     answer,
			Channel:    channel,
			Timestamp:  answerTime,
		})
	}

	err = rows.Err()
	if err != nil {
		return answerData, err
	}

	return answerData, nil
}

func (s *Sql) applyWelcomeTpl(msg string, tplVars *WelcomeMsgTplVars) (string, error) {
	tmpl, err := template.New("welcomeMsg").Parse(msg)
	if err != nil {
		return "", err
	}
	buf := new(bytes.Buffer)
	err = tmpl.Execute(buf, tplVars)
	if err != nil {
		return "", err
	}
	return buf.String(), nil
}

func (s *Sql) UserAddAnswer(answer domain.Answer) error {
	stmt, err := s.Db.Prepare("INSERT INTO answers(userId, questionId, answer, channel, timestamp) VALUES(?, ?, ?, ?, ?)")
	if err != nil {
		return err
	}
	defer stmt.Close()
	_, err = stmt.Exec(answer.UserId, answer.QuestionId, answer.Answer, answer.Channel, int32(time.Now().UTC().Unix()))

	if err != nil {
		return err
	}

	return nil
}

func (s *Sql) WipeUser(userId string) error {
	for _, table := range []string{"user_profiles", "answers", "answers_gps"} {
		err := s.deleteFromTableUserId(table, userId)
		if err != nil {
			return err
		}
	}

	return nil
}

func (s *Sql) deleteFromTableUserId(table string, userId string) error {
	// @TODO find out how to use dynamic table name in prepared query
	q := fmt.Sprintf("DELETE FROM %s WHERE userId = ?", table)
	stmt, err := s.Db.Prepare(q)
	if err != nil {
		return err
	}
	defer stmt.Close()
	_, err = stmt.Exec(userId)
	if err != nil {
		return err
	}

	return nil
}

func (s *Sql) UserAddGpsAnswer(answer domain.AnswerGps) error {
	stmt, err := s.Db.Prepare("INSERT INTO answers_gps(userId, lat, lon, address, channel, timestamp) VALUES(?, ?, ?, ?, ?, ?)")
	if err != nil {
		return err
	}
	defer stmt.Close()
	_, err = stmt.Exec(answer.UserId, answer.Lat, answer.Lon, answer.Address, answer.Channel, int32(time.Now().UTC().Unix()))

	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Fixed null values on both userdata and usergpsdata export<commit_after>package storage

import (
	"bytes"
	"database/sql"
	"fmt"
	"html/template"
	"time"

	"github.com/VagabondDataNinjas/gizlinebot/domain"
	"github.com/go-sql-driver/mysql"

	_ "github.com/go-sql-driver/mysql"
)

type Sql struct {
	Db *sql.DB
}

func NewSql(conDsn string) (s *Sql, err error) {
	db, err := sql.Open("mysql", conDsn)
	if err != nil {
		return s, err
	}

	return &Sql{
		Db: db,
	}, nil
}

func (s *Sql) Close() error {
	return s.Db.Close()
}

func (s *Sql) AddRawLineEvent(eventType, rawevent string) error {
	stmt, err := s.Db.Prepare("INSERT INTO linebot_raw_events(eventtype, rawevent, timestamp) VALUES(?, ?, ?)")
	if err != nil {
		return err
	}
	defer stmt.Close()
	_, err = stmt.Exec(eventType, rawevent, int32(time.Now().UTC().Unix()))
	if err != nil {
		return err
	}

	return nil
}

// AddUserProfile adds a user profile
// if the user already exists in the table this method does nothing
func (s *Sql) AddUserProfile(userID, displayName string) error {
	stmt, err := s.Db.Prepare("INSERT INTO user_profiles(userId, displayName, timestamp) VALUES(?, ?, ?)")
	if err != nil {
		return err
	}
	defer stmt.Close()
	_, err = stmt.Exec(userID, displayName, int32(time.Now().UTC().Unix()))

	if err != nil {
		if mysqlErr := err.(*mysql.MySQLError); mysqlErr.Number == 1062 {
			// ignore duplicate entry errors for profiles
			return nil
		}
		return err
	}

	return nil
}

func (s *Sql) MarkProfileBotSurveyInited(userId string) error {
	stmt, err := s.Db.Prepare("UPDATE user_profiles SET bot_survey_inited = 1 WHERE userId = ?")
	if err != nil {
		return err
	}
	defer stmt.Close()
	_, err = stmt.Exec(userId)

	if err != nil {
		return err
	}

	return nil
}

func (s *Sql) GetUsersWithoutAnswers(delaySecs int64) (userIds []string, err error) {
	var (
		userId string
	)

	tsCompare := time.Now().UTC().Unix() - delaySecs
	rows, err := s.Db.Query(`SELECT p.userId FROM user_profiles p
		LEFT JOIN answers a ON a.userId = p.userId
		WHERE a.userId IS NULL AND p.bot_survey_inited = 0 AND p.timestamp < ?`, tsCompare)
	if err != nil {
		return userIds, err
	}
	defer rows.Close()

	userIds = make([]string, 0)
	for rows.Next() {
		err := rows.Scan(&userId)
		if err != nil {
			return userIds, err
		}
		userIds = append(userIds, userId)
	}

	err = rows.Err()
	if err != nil {
		return userIds, err
	}

	return userIds, nil
}

func (s *Sql) GetUserProfile(userId string) (profile domain.UserProfile, err error) {
	var (
		displayName string
		timestamp   int
	)
	err = s.Db.QueryRow(`SELECT displayName, timestamp
		FROM user_profiles where userId = ?`, userId).Scan(&displayName, &timestamp)
	if err != nil {
		if err == sql.ErrNoRows {
			return profile, nil
		}
		return profile, err
	}

	return domain.UserProfile{
		UserId:      userId,
		DisplayName: displayName,
		Timestamp:   timestamp,
	}, nil
}

func (s *Sql) UserHasAnswers(userId string) (bool, error) {
	var hasAnswers int
	err := s.Db.QueryRow(`SELECT count(id) FROM answers
		WHERE userId = ?`, userId).Scan(&hasAnswers)
	if err != nil {
		return false, err
	}

	if hasAnswers > 0 {
		return true, nil
	}
	return false, nil
}

func (s *Sql) UserGetLastAnswer(uid string) (domain.Answer, error) {
	var id uint
	var userId string
	var questionId string
	var answer string
	var timestamp int64
	err := s.Db.QueryRow(`SELECT id, userId, questionId, answer, timestamp FROM answers
		WHERE userId = ? AND answer != ""
		ORDER BY timestamp DESC
		LIMIT 0,1
		`, uid).Scan(&id, &userId, &questionId, &answer, &timestamp)
	if err != nil {
		var emptyAnswer domain.Answer
		return emptyAnswer, err
	}

	return domain.Answer{
		Id:         id,
		UserId:     userId,
		QuestionId: questionId,
		Answer:     answer,
		Timestamp:  time.Unix(timestamp, 0),
	}, nil
}

func (s *Sql) GetQuestions() (qs *domain.Questions, err error) {
	var (
		id           string
		questionText string
		weight       int
		channel      string
	)
	rows, err := s.Db.Query(`SELECT id, question, weight, channel FROM questions ORDER BY weight ASC`)
	if err != nil {
		return qs, err
	}
	defer rows.Close()

	qs = domain.NewQuestions()
	for rows.Next() {
		err := rows.Scan(&id, &questionText, &weight, &channel)
		if err != nil {
			return qs, err
		}
		err = qs.Add(id, questionText, weight, channel)
		if err != nil {
			return qs, err
		}
	}

	err = rows.Err()
	if err != nil {
		return qs, err
	}

	return qs, nil
}

type WelcomeMsgTplVars struct {
	UserId   string
	Hostname string
}

func (s *Sql) GetWelcomeMsgs(tplVars *WelcomeMsgTplVars) (msgs []string, err error) {
	var (
		msgRaw string
	)
	rows, err := s.Db.Query(`SELECT msg FROM welcome_msgs WHERE channel IN ("line", "both") ORDER BY weight ASC`)
	if err != nil {
		return msgs, err
	}
	defer rows.Close()

	msgs = make([]string, 0)
	for rows.Next() {
		err := rows.Scan(&msgRaw)
		if err != nil {
			return msgs, err
		}
		msg, err := s.applyWelcomeTpl(msgRaw, tplVars)
		if err != nil {
			return msgs, err
		}
		msgs = append(msgs, msg)
	}

	err = rows.Err()
	if err != nil {
		return msgs, err
	}

	return msgs, nil
}

type UserAnswerData struct {
	// @TODO embed domain.Answer
	// domain.Answer
	Id         uint
	UserId     string
	QuestionId string
	Answer     string
	Channel    string
	Timestamp  int
}

type UserGpsAnswerData struct {
	Id        uint
	UserId    string
	Address   string
	Lat       float64
	Lon       float64
	Timestamp int
	Channel   string
}

func (s *Sql) GetGpsAnswerData() (answerGpsData []UserGpsAnswerData, err error) {
	rows, err := s.Db.Query(`SELECT p.id, p.userId, IFNULL(a.address, ""), IFNULL(a.lat, 0.0), IFNULL(a.lon, 0.0), IFNULL(a.channel, ""), IFNULL(a.timestamp, 0) FROM user_profiles p
		LEFT JOIN answers_gps a ON a.userId = p.userId
		ORDER BY a.timestamp ASC
		`)
	if err != nil {
		return answerGpsData, err
	}
	defer rows.Close()

	answerGpsData = make([]UserGpsAnswerData, 0)
	for rows.Next() {
		a := UserGpsAnswerData{}
		err := rows.Scan(&a.Id, &a.UserId, &a.Address, &a.Lat, &a.Lon, &a.Channel, &a.Timestamp)
		if err != nil {
			return answerGpsData, err
		}
		answerGpsData = append(answerGpsData, a)
	}

	err = rows.Err()
	if err != nil {
		return answerGpsData, err
	}

	return answerGpsData, nil

}
func (s *Sql) GetUserAnswerData() (answerData []UserAnswerData, err error) {
	var (
		userId     string
		questionId string
		answer     string
		channel    string
		answerTime int
	)
	rows, err := s.Db.Query(`SELECT p.userId, IFNULL(a.questionId, ""), IFNULL(a.answer, ""), IFNULL(a.channel, ""), IFNULL(a.timestamp, 0) as answerTime FROM user_profiles p
		LEFT JOIN answers a ON a.userId = p.userId
		ORDER BY a.timestamp ASC
		`)
	if err != nil {
		return answerData, err
	}
	defer rows.Close()

	answerData = make([]UserAnswerData, 0)
	for rows.Next() {
		err := rows.Scan(&userId, &questionId, &answer, &channel, &answerTime)
		if err != nil {
			return answerData, err
		}
		answerData = append(answerData, UserAnswerData{
			UserId:     userId,
			QuestionId: questionId,
			Answer:     answer,
			Channel:    channel,
			Timestamp:  answerTime,
		})
	}

	err = rows.Err()
	if err != nil {
		return answerData, err
	}

	return answerData, nil
}

func (s *Sql) applyWelcomeTpl(msg string, tplVars *WelcomeMsgTplVars) (string, error) {
	tmpl, err := template.New("welcomeMsg").Parse(msg)
	if err != nil {
		return "", err
	}
	buf := new(bytes.Buffer)
	err = tmpl.Execute(buf, tplVars)
	if err != nil {
		return "", err
	}
	return buf.String(), nil
}

func (s *Sql) UserAddAnswer(answer domain.Answer) error {
	stmt, err := s.Db.Prepare("INSERT INTO answers(userId, questionId, answer, channel, timestamp) VALUES(?, ?, ?, ?, ?)")
	if err != nil {
		return err
	}
	defer stmt.Close()
	_, err = stmt.Exec(answer.UserId, answer.QuestionId, answer.Answer, answer.Channel, int32(time.Now().UTC().Unix()))

	if err != nil {
		return err
	}

	return nil
}

func (s *Sql) WipeUser(userId string) error {
	for _, table := range []string{"user_profiles", "answers", "answers_gps"} {
		err := s.deleteFromTableUserId(table, userId)
		if err != nil {
			return err
		}
	}

	return nil
}

func (s *Sql) deleteFromTableUserId(table string, userId string) error {
	// @TODO find out how to use dynamic table name in prepared query
	q := fmt.Sprintf("DELETE FROM %s WHERE userId = ?", table)
	stmt, err := s.Db.Prepare(q)
	if err != nil {
		return err
	}
	defer stmt.Close()
	_, err = stmt.Exec(userId)
	if err != nil {
		return err
	}

	return nil
}

func (s *Sql) UserAddGpsAnswer(answer domain.AnswerGps) error {
	stmt, err := s.Db.Prepare("INSERT INTO answers_gps(userId, lat, lon, address, channel, timestamp) VALUES(?, ?, ?, ?, ?, ?)")
	if err != nil {
		return err
	}
	defer stmt.Close()
	_, err = stmt.Exec(answer.UserId, answer.Lat, answer.Lon, answer.Address, answer.Channel, int32(time.Now().UTC().Unix()))

	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015, David Howden
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package cmdflag unifies the configuration of stores using command line flags across
// several tools.
package cmdflag

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"strings"

	"github.com/mitchellh/goamz/aws"

	"github.com/dhowden/tchaik/store"
	"github.com/dhowden/tchaik/store/cafs"
)

var localStore, remoteStore string
var mediaFileSystemCache, artworkFileSystemCache string
var trimPathPrefix, addPathPrefix string

func init() {
	flag.StringVar(&localStore, "local-store", "/", "local media store, full local path /path/to/root")
	flag.StringVar(&remoteStore, "remote-store", "", "remote media store, tchstore server address <hostname>:<port>, or s3://<bucket>/path/to/root for S3")

	flag.StringVar(&artworkFileSystemCache, "artwork-cache", "", "path to local artwork cache (content addressable)")
	flag.StringVar(&mediaFileSystemCache, "media-cache", "", "path to local media cache")

	flag.StringVar(&trimPathPrefix, "trim-path-prefix", "", "remove prefix from every path")
	flag.StringVar(&addPathPrefix, "add-path-prefix", "", "add prefix to every path")
}

type stores struct {
	media, artwork http.FileSystem
}

func buildRemoteStore(s *stores) (err error) {
	if remoteStore == "" {
		return nil
	}
	var c store.Client
	if strings.HasPrefix(remoteStore, "s3://") {
		path := strings.TrimPrefix(remoteStore, "s3://")
		bucketPathSplit := strings.Split(path, "/")

		if len(bucketPathSplit) == 0 {
			return fmt.Errorf("invalid S3 path: %#v\n", remoteStore)
		}
		bucket := bucketPathSplit[0]
		var auth aws.Auth
		auth, err = aws.GetAuth("", "") // Extract credentials from the current instance.
		if err != nil {
			return fmt.Errorf("error getting AWS credentials: %v", err)
		}
		c = store.NewS3Client(bucket, auth, aws.APSoutheast2)
	} else {
		c = store.NewClient(remoteStore, "")
		s.artwork = store.NewFileSystem(store.NewClient(remoteStore, "artwork"))
	}
	s.media = store.NewRemoteChunkedFileSystem(c, 32*1024)
	return nil
}

func buildLocalStore(s *stores) {
	if localStore != "" {
		fs := http.Dir(localStore)
		if s.media != nil {
			s.media = store.MultiFileSystem(fs, s.media)
		} else {
			s.media = fs
		}

		if s.artwork != nil {
			s.artwork = store.MultiFileSystem(store.ArtworkFileSystem(fs), s.artwork)
		} else {
			s.artwork = store.ArtworkFileSystem(fs)
		}
	}
}

func buildMediaCache(s *stores) {
	if mediaFileSystemCache != "" {
		var errCh <-chan error
		localCache := store.Dir(mediaFileSystemCache)
		s.media, errCh = store.NewCachedFileSystem(
			s.media,
			localCache,
		)
		go func() {
			for err := range errCh {
				// TODO: pull this out!
				log.Printf("mediaFileSystem cache: %v", err)
			}
		}()
	}
}

func buildArtworkCache(s *stores) error {
	if artworkFileSystemCache != "" {
		cfs, err := cafs.New(store.Dir(artworkFileSystemCache))
		if err != nil {
			return fmt.Errorf("error creating artwork cafs: %v", err)
		}

		var errCh <-chan error
		s.artwork, errCh = store.NewCachedFileSystem(
			s.artwork,
			cfs,
		)
		go func() {
			for err := range errCh {
				// TODO: pull this out!
				log.Printf("artwork cache: %v", err)
			}
		}()
	}
	return nil
}

// Stores returns a media and artwork filesystem as defined by the command line flags.
func Stores() (media, artwork http.FileSystem, err error) {
	s := &stores{}
	err = buildRemoteStore(s)
	if err != nil {
		return nil, nil, err
	}

	buildLocalStore(s)
	buildMediaCache(s)

	err = buildArtworkCache(s)
	if err != nil {
		return nil, nil, err
	}

	if trimPathPrefix != "" || addPathPrefix != "" {
		s.media = store.NewPathRewrite(s.media, trimPathPrefix, addPathPrefix)
		s.artwork = store.NewPathRewrite(s.artwork, trimPathPrefix, addPathPrefix)
	}
	return s.media, s.artwork, nil
}
<commit_msg>Tidy up buildLocalStore<commit_after>// Copyright 2015, David Howden
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package cmdflag unifies the configuration of stores using command line flags across
// several tools.
package cmdflag

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"strings"

	"github.com/mitchellh/goamz/aws"

	"github.com/dhowden/tchaik/store"
	"github.com/dhowden/tchaik/store/cafs"
)

var localStore, remoteStore string
var mediaFileSystemCache, artworkFileSystemCache string
var trimPathPrefix, addPathPrefix string

func init() {
	flag.StringVar(&localStore, "local-store", "/", "local media store, full local path /path/to/root")
	flag.StringVar(&remoteStore, "remote-store", "", "remote media store, tchstore server address <hostname>:<port>, or s3://<bucket>/path/to/root for S3")

	flag.StringVar(&artworkFileSystemCache, "artwork-cache", "", "path to local artwork cache (content addressable)")
	flag.StringVar(&mediaFileSystemCache, "media-cache", "", "path to local media cache")

	flag.StringVar(&trimPathPrefix, "trim-path-prefix", "", "remove prefix from every path")
	flag.StringVar(&addPathPrefix, "add-path-prefix", "", "add prefix to every path")
}

type stores struct {
	media, artwork http.FileSystem
}

func buildRemoteStore(s *stores) (err error) {
	if remoteStore == "" {
		return nil
	}
	var c store.Client
	if strings.HasPrefix(remoteStore, "s3://") {
		path := strings.TrimPrefix(remoteStore, "s3://")
		bucketPathSplit := strings.Split(path, "/")

		if len(bucketPathSplit) == 0 {
			return fmt.Errorf("invalid S3 path: %#v\n", remoteStore)
		}
		bucket := bucketPathSplit[0]
		var auth aws.Auth
		auth, err = aws.GetAuth("", "") // Extract credentials from the current instance.
		if err != nil {
			return fmt.Errorf("error getting AWS credentials: %v", err)
		}
		c = store.NewS3Client(bucket, auth, aws.APSoutheast2)
	} else {
		c = store.NewClient(remoteStore, "")
		s.artwork = store.NewFileSystem(store.NewClient(remoteStore, "artwork"))
	}
	s.media = store.NewRemoteChunkedFileSystem(c, 32*1024)
	return nil
}

func buildLocalStore(s *stores) {
	if localStore != "" {
		fs := http.Dir(localStore)
		if s.media != nil {
			s.media = store.MultiFileSystem(fs, s.media)
		} else {
			s.media = fs
		}

		afs := store.ArtworkFileSystem(fs)
		if s.artwork != nil {
			s.artwork = store.MultiFileSystem(afs, s.artwork)
		} else {
			s.artwork = afs
		}
	}
}

func buildMediaCache(s *stores) {
	if mediaFileSystemCache != "" {
		var errCh <-chan error
		localCache := store.Dir(mediaFileSystemCache)
		s.media, errCh = store.NewCachedFileSystem(
			s.media,
			localCache,
		)
		go func() {
			for err := range errCh {
				// TODO: pull this out!
				log.Printf("mediaFileSystem cache: %v", err)
			}
		}()
	}
}

func buildArtworkCache(s *stores) error {
	if artworkFileSystemCache != "" {
		cfs, err := cafs.New(store.Dir(artworkFileSystemCache))
		if err != nil {
			return fmt.Errorf("error creating artwork cafs: %v", err)
		}

		var errCh <-chan error
		s.artwork, errCh = store.NewCachedFileSystem(
			s.artwork,
			cfs,
		)
		go func() {
			for err := range errCh {
				// TODO: pull this out!
				log.Printf("artwork cache: %v", err)
			}
		}()
	}
	return nil
}

// Stores returns a media and artwork filesystem as defined by the command line flags.
func Stores() (media, artwork http.FileSystem, err error) {
	s := &stores{}
	err = buildRemoteStore(s)
	if err != nil {
		return nil, nil, err
	}

	buildLocalStore(s)
	buildMediaCache(s)

	err = buildArtworkCache(s)
	if err != nil {
		return nil, nil, err
	}

	if trimPathPrefix != "" || addPathPrefix != "" {
		s.media = store.NewPathRewrite(s.media, trimPathPrefix, addPathPrefix)
		s.artwork = store.NewPathRewrite(s.artwork, trimPathPrefix, addPathPrefix)
	}
	return s.media, s.artwork, nil
}
<|endoftext|>


<commit_before>package store

import (
	"errors"

	consul "github.com/armon/consul-api"
)

type ConsulStore struct {
	Client *consul.Client
}

func (c *ConsulStore) Get(key string) (Item, error) {
	kv, _, err := c.Client.KV().Get(key, nil)
	if err != nil {
		return nil, err
	}
	if kv == nil {
		return nil, errors.New("key not found")
	}

	return &ConsulItem{
		key:   kv.Key,
		value: kv.Value,
	}, nil
}

func (c *ConsulStore) Del(key string) error {
	_, err := c.Client.KV().Delete(key, nil)
	return err
}

func (c *ConsulStore) Put(item Item) error {
	_, err := c.Client.KV().Put(&consul.KVPair{
		Key:   item.Key(),
		Value: item.Value(),
	}, nil)

	return err
}

func (c *ConsulStore) NewItem(key string, value []byte) Item {
	return &ConsulItem{
		key:   key,
		value: value,
	}
}

func NewConsulStore() Store {
	client, _ := consul.NewClient(&consul.Config{})

	return &ConsulStore{
		Client: client,
	}
}
<commit_msg>Update the api in store package, github.com/armon/consul-api is deprecated<commit_after>package store

import (
	"errors"

	consul "github.com/hashicorp/consul/api"
)

type ConsulStore struct {
	Client *consul.Client
}

func (c *ConsulStore) Get(key string) (Item, error) {
	kv, _, err := c.Client.KV().Get(key, nil)
	if err != nil {
		return nil, err
	}
	if kv == nil {
		return nil, errors.New("key not found")
	}

	return &ConsulItem{
		key:   kv.Key,
		value: kv.Value,
	}, nil
}

func (c *ConsulStore) Del(key string) error {
	_, err := c.Client.KV().Delete(key, nil)
	return err
}

func (c *ConsulStore) Put(item Item) error {
	_, err := c.Client.KV().Put(&consul.KVPair{
		Key:   item.Key(),
		Value: item.Value(),
	}, nil)

	return err
}

func (c *ConsulStore) NewItem(key string, value []byte) Item {
	return &ConsulItem{
		key:   key,
		value: value,
	}
}

func NewConsulStore() Store {
	client, _ := consul.NewClient(consul.DefaultConfig())

	return &ConsulStore{
		Client: client,
	}
}
<|endoftext|>


<commit_before>package router

import (
	"compress/gzip"
	"context"
	"encoding/json"
	"errors"
	"io"
	"net/http"
	"strings"

	"github.com/alphastyle/logger"
)

// mux is the multiplexer struct
type mux struct {
	*http.ServeMux
	middle     []handlerFunc
	middleHTTP []http.HandlerFunc
}

// group is to divide request middleware
type group struct {
	*mux
	middleware []handlerFunc
	prefix     string
}

// Context is a custome ResponseWriter and Request
type Context struct {
	http.ResponseWriter
	*http.Request
}

// handlerFunc is custome http.HandleFunc type
type handlerFunc func(Context)

// Gzip Compression
type gzipResponseWriter struct {
	io.Writer
	http.ResponseWriter
}

// Gzip Write
func (w gzipResponseWriter) Write(b []byte) (int, error) {
	return w.Writer.Write(b)
}

// New will create a new group
func New() *group {
	m := &mux{
		ServeMux: http.NewServeMux(),
	}

	return &group{
		mux: m,
	}
}

func (h handlerFunc) ServeHTTP(c Context) {
	h(c)
}

// HandlerFunc is a custom http.HandlerFunc
func (g *group) HandlerFunc(pattern string, h http.HandlerFunc) {
	handler := g.handleRequestHTTP(h, "ALL")
	g.Handle(g.prefix+pattern, http.HandlerFunc(handler))
}

// GET is a custom http.HandlerFunc that only allow GET requests
func (g *group) GET(pattern string, h handlerFunc) {
	handler := g.handleRequest(h, "GET")
	g.Handle(g.prefix+pattern, http.HandlerFunc(handler))
}

// POST is a custom http.HandlerFunc that only allow POST requests
func (g *group) POST(pattern string, h handlerFunc) {
	handler := g.handleRequest(h, "POST")
	g.Handle(g.prefix+pattern, http.HandlerFunc(handler))
}

// handleRequestHTTP will handle http.HandlerFunc requests
// and handle middleware
func (g *group) handleRequestHTTP(h http.HandlerFunc, method string) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if method == "ALL" {
			g.handleMiddlewareHTTP(w, r)
			h.ServeHTTP(w, r)
		} else {
			http.NotFound(w, r)
		}
	}
}

// handleRequest will check the request method and handle middleware
func (g *group) handleRequest(h handlerFunc, method string) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if r.Method == method {
			mrw := Context{w, r}
			g.handleMiddleware(mrw)
			h.ServeHTTP(mrw)
		} else {
			http.NotFound(w, r)
		}
	}
}

// handleMiddlewareHTTP will serve the correct middleware for the request
// meant for http.HandlerFunc
func (g *group) handleMiddlewareHTTP(w http.ResponseWriter, r *http.Request) {
	// Global Middleware
	for _, v := range g.middleHTTP {
		v.ServeHTTP(w, r)
	}

	// Group Middleware
	// TODO make middleware work for HandlerFunc (http.HanderFunc)
	// for _, v := range g.middleware {
	// 	v.ServeHTTP(w, r)
	// }
}

// handleMiddleware will serve the correct middleware for the request
func (g *group) handleMiddleware(c Context) {
	// Global Middleware
	for _, v := range g.middle {
		v.ServeHTTP(c)
	}

	// Group Middleware
	for _, v := range g.middleware {
		v.ServeHTTP(c)
	}
}

// Use is to make custome global middleware
func (g *group) Use(h ...handlerFunc) {
	if g.prefix == "" {
		for _, v := range h {
			g.middle = append(g.middle, v)
		}
	}
}

// Use is to make custome global middleware
func (g *group) UseHTTP(h ...http.HandlerFunc) {
	if g.prefix == "" {
		for _, v := range h {
			g.middleHTTP = append(g.middleHTTP, v)
		}
	}
}

// Group makes it possible to have custome group middleware
func (g *group) Group(pattern string, h ...handlerFunc) *group {
	// Initialize new group
	newGroup := &group{
		mux: g.mux,
	}

	if pattern != "" && strings.HasPrefix(pattern, "/") {
		newGroup.prefix = pattern

		for _, v := range h {
			newGroup.middleware = append(newGroup.middleware, v)
		}

	} else {
		err := errors.New("Url pattern can't be empty and has to start with / (slash)!")
		logger.Error(err, "Group error")
	}

	return newGroup
}

// Gzip compress all served files
func Gzip(handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		// Allow the browser to cache content for 1 day (less traffic)
		w.Header().Set("Cache-Control", "max-age:86400")

		// if request does not accept Gzip then return without Gzip
		if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
			handler.ServeHTTP(w, r)
		}

		// Allow gzip
		w.Header().Set("Content-Encoding", "gzip")
		gz := gzip.NewWriter(w)
		defer gz.Close()
		gzw := gzipResponseWriter{Writer: gz, ResponseWriter: w}
		handler.ServeHTTP(gzw, r)
	})
}

// ServeFiles serve static files
func (g *group) ServeFiles(urlPath string, dirPath string, prefix string) {
	g.Handle(urlPath, Gzip(http.StripPrefix(prefix, http.FileServer(http.Dir(dirPath)))))
}

// ServeFavicon will serve the favicon you choose
func (g *group) ServeFavicon(filePath string) {
	g.HandlerFunc("/favicon.ico", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, filePath)
	})
}

// JSON for json handling
func (c Context) JSON(data interface{}) {
	Data, err := json.Marshal(data)
	if err != nil {
		logger.Error(err, "JSON Marshal error")
	}

	c.ResponseWriter.Header().Set("Content-Type", "application/json")
	c.ResponseWriter.Write(Data)
}

func (c Context) Write(str string) {
	c.ResponseWriter.Write([]byte(str))
}

// NewContext creates and return the request with context
func (c Context) NewContext(key, value interface{}) {
	ctx := c.Context()
	ctx = context.WithValue(ctx, key, value)
	c.Request = c.WithContext(ctx)
}

// GetContext will get the context from the spesefic request
func (c Context) GetContext(key string) interface{} {
	ctx := c.Context()
	val := ctx.Value(key)
	return val
}

// Listen will start the server (http.ListenAndServe)
func (g *group) Listen(serve string) error {
	// listening @ :PORT
	logger.Info("listening @" + serve)

	// start listening
	return http.ListenAndServe(serve, g)
}
<commit_msg>pointer to Context struct<commit_after>package router

import (
	"compress/gzip"
	"context"
	"encoding/json"
	"errors"
	"io"
	"net/http"
	"strings"

	"github.com/alphastyle/logger"
)

// mux is the multiplexer struct
type mux struct {
	*http.ServeMux
	middle     []handlerFunc
	middleHTTP []http.HandlerFunc
}

// group is to divide request middleware
type group struct {
	*mux
	middleware []handlerFunc
	prefix     string
}

// Context is a custome ResponseWriter and Request
type Context struct {
	http.ResponseWriter
	*http.Request
}

// handlerFunc is custome http.HandleFunc type
type handlerFunc func(*Context)

// Gzip Compression
type gzipResponseWriter struct {
	io.Writer
	http.ResponseWriter
}

// Gzip Write
func (w gzipResponseWriter) Write(b []byte) (int, error) {
	return w.Writer.Write(b)
}

// New will create a new group
func New() *group {
	m := &mux{
		ServeMux: http.NewServeMux(),
	}

	return &group{
		mux: m,
	}
}

func (h handlerFunc) ServeHTTP(c *Context) {
	h(c)
}

// HandlerFunc is a custom http.HandlerFunc
func (g *group) HandlerFunc(pattern string, h http.HandlerFunc) {
	handler := g.handleRequestHTTP(h, "ALL")
	g.Handle(g.prefix+pattern, http.HandlerFunc(handler))
}

// GET is a custom http.HandlerFunc that only allow GET requests
func (g *group) GET(pattern string, h handlerFunc) {
	handler := g.handleRequest(h, "GET")
	g.Handle(g.prefix+pattern, http.HandlerFunc(handler))
}

// POST is a custom http.HandlerFunc that only allow POST requests
func (g *group) POST(pattern string, h handlerFunc) {
	handler := g.handleRequest(h, "POST")
	g.Handle(g.prefix+pattern, http.HandlerFunc(handler))
}

// handleRequestHTTP will handle http.HandlerFunc requests
// and handle middleware
func (g *group) handleRequestHTTP(h http.HandlerFunc, method string) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if method == "ALL" {
			g.handleMiddlewareHTTP(w, r)
			h.ServeHTTP(w, r)
		} else {
			http.NotFound(w, r)
		}
	}
}

// handleRequest will check the request method and handle middleware
func (g *group) handleRequest(h handlerFunc, method string) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if r.Method == method {
			mrw := &Context{w, r}
			g.handleMiddleware(mrw)
			h.ServeHTTP(mrw)
		} else {
			http.NotFound(w, r)
		}
	}
}

// handleMiddlewareHTTP will serve the correct middleware for the request
// meant for http.HandlerFunc
func (g *group) handleMiddlewareHTTP(w http.ResponseWriter, r *http.Request) {
	// Global Middleware
	for _, v := range g.middleHTTP {
		v.ServeHTTP(w, r)
	}

	// Group Middleware
	// TODO make middleware work for HandlerFunc (http.HanderFunc)
	// for _, v := range g.middleware {
	// 	v.ServeHTTP(w, r)
	// }
}

// handleMiddleware will serve the correct middleware for the request
func (g *group) handleMiddleware(c *Context) {
	// Global Middleware
	for _, v := range g.middle {
		v.ServeHTTP(c)
	}

	// Group Middleware
	for _, v := range g.middleware {
		v.ServeHTTP(c)
	}
}

// Use is to make custome global middleware
func (g *group) Use(h ...handlerFunc) {
	if g.prefix == "" {
		for _, v := range h {
			g.middle = append(g.middle, v)
		}
	}
}

// Use is to make custome global middleware
func (g *group) UseHTTP(h ...http.HandlerFunc) {
	if g.prefix == "" {
		for _, v := range h {
			g.middleHTTP = append(g.middleHTTP, v)
		}
	}
}

// Group makes it possible to have custome group middleware
func (g *group) Group(pattern string, h ...handlerFunc) *group {
	// Initialize new group
	newGroup := &group{
		mux: g.mux,
	}

	if pattern != "" && strings.HasPrefix(pattern, "/") {
		newGroup.prefix = pattern

		for _, v := range h {
			newGroup.middleware = append(newGroup.middleware, v)
		}

	} else {
		err := errors.New("Url pattern can't be empty and has to start with / (slash)!")
		logger.Error(err, "Group error")
	}

	return newGroup
}

// Gzip compress all served files
func Gzip(handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		// Allow the browser to cache content for 1 day (less traffic)
		w.Header().Set("Cache-Control", "max-age:86400")

		// if request does not accept Gzip then return without Gzip
		if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
			handler.ServeHTTP(w, r)
		}

		// Allow gzip
		w.Header().Set("Content-Encoding", "gzip")
		gz := gzip.NewWriter(w)
		defer gz.Close()
		gzw := gzipResponseWriter{Writer: gz, ResponseWriter: w}
		handler.ServeHTTP(gzw, r)
	})
}

// ServeFiles serve static files
func (g *group) ServeFiles(urlPath string, dirPath string, prefix string) {
	g.Handle(urlPath, Gzip(http.StripPrefix(prefix, http.FileServer(http.Dir(dirPath)))))
}

// ServeFavicon will serve the favicon you choose
func (g *group) ServeFavicon(filePath string) {
	g.HandlerFunc("/favicon.ico", func(w http.ResponseWriter, r *http.Request) {
		http.ServeFile(w, r, filePath)
	})
}

// JSON for json handling
func (c *Context) JSON(data interface{}) {
	Data, err := json.Marshal(data)
	if err != nil {
		logger.Error(err, "JSON Marshal error")
	}

	c.ResponseWriter.Header().Set("Content-Type", "application/json")
	c.ResponseWriter.Write(Data)
}

func (c *Context) Write(str string) {
	c.ResponseWriter.Write([]byte(str))
}

// NewContext creates and return the request with context
func (c *Context) NewContext(key, value interface{}) {
	ctx := c.Context()
	ctx = context.WithValue(ctx, key, value)
	c.Request = c.WithContext(ctx)
}

// GetContext will get the context from the spesefic request
func (c *Context) GetContext(key string) interface{} {
	ctx := c.Context()
	val := ctx.Value(key)
	return val
}

// Listen will start the server (http.ListenAndServe)
func (g *group) Listen(serve string) error {
	// listening @ :PORT
	logger.Info("listening @" + serve)

	// start listening
	return http.ListenAndServe(serve, g)
}
<|endoftext|>


<commit_before>package main

import (
  "log"
  "net/http"
  "github.com/gorilla/mux"
)

// A common type used to express what middleware and route handler function signatures look like
type HttpHandler func(w http.ResponseWriter, r *http.Request)

// The MethodRouter type provides a convenience wrapper around mux.Router that provides easy-to-use
// http method-based routing.
type MethodRouter struct {
  methods []string
  primaryRouter *mux.Router
  subRouters map[string]*mux.Router
}

// Get a new MethodRouter
func NewMethodRouter() *MethodRouter {
  methods := []string{"GET", "POST", "PUT", "DELETE"}
  router := &MethodRouter{methods, mux.NewRouter(), make(map[string]*mux.Router)}

  for _, method := range router.methods {
    router.subRouters[method] = router.primaryRouter.Methods(method).Subrouter()
  }

  return router
}

// Register a router. See mux.Router documentation for regexp path rules, etc.
func (mr MethodRouter) HandleFunc(method string, path string, handleFunc HttpHandler) {
  mr.subRouters[method].HandleFunc(path, DecoratorMdw(handleFunc))
  return
}

// Start server.
func (mr MethodRouter) ListenAndServe(addr string) {
  log.Fatal(http.ListenAndServe(addr, mr.primaryRouter))
  return
}<commit_msg>more documentation for MethodRouter.HandleFunc<commit_after>package main

import (
  "log"
  "net/http"
  "github.com/gorilla/mux"
)

// A common type used to express what middleware and route handler function signatures look like
type HttpHandler func(w http.ResponseWriter, r *http.Request)

// The MethodRouter type provides a convenience wrapper around mux.Router that provides easy-to-use
// http method-based routing.
type MethodRouter struct {
  methods []string
  primaryRouter *mux.Router
  subRouters map[string]*mux.Router
}

// Get a new MethodRouter
func NewMethodRouter() *MethodRouter {
  methods := []string{"GET", "POST", "PUT", "DELETE"}
  router := &MethodRouter{methods, mux.NewRouter(), make(map[string]*mux.Router)}

  for _, method := range router.methods {
    router.subRouters[method] = router.primaryRouter.Methods(method).Subrouter()
  }

  return router
}

// Register a route. See mux.Router documentation for regexp path rules, etc.
//
// This method also wraps the provided handler with a decorator middleware.
func (mr MethodRouter) HandleFunc(method string, path string, handleFunc HttpHandler) {
  mr.subRouters[method].HandleFunc(path, DecoratorMdw(handleFunc))
  return
}

// Start server.
func (mr MethodRouter) ListenAndServe(addr string) {
  log.Fatal(http.ListenAndServe(addr, mr.primaryRouter))
  return
}<|endoftext|>