<commit_before>/*
Copyright 2017 The OpenEBS Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package command

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"net/http"
	"strings"
	"time"

	"github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1"
	"github.com/openebs/maya/pkg/client/mapiserver"
	"github.com/openebs/maya/pkg/util"
	"github.com/spf13/cobra"
)

// VolumeInfo stores the volume information
type VolumeInfo struct {
	Volume v1alpha1.CASVolume
}

// CmdVolumeOptions stores information of volume being operated
type CmdVolumeOptions struct {
	volName          string
	sourceVolumeName string
	snapshotName     string
	size             string
	namespace        string
	json             string
}

// CASType is engine type
type CASType string

const (
	// VolumeAPIPath is the api path to get volume information
	VolumeAPIPath      = "/latest/volumes/"
	controllerStatusOk = "running"
	volumeStatusOK     = "Running"
	// JivaStorageEngine is constant for jiva engine
	JivaStorageEngine CASType = "jiva"
	// CstorStorageEngine is constant for cstor engine
	CstorStorageEngine CASType = "cstor"
	timeout                    = 5 * time.Second
)

// # Create a Volume:
// $ mayactl volume create --volname <vol> --size <size>

var (
	volumeCommandHelpText = `
The following commands helps in operating a Volume such as create, list, and so on.

Usage: mayactl volume <subcommand> [options] [args]

Examples:
 # List Volumes:
   $ mayactl volume list

 # Statistics of a Volume:
   $ mayactl volume stats --volname <vol>

 # Statistics of a Volume created in 'test' namespace:
   $ mayactl volume stats --volname <vol> --namespace test

 # Info of a Volume:
   $ mayactl volume info --volname <vol>

 # Info of a Volume created in 'test' namespace:
   $ mayactl volume info --volname <vol> --namespace test

 # Delete a Volume:
   $ mayactl volume delete --volname <vol>

 # Delete a Volume created in 'test' namespace:
   $ mayactl volume delete --volname <vol> --namespace test
`
	options = &CmdVolumeOptions{
		namespace: "default",
	}
)

// NewCmdVolume provides options for managing OpenEBS Volume
func NewCmdVolume() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "volume",
		Short: "Provides operations related to a Volume",
		Long:  volumeCommandHelpText,
	}

	cmd.AddCommand(
		// NewCmdVolumeCreate(),
		NewCmdVolumesList(),
		NewCmdVolumeDelete(),
		NewCmdVolumeStats(),
		NewCmdVolumeInfo(),
	)
	cmd.PersistentFlags().StringVarP(&options.namespace, "namespace", "n", options.namespace,
		"namespace name, required if volume is not in the default namespace")

	cmd.PersistentFlags().AddGoFlagSet(flag.CommandLine)
	flag.CommandLine.Parse([]string{})
	return cmd
}

// Validate verifies whether a volume name,source name or snapshot name is provided or not followed by
// stats command. It returns nil and proceeds to execute the command if there is
// no error and returns an error if it is missing.
func (c *CmdVolumeOptions) Validate(cmd *cobra.Command, snapshotnameverify, sourcenameverify, volnameverify bool) error {
	if snapshotnameverify {
		if len(c.snapshotName) == 0 {
			return errors.New("--snapname is missing. Please provide a snapshotname")
		}
	}
	if sourcenameverify {
		if len(c.sourceVolumeName) == 0 {
			return errors.New("--sourcevol is missing. Please specify a sourcevolumename")
		}
	}
	if volnameverify {
		if len(c.volName) == 0 {
			return errors.New("--volname is missing. Please specify a unique volumename")
		}
	}
	return nil
}

// NewVolumeInfo fetches and fills CASVolume structure from URL given to it
func NewVolumeInfo(URL string, volname string, namespace string) (volInfo *VolumeInfo, err error) {
	url := URL
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return
	}
	req.Header.Set("namespace", namespace)

	c := &http.Client{
		Timeout: timeout,
	}
	resp, err := c.Do(req)
	if err != nil {
		fmt.Printf("Can't get a response, error found: %v", err)
		return
	}
	if resp != nil && resp.StatusCode != 200 {
		if resp.StatusCode == 500 {
			fmt.Printf("Sorry something went wrong with service. Please raise an issue on: https://github.com/openebs/openebs/issues")
			err = util.InternalServerError
			return
		} else if resp.StatusCode == 503 {
			fmt.Printf("maya apiservice not reachable at %q\n", mapiserver.GetURL())
			err = util.ServerUnavailable
			return
		} else if resp.StatusCode == 404 {
			fmt.Printf("Volume: %s not found at namespace: %q error: %s\n", volname, namespace, http.StatusText(resp.StatusCode))
			err = util.PageNotFound
			return
		}
		fmt.Printf("Received an error from maya apiservice: statuscode: %d", resp.StatusCode)
		err = fmt.Errorf("Received an error from maya apiservice: statuscode: %d", resp.StatusCode)
		return
	}
	defer resp.Body.Close()
	casVol := v1alpha1.CASVolume{}
	err = json.NewDecoder(resp.Body).Decode(&casVol)
	if err != nil {
		fmt.Printf("Response decode failed: error '%+v'", err)
		return
	}
	if casVol.Status.Reason == "pending" {
		fmt.Println("VOLUME status Unknown to maya apiservice")
		err = fmt.Errorf("VOLUME status Unknown to maya apiservice")
		return
	}
	volInfo = &VolumeInfo{
		Volume: casVol,
	}
	return
}

// GetCASType returns the CASType of the volume in lowercase
func (volInfo *VolumeInfo) GetCASType() string {
	if len(volInfo.Volume.Spec.CasType) == 0 {
		return string(JivaStorageEngine)
	}
	return strings.ToLower(volInfo.Volume.Spec.CasType)
}

// GetClusterIP returns the ClusterIP of the cluster
func (volInfo *VolumeInfo) GetClusterIP() string {
	if val, ok := volInfo.Volume.ObjectMeta.Annotations["openebs.io/cluster-ips"]; ok {
		return val
	} else if val, ok := volInfo.Volume.ObjectMeta.Annotations["vsm.openebs.io/cluster-ips"]; ok {
		return val
	}
	return ""
}

// GetControllerStatus returns the status of the volume controller
func (volInfo *VolumeInfo) GetControllerStatus() string {
	if val, ok := volInfo.Volume.ObjectMeta.Annotations["openebs.io/controller-status"]; ok {
		return val
	} else if val, ok := volInfo.Volume.ObjectMeta.Annotations["vsm.openebs.io/controller-status"]; ok {
		return val
	}
	return ""
}

// GetIQN returns the IQN of the volume
func (volInfo *VolumeInfo) GetIQN() string {
	if len(volInfo.Volume.Spec.Iqn) > 0 {
		return volInfo.Volume.Spec.Iqn
	} else if val, ok := volInfo.Volume.ObjectMeta.Annotations["openebs.io/iqn"]; ok {
		return val
	} else if val, ok := volInfo.Volume.ObjectMeta.Annotations["vsm.openebs.io/iqn"]; ok {
		return val
	}
	return ""

}

// GetVolumeName returns the volume name
func (volInfo *VolumeInfo) GetVolumeName() string {
	return volInfo.Volume.ObjectMeta.Name
}

// GetTargetPortal returns the TargetPortal of the volume
func (volInfo *VolumeInfo) GetTargetPortal() string {
	if len(volInfo.Volume.Spec.TargetPortal) > 0 {
		return volInfo.Volume.Spec.TargetPortal
	} else if val, ok := volInfo.Volume.ObjectMeta.Annotations["openebs.io/targetportals"]; ok {
		return val
	} else if val, ok := volInfo.Volume.ObjectMeta.Annotations["vsm.openebs.io/targetportals"]; ok {
		return val
	}
	return ""
}

// GetVolumeSize returns the capacity of the volume
func (volInfo *VolumeInfo) GetVolumeSize() string {
	if len(volInfo.Volume.Spec.Capacity) > 0 {
		return volInfo.Volume.Spec.Capacity
	} else if val, ok := volInfo.Volume.ObjectMeta.Annotations["openebs.io/volume-size"]; ok {
		return val
	} else if val, ok := volInfo.Volume.ObjectMeta.Annotations["vsm.openebs.io/volume-size"]; ok {
		return val
	}
	return ""
}

// GetReplicaCount returns the volume replica count
func (volInfo *VolumeInfo) GetReplicaCount() string {
	if len(volInfo.Volume.Spec.Replicas) > 0 {
		return volInfo.Volume.Spec.Replicas
	} else if val, ok := volInfo.Volume.ObjectMeta.Annotations["openebs.io/replica-count"]; ok {
		return val
	} else if val, ok := volInfo.Volume.ObjectMeta.Annotations["vsm.openebs.io/replica-count"]; ok {
		return val
	}
	return ""
}

// GetReplicaStatus returns the replica status of the volume replica
func (volInfo *VolumeInfo) GetReplicaStatus() string {
	if val, ok := volInfo.Volume.ObjectMeta.Annotations["openebs.io/replica-status"]; ok {
		return val
	} else if val, ok := volInfo.Volume.ObjectMeta.Annotations["vsm.openebs.io/replica-status"]; ok {
		return val
	}
	return ""
}

// GetReplicaIP returns the IP of volume replica
func (volInfo *VolumeInfo) GetReplicaIP() string {
	if val, ok := volInfo.Volume.ObjectMeta.Annotations["openebs.io/replica-ips"]; ok {
		return val
	} else if val, ok := volInfo.Volume.ObjectMeta.Annotations["vsm.openebs.io/replica-ips"]; ok {
		return val
	}
	return ""
}

// GetStoragePool returns the name of the storage pool
func (volInfo *VolumeInfo) GetStoragePool() string {
	if val, ok := volInfo.Volume.ObjectMeta.Annotations["openebs.io/pool-names"]; ok {
		return val
	}
	return ""
}

// GetCVRName returns the name of the CVR
func (volInfo *VolumeInfo) GetCVRName() string {
	if val, ok := volInfo.Volume.ObjectMeta.Annotations["openebs.io/cvr-names"]; ok {
		return val
	}
	return ""
}

// GetNodeName returns the name of the node
func (volInfo *VolumeInfo) GetNodeName() string {
	if val, ok := volInfo.Volume.ObjectMeta.Annotations["openebs.io/node-names"]; ok {
		return val
	}
	return ""
}

// GetControllerNode returns the node name of the controller
func (volInfo *VolumeInfo) GetControllerNode() string {
	if val, ok := volInfo.Volume.ObjectMeta.Annotations["openebs.io/controller-node-name"]; ok {
		return val
	}
	return ""
}
<commit_msg>Issue-1991 -> changed error message<commit_after>/*
Copyright 2017 The OpenEBS Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package command

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"net/http"
	"strings"
	"time"

	"github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1"
	"github.com/openebs/maya/pkg/client/mapiserver"
	"github.com/openebs/maya/pkg/util"
	"github.com/spf13/cobra"
)

// VolumeInfo stores the volume information
type VolumeInfo struct {
	Volume v1alpha1.CASVolume
}

// CmdVolumeOptions stores information of volume being operated
type CmdVolumeOptions struct {
	volName          string
	sourceVolumeName string
	snapshotName     string
	size             string
	namespace        string
	json             string
}

// CASType is engine type
type CASType string

const (
	// VolumeAPIPath is the api path to get volume information
	VolumeAPIPath      = "/latest/volumes/"
	controllerStatusOk = "running"
	volumeStatusOK     = "Running"
	// JivaStorageEngine is constant for jiva engine
	JivaStorageEngine CASType = "jiva"
	// CstorStorageEngine is constant for cstor engine
	CstorStorageEngine CASType = "cstor"
	timeout                    = 5 * time.Second
)

// # Create a Volume:
// $ mayactl volume create --volname <vol> --size <size>

var (
	volumeCommandHelpText = `
The following commands helps in operating a Volume such as create, list, and so on.

Usage: mayactl volume <subcommand> [options] [args]

Examples:
 # List Volumes:
   $ mayactl volume list

 # Statistics of a Volume:
   $ mayactl volume stats --volname <vol>

 # Statistics of a Volume created in 'test' namespace:
   $ mayactl volume stats --volname <vol> --namespace test

 # Info of a Volume:
   $ mayactl volume info --volname <vol>

 # Info of a Volume created in 'test' namespace:
   $ mayactl volume info --volname <vol> --namespace test

 # Delete a Volume:
   $ mayactl volume delete --volname <vol>

 # Delete a Volume created in 'test' namespace:
   $ mayactl volume delete --volname <vol> --namespace test
`
	options = &CmdVolumeOptions{
		namespace: "default",
	}
)

// NewCmdVolume provides options for managing OpenEBS Volume
func NewCmdVolume() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "volume",
		Short: "Provides operations related to a Volume",
		Long:  volumeCommandHelpText,
	}

	cmd.AddCommand(
		// NewCmdVolumeCreate(),
		NewCmdVolumesList(),
		NewCmdVolumeDelete(),
		NewCmdVolumeStats(),
		NewCmdVolumeInfo(),
	)
	cmd.PersistentFlags().StringVarP(&options.namespace, "namespace", "n", options.namespace,
		"namespace name, required if volume is not in the default namespace")

	cmd.PersistentFlags().AddGoFlagSet(flag.CommandLine)
	flag.CommandLine.Parse([]string{})
	return cmd
}

// Validate verifies whether a volume name,source name or snapshot name is provided or not followed by
// stats command. It returns nil and proceeds to execute the command if there is
// no error and returns an error if it is missing.
func (c *CmdVolumeOptions) Validate(cmd *cobra.Command, snapshotnameverify, sourcenameverify, volnameverify bool) error {
	if snapshotnameverify {
		if len(c.snapshotName) == 0 {
			return errors.New("--snapname is missing. Please provide a snapshotname")
		}
	}
	if sourcenameverify {
		if len(c.sourceVolumeName) == 0 {
			return errors.New("--sourcevol is missing. Please specify a sourcevolumename")
		}
	}
	if volnameverify {
		if len(c.volName) == 0 {
			return errors.New("error: --volname not specified")
		}
	}
	return nil
}

// NewVolumeInfo fetches and fills CASVolume structure from URL given to it
func NewVolumeInfo(URL string, volname string, namespace string) (volInfo *VolumeInfo, err error) {
	url := URL
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return
	}
	req.Header.Set("namespace", namespace)

	c := &http.Client{
		Timeout: timeout,
	}
	resp, err := c.Do(req)
	if err != nil {
		fmt.Printf("Can't get a response, error found: %v", err)
		return
	}
	if resp != nil && resp.StatusCode != 200 {
		if resp.StatusCode == 500 {
			fmt.Printf("Sorry something went wrong with service. Please raise an issue on: https://github.com/openebs/openebs/issues")
			err = util.InternalServerError
			return
		} else if resp.StatusCode == 503 {
			fmt.Printf("maya apiservice not reachable at %q\n", mapiserver.GetURL())
			err = util.ServerUnavailable
			return
		} else if resp.StatusCode == 404 {
			fmt.Printf("Volume: %s not found at namespace: %q error: %s\n", volname, namespace, http.StatusText(resp.StatusCode))
			err = util.PageNotFound
			return
		}
		fmt.Printf("Received an error from maya apiservice: statuscode: %d", resp.StatusCode)
		err = fmt.Errorf("Received an error from maya apiservice: statuscode: %d", resp.StatusCode)
		return
	}
	defer resp.Body.Close()
	casVol := v1alpha1.CASVolume{}
	err = json.NewDecoder(resp.Body).Decode(&casVol)
	if err != nil {
		fmt.Printf("Response decode failed: error '%+v'", err)
		return
	}
	if casVol.Status.Reason == "pending" {
		fmt.Println("VOLUME status Unknown to maya apiservice")
		err = fmt.Errorf("VOLUME status Unknown to maya apiservice")
		return
	}
	volInfo = &VolumeInfo{
		Volume: casVol,
	}
	return
}

// GetCASType returns the CASType of the volume in lowercase
func (volInfo *VolumeInfo) GetCASType() string {
	if len(volInfo.Volume.Spec.CasType) == 0 {
		return string(JivaStorageEngine)
	}
	return strings.ToLower(volInfo.Volume.Spec.CasType)
}

// GetClusterIP returns the ClusterIP of the cluster
func (volInfo *VolumeInfo) GetClusterIP() string {
	if val, ok := volInfo.Volume.ObjectMeta.Annotations["openebs.io/cluster-ips"]; ok {
		return val
	} else if val, ok := volInfo.Volume.ObjectMeta.Annotations["vsm.openebs.io/cluster-ips"]; ok {
		return val
	}
	return ""
}

// GetControllerStatus returns the status of the volume controller
func (volInfo *VolumeInfo) GetControllerStatus() string {
	if val, ok := volInfo.Volume.ObjectMeta.Annotations["openebs.io/controller-status"]; ok {
		return val
	} else if val, ok := volInfo.Volume.ObjectMeta.Annotations["vsm.openebs.io/controller-status"]; ok {
		return val
	}
	return ""
}

// GetIQN returns the IQN of the volume
func (volInfo *VolumeInfo) GetIQN() string {
	if len(volInfo.Volume.Spec.Iqn) > 0 {
		return volInfo.Volume.Spec.Iqn
	} else if val, ok := volInfo.Volume.ObjectMeta.Annotations["openebs.io/iqn"]; ok {
		return val
	} else if val, ok := volInfo.Volume.ObjectMeta.Annotations["vsm.openebs.io/iqn"]; ok {
		return val
	}
	return ""

}

// GetVolumeName returns the volume name
func (volInfo *VolumeInfo) GetVolumeName() string {
	return volInfo.Volume.ObjectMeta.Name
}

// GetTargetPortal returns the TargetPortal of the volume
func (volInfo *VolumeInfo) GetTargetPortal() string {
	if len(volInfo.Volume.Spec.TargetPortal) > 0 {
		return volInfo.Volume.Spec.TargetPortal
	} else if val, ok := volInfo.Volume.ObjectMeta.Annotations["openebs.io/targetportals"]; ok {
		return val
	} else if val, ok := volInfo.Volume.ObjectMeta.Annotations["vsm.openebs.io/targetportals"]; ok {
		return val
	}
	return ""
}

// GetVolumeSize returns the capacity of the volume
func (volInfo *VolumeInfo) GetVolumeSize() string {
	if len(volInfo.Volume.Spec.Capacity) > 0 {
		return volInfo.Volume.Spec.Capacity
	} else if val, ok := volInfo.Volume.ObjectMeta.Annotations["openebs.io/volume-size"]; ok {
		return val
	} else if val, ok := volInfo.Volume.ObjectMeta.Annotations["vsm.openebs.io/volume-size"]; ok {
		return val
	}
	return ""
}

// GetReplicaCount returns the volume replica count
func (volInfo *VolumeInfo) GetReplicaCount() string {
	if len(volInfo.Volume.Spec.Replicas) > 0 {
		return volInfo.Volume.Spec.Replicas
	} else if val, ok := volInfo.Volume.ObjectMeta.Annotations["openebs.io/replica-count"]; ok {
		return val
	} else if val, ok := volInfo.Volume.ObjectMeta.Annotations["vsm.openebs.io/replica-count"]; ok {
		return val
	}
	return ""
}

// GetReplicaStatus returns the replica status of the volume replica
func (volInfo *VolumeInfo) GetReplicaStatus() string {
	if val, ok := volInfo.Volume.ObjectMeta.Annotations["openebs.io/replica-status"]; ok {
		return val
	} else if val, ok := volInfo.Volume.ObjectMeta.Annotations["vsm.openebs.io/replica-status"]; ok {
		return val
	}
	return ""
}

// GetReplicaIP returns the IP of volume replica
func (volInfo *VolumeInfo) GetReplicaIP() string {
	if val, ok := volInfo.Volume.ObjectMeta.Annotations["openebs.io/replica-ips"]; ok {
		return val
	} else if val, ok := volInfo.Volume.ObjectMeta.Annotations["vsm.openebs.io/replica-ips"]; ok {
		return val
	}
	return ""
}

// GetStoragePool returns the name of the storage pool
func (volInfo *VolumeInfo) GetStoragePool() string {
	if val, ok := volInfo.Volume.ObjectMeta.Annotations["openebs.io/pool-names"]; ok {
		return val
	}
	return ""
}

// GetCVRName returns the name of the CVR
func (volInfo *VolumeInfo) GetCVRName() string {
	if val, ok := volInfo.Volume.ObjectMeta.Annotations["openebs.io/cvr-names"]; ok {
		return val
	}
	return ""
}

// GetNodeName returns the name of the node
func (volInfo *VolumeInfo) GetNodeName() string {
	if val, ok := volInfo.Volume.ObjectMeta.Annotations["openebs.io/node-names"]; ok {
		return val
	}
	return ""
}

// GetControllerNode returns the node name of the controller
func (volInfo *VolumeInfo) GetControllerNode() string {
	if val, ok := volInfo.Volume.ObjectMeta.Annotations["openebs.io/controller-node-name"]; ok {
		return val
	}
	return ""
}
<|endoftext|>


<commit_before>package http_json_api

import (
	"encoding/json"
	"github.com/gogap/errors"
	"github.com/rs/xid"
	"io/ioutil"
	gohttp "net/http"
	"strconv"
	"strings"
	"time"

	"github.com/go-martini/martini"
	"github.com/gogap/spirit"
	"github.com/spirit-contrib/http"
)

var (
	receiverURN = "urn:spirit-contrib:receiver:http:json_api"
)

type JsonApiReceiver struct {
	*http.HTTPReceiver

	conf JsonApiReceiverConfig

	responseRenderer *APIResponseRenderer

	htmlProxy string
}

var (
	_ spirit.Receiver = new(JsonApiReceiver)
)

func init() {
	spirit.RegisterReceiver(receiverURN, NewJsonApiReceiver)
}

func NewJsonApiReceiver(config spirit.Config) (receiver spirit.Receiver, err error) {
	conf := JsonApiReceiverConfig{}

	if err = config.ToObject(&conf); err != nil {
		return
	}

	conf.initial()

	jsonApiReceiver := &JsonApiReceiver{
		conf: conf,
	}

	if jsonApiReceiver.HTTPReceiver, err = http.NewHTTPReceiver(conf.Http, jsonApiReceiver.requestHandler); err != nil {
		return
	}

	jsonApiReceiver.responseRenderer = NewAPIResponseRenderer()

	path := strings.TrimRight(conf.Path, "/")
	jsonApiReceiver.Group(path, func(r martini.Router) {
		r.Post("", jsonApiReceiver.HTTPReceiver.Handler)
		r.Post("/:apiName", jsonApiReceiver.HTTPReceiver.Handler)
		r.Options("", jsonApiReceiver.optionHandle)
		r.Options("/:apiName", jsonApiReceiver.optionHandle)
	})

	jsonApiReceiver.htmlProxy = strings.Replace(proxyHtml, "{{#XDomainLib#}}", conf.XDomain.LibUrl, 1)

	jsonApiReceiver.Group("/", func(r martini.Router) {
		r.Get("ping", func() string {
			return "pong"
		})

		r.Get(conf.XDomain.HtmlPath, func(r *gohttp.Request) string {
			refer := r.Referer()
			if refer == "" {
				refer = r.Header.Get("Origin")
			}

			html := jsonApiReceiver.htmlProxy

			if conf.XDomain.Masters != nil && refer != "" {
				protocol, domain := parseRefer(refer)
				origin := protocol + "://" + domain

				if path, exist := conf.XDomain.Masters[protocol+"://"+domain]; exist {
					master := map[string]string{origin: path}

					jsonData, _ := json.MarshalIndent(master, "", "  ")

					html = strings.Replace(html, "{{#Masters#}}", string(jsonData), 1)
					return html
				}
			}

			html = strings.Replace(html, "{{#Masters#}}", "{}", 1)

			return html
		})

		if conf.XDomain.LibPath != "" {
			r.Get(conf.XDomain.LibPath, func() string {
				return xdomainLib
			})
		}
	})

	receiver = jsonApiReceiver
	return
}

func (p *JsonApiReceiver) optionHandle(w gohttp.ResponseWriter, r *gohttp.Request) {
	if r.Method == "OPTIONS" {
		p.writeAccessHeaders(w, r)
		p.writeBasicHeaders(w, r)
		w.Write([]byte(""))
	}
}

func (p *JsonApiReceiver) requestHandler(
	res gohttp.ResponseWriter,
	req *gohttp.Request,
	deliveryChan <-chan spirit.Delivery,
	done chan<- bool,
) (deliveries []spirit.Delivery, err error) {

	var apiIds map[string]string

	// request to deliveries
	if deliveries, apiIds, err = p.toDeliveries(req); err != nil {

		var apiResponse APIResponse

		switch errCode := err.(type) {
		case errors.ErrCode:
			{
				apiResponse = APIResponse{
					Code:           errCode.Code(),
					ErrorId:        errCode.Id(),
					ErrorNamespace: errCode.Namespace(),
					Message:        errCode.Error(),
					Result:         nil,
				}
			}
		default:
			e := ErrApiGenericError.New().Append(err)
			apiResponse = APIResponse{
				Code:           e.Code(),
				ErrorId:        e.Id(),
				ErrorNamespace: e.Namespace(),
				Message:        e.Error(),
				Result:         nil,
			}
		}

		if data, e := json.Marshal(apiResponse); e != nil {
			spirit.Logger().WithField("event", "to deliveries").Println(err)
		} else {
			p.writeResponse(data, res, req)
		}

		return
	}

	go func(
		count int,
		apiIds map[string]string,
		res gohttp.ResponseWriter,
		req *gohttp.Request,
		deliveryChan <-chan spirit.Delivery,
		done chan<- bool) {

		defer func() {
			// notify the main handler finished
			select {
			case done <- true:
				{
				}
			case <-time.After(time.Second * 3):
				{
				}
			}
		}()

		// get timeout duration
		timeout := time.Duration(p.conf.Timeout) * time.Millisecond

		if strTimeout := req.Header.Get(p.conf.HeaderDefines.TimeoutHeader); strTimeout != "" {
			if i, e := strconv.Atoi(strTimeout); e == nil {
				timeout = time.Duration(i) * time.Millisecond
			}
		}

		if timeout <= 0 {
			timeout = DefaultTimeout
		}

		apiResponse := map[string]APIResponse{}

		i := count
		// get deliveries
	label_timeout_or_finished:
		for i > 0 {
			select {
			case delivery := <-deliveryChan:
				{
					if api, exist := apiIds[delivery.Id()]; !exist {
						spirit.Logger().
							WithField("api", api).
							WithField("delivery_id", delivery.Id()).
							Errorln("api not exist in request while delivery response")

					} else {
						apiResponse[api] = p.deliveryToApiResponse(delivery)
					}

					i = i - 1
				}
			case <-time.After(timeout):
				{
					break label_timeout_or_finished
				}
			}
		}

		// request timeout
		for _, api := range apiIds {
			if _, exist := apiResponse[api]; !exist {
				errCode := ErrRequestTimeout.New()
				apiResponse[api] = APIResponse{
					Code:           errCode.Code(),
					ErrorId:        errCode.Id(),
					ErrorNamespace: errCode.Namespace(),
					Message:        errCode.Error(),
					Result:         nil,
				}
			}
		}

		// render deliveries to json response
		// normal response: {"code": 0, "message": "", "result": null}
		// error response: {"code": 212, "error_namespace": "xxxx", "message": "something wrong", "result": null}

		isMultiCall := req.Header.Get(p.conf.HeaderDefines.MultiCallHeader) == "1" ||
			req.Header.Get(p.conf.HeaderDefines.MultiCallHeader) == "on" ||
			req.Header.Get(p.conf.HeaderDefines.MultiCallHeader) == "true"

		if renderedData, e := p.responseRenderer.Render(isMultiCall, apiResponse); e != nil {
			err := ErrRenderApiDataFailed.New(errors.Params{"err": e})
			resp := APIResponse{
				Code:           err.Code(),
				ErrorId:        err.Id(),
				ErrorNamespace: err.Namespace(),
				Message:        err.Error(),
				Result:         nil,
			}

			if errRespData, e := json.Marshal(resp); e != nil {
				strInternalErr := `{"code": 500, "message": "api server internal error", "result": null}`
				p.writeResponseWithStatusCode([]byte(strInternalErr), res, req, gohttp.StatusInternalServerError)
			} else {
				p.writeResponse(errRespData, res, req)
			}
		} else {
			p.writeResponse(renderedData, res, req)
		}
		return
	}(len(deliveries), apiIds, res, req, deliveryChan, done)

	return
}

func (p *JsonApiReceiver) toDeliveries(req *gohttp.Request) (deliveries []spirit.Delivery, apiIds map[string]string, err error) {
	isMultiCall := req.Header.Get(p.conf.HeaderDefines.MultiCallHeader) == "1" ||
		req.Header.Get(p.conf.HeaderDefines.MultiCallHeader) == "on" ||
		req.Header.Get(p.conf.HeaderDefines.MultiCallHeader) == "true"

	isForwarded := req.Header.Get(HeaderForwardedPayload) == "1" ||
		req.Header.Get(HeaderForwardedPayload) == "on" ||
		req.Header.Get(HeaderForwardedPayload) == "true"

	if isMultiCall == true && isForwarded == true {
		err = ErrNotSupportMultiCallForward.New()
		return
	}

	idMapping := make(map[string]string)

	var body []byte
	if body, err = ioutil.ReadAll(req.Body); err != nil {
		return
	}

	var apiDatas map[string]interface{} = make(map[string]interface{})

	if isMultiCall {
		if err = json.Unmarshal(body, &apiDatas); err != nil {
			return
		}
	} else {

		apiName := req.Header.Get(p.conf.HeaderDefines.ApiHeader)

		if apiName == "" {
			if p.conf.Path != req.RequestURI {
				apiName = strings.TrimPrefix(req.RequestURI, p.conf.Path)
				apiName = strings.TrimRight(apiName, "/")
			}
		}

		if apiName == "" {
			err = ErrApiNameIsEmpty
			return
		}

		if isForwarded {
			apiData := JsonPayload{}
			if json.Unmarshal(body, &apiData); err != nil {
				return
			}
			apiDatas[apiName] = apiData
		} else {
			var apiData map[string]interface{}
			if json.Unmarshal(body, &apiData); err != nil {
				return
			}
			apiDatas[apiName] = apiData
		}

	}

	var tmpDeliveries []spirit.Delivery
	for api, apiData := range apiDatas {

		payload := NewHttpJsonApiPayload()

		if isForwarded {
			if jsonPayload, ok := apiData.(JsonPayload); ok {
				payload.id = jsonPayload.Id
				payload.data = jsonPayload.Data
				payload.metadata = jsonPayload.Metadata
				payload.contexts = jsonPayload.Context

				if jsonPayload.Error != nil {
					errCode := errors.NewErrorCode(
						jsonPayload.Error.Id,
						jsonPayload.Error.Code,
						jsonPayload.Error.Namespace,
						jsonPayload.Error.Message,
						jsonPayload.Error.StackTrace,
						jsonPayload.Error.Context)

					payload.SetError(errCode)
				}
			}
		} else {
			payload.SetData(apiData)

			headerContexts := map[string]interface{}{}
			cookiesContexts := map[string]interface{}{}

			for _, key := range p.conf.ToContext.Headers {
				if req.Header.Get(key) != "" {
					headerContexts[key] = req.Header.Get(key)
				}
			}

			for _, key := range p.conf.ToContext.Cookies {
				if cookie, e := req.Cookie(key); e == nil {
					cookiesContexts[cookie.Name] = cookie
				}
			}

			if len(cookiesContexts) > 0 {
				payload.SetContext(CtxHttpCookies, cookiesContexts)
			}

			if len(headerContexts) > 0 {
				payload.SetContext(CtxHttpHeaders, headerContexts)
			}

			if len(p.conf.ToContext.Customs) > 0 {
				payload.SetContext(CtxHttpCustom, p.conf.ToContext.Customs)
			}
		}

		deliveryURN := ""
		if urn, exist := p.conf.ApiURN[api]; exist {
			deliveryURN = urn
		} else {
			deliveryURN = p.conf.BindURN
		}

		deliveryLabels := spirit.Labels{}
		if labels, exist := p.conf.ApiLabels[api]; exist {
			deliveryLabels = labels
		}

		de := &HttpJsonApiDelivery{
			id:        xid.New().String(),
			payload:   payload,
			urn:       deliveryURN,
			labels:    deliveryLabels,
			timestamp: time.Now(),
		}

		tmpDeliveries = append(tmpDeliveries, de)

		idMapping[de.id] = api
	}

	deliveries = tmpDeliveries
	apiIds = idMapping

	return
}

func (p *JsonApiReceiver) deliveryToApiResponse(delivery spirit.Delivery) (resp APIResponse) {

	var apiResp APIResponse

	toErrResponseFunc := func(err error) APIResponse {
		if errCode, ok := delivery.Payload().GetError().(errors.ErrCode); ok {
			return APIResponse{
				Code:           errCode.Code(),
				ErrorId:        errCode.Id(),
				ErrorNamespace: errCode.Namespace(),
				Message:        errCode.Error(),
				Result:         nil,
			}
		} else {
			errCode := ErrApiGenericError.New().Append(err)
			return APIResponse{
				Code:           errCode.Code(),
				ErrorId:        errCode.Id(),
				ErrorNamespace: errCode.Namespace(),
				Message:        errCode.Error(),
				Result:         nil,
			}
		}
	}

	if e := delivery.Payload().GetError(); e != nil {
		apiResp = toErrResponseFunc(e)
	} else {
		if data, e := delivery.Payload().GetData(); e != nil {
			apiResp = toErrResponseFunc(e)
		} else {
			apiResp = APIResponse{
				Code:   0,
				Result: data,
			}
		}
	}

	resp = apiResp

	return
}

func (p *JsonApiReceiver) writeResponse(data []byte, w gohttp.ResponseWriter, r *gohttp.Request) {
	p.writeResponseWithStatusCode(data, w, r, gohttp.StatusOK)
}

func (p *JsonApiReceiver) writeResponseWithStatusCode(data []byte, w gohttp.ResponseWriter, r *gohttp.Request, code int) {
	p.writeAccessHeaders(w, r)
	p.writeBasicHeaders(w, r)
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(code)
	w.Write(data)
}

func (p *JsonApiReceiver) writeAccessHeaders(w gohttp.ResponseWriter, r *gohttp.Request) {
	refer := r.Referer()
	if refer == "" {
		refer = r.Header.Get("Origin")
	}

	if origin, isAllowed := p.conf.AccessControl.ParseOrigin(refer); isAllowed {
		w.Header().Set("Access-Control-Allow-Origin", origin)
	}

	if p.conf.AccessControl.Credentials {
		w.Header().Set("Access-Control-Allow-Credentials", "true")
	}

	if len(p.conf.AccessControl.methods) > 0 {
		w.Header().Set("Access-Control-Allow-Methods", p.conf.AccessControl.methods)
	}

	if len(p.conf.AccessControl.headers) > 0 {
		w.Header().Set("Access-Control-Allow-Headers", p.conf.AccessControl.headers)
	}
}

func (p *JsonApiReceiver) writeBasicHeaders(w gohttp.ResponseWriter, r *gohttp.Request) {
	if p.conf.ResponseHeaders != nil {
		for key, value := range p.conf.ResponseHeaders {
			w.Header().Set(key, value)
		}
	}
}
<commit_msg>if xdomain config is empty, it will disable to access<commit_after>package http_json_api

import (
	"encoding/json"
	"github.com/gogap/errors"
	"github.com/rs/xid"
	"io/ioutil"
	gohttp "net/http"
	"strconv"
	"strings"
	"time"

	"github.com/go-martini/martini"
	"github.com/gogap/spirit"
	"github.com/spirit-contrib/http"
)

var (
	receiverURN = "urn:spirit-contrib:receiver:http:json_api"
)

type JsonApiReceiver struct {
	*http.HTTPReceiver

	conf JsonApiReceiverConfig

	responseRenderer *APIResponseRenderer

	htmlProxy string
}

var (
	_ spirit.Receiver = new(JsonApiReceiver)
)

func init() {
	spirit.RegisterReceiver(receiverURN, NewJsonApiReceiver)
}

func NewJsonApiReceiver(config spirit.Config) (receiver spirit.Receiver, err error) {
	conf := JsonApiReceiverConfig{}

	if err = config.ToObject(&conf); err != nil {
		return
	}

	conf.initial()

	jsonApiReceiver := &JsonApiReceiver{
		conf: conf,
	}

	if jsonApiReceiver.HTTPReceiver, err = http.NewHTTPReceiver(conf.Http, jsonApiReceiver.requestHandler); err != nil {
		return
	}

	jsonApiReceiver.responseRenderer = NewAPIResponseRenderer()

	path := strings.TrimRight(conf.Path, "/")
	jsonApiReceiver.Group(path, func(r martini.Router) {
		r.Post("", jsonApiReceiver.HTTPReceiver.Handler)
		r.Post("/:apiName", jsonApiReceiver.HTTPReceiver.Handler)
		r.Options("", jsonApiReceiver.optionHandle)
		r.Options("/:apiName", jsonApiReceiver.optionHandle)
	})

	jsonApiReceiver.htmlProxy = strings.Replace(proxyHtml, "{{#XDomainLib#}}", conf.XDomain.LibUrl, 1)

	jsonApiReceiver.Group("/", func(r martini.Router) {
		r.Get("ping", func() string {
			return "pong"
		})

		if conf.XDomain.HtmlPath != "" {
			r.Get(conf.XDomain.HtmlPath, func(r *gohttp.Request) string {
				refer := r.Referer()
				if refer == "" {
					refer = r.Header.Get("Origin")
				}

				html := jsonApiReceiver.htmlProxy

				if conf.XDomain.Masters != nil && refer != "" {
					protocol, domain := parseRefer(refer)
					origin := protocol + "://" + domain

					if path, exist := conf.XDomain.Masters[protocol+"://"+domain]; exist {
						master := map[string]string{origin: path}

						jsonData, _ := json.MarshalIndent(master, "", "  ")

						html = strings.Replace(html, "{{#Masters#}}", string(jsonData), 1)
						return html
					}
				}

				html = strings.Replace(html, "{{#Masters#}}", "{}", 1)

				return html
			})
		}

		if conf.XDomain.LibPath != "" {
			r.Get(conf.XDomain.LibPath, func() string {
				return xdomainLib
			})
		}
	})

	receiver = jsonApiReceiver
	return
}

func (p *JsonApiReceiver) optionHandle(w gohttp.ResponseWriter, r *gohttp.Request) {
	if r.Method == "OPTIONS" {
		p.writeAccessHeaders(w, r)
		p.writeBasicHeaders(w, r)
		w.Write([]byte(""))
	}
}

func (p *JsonApiReceiver) requestHandler(
	res gohttp.ResponseWriter,
	req *gohttp.Request,
	deliveryChan <-chan spirit.Delivery,
	done chan<- bool,
) (deliveries []spirit.Delivery, err error) {

	var apiIds map[string]string

	// request to deliveries
	if deliveries, apiIds, err = p.toDeliveries(req); err != nil {

		var apiResponse APIResponse

		switch errCode := err.(type) {
		case errors.ErrCode:
			{
				apiResponse = APIResponse{
					Code:           errCode.Code(),
					ErrorId:        errCode.Id(),
					ErrorNamespace: errCode.Namespace(),
					Message:        errCode.Error(),
					Result:         nil,
				}
			}
		default:
			e := ErrApiGenericError.New().Append(err)
			apiResponse = APIResponse{
				Code:           e.Code(),
				ErrorId:        e.Id(),
				ErrorNamespace: e.Namespace(),
				Message:        e.Error(),
				Result:         nil,
			}
		}

		if data, e := json.Marshal(apiResponse); e != nil {
			spirit.Logger().WithField("event", "to deliveries").Println(err)
		} else {
			p.writeResponse(data, res, req)
		}

		return
	}

	go func(
		count int,
		apiIds map[string]string,
		res gohttp.ResponseWriter,
		req *gohttp.Request,
		deliveryChan <-chan spirit.Delivery,
		done chan<- bool) {

		defer func() {
			// notify the main handler finished
			select {
			case done <- true:
				{
				}
			case <-time.After(time.Second * 3):
				{
				}
			}
		}()

		// get timeout duration
		timeout := time.Duration(p.conf.Timeout) * time.Millisecond

		if strTimeout := req.Header.Get(p.conf.HeaderDefines.TimeoutHeader); strTimeout != "" {
			if i, e := strconv.Atoi(strTimeout); e == nil {
				timeout = time.Duration(i) * time.Millisecond
			}
		}

		if timeout <= 0 {
			timeout = DefaultTimeout
		}

		apiResponse := map[string]APIResponse{}

		i := count
		// get deliveries
	label_timeout_or_finished:
		for i > 0 {
			select {
			case delivery := <-deliveryChan:
				{
					if api, exist := apiIds[delivery.Id()]; !exist {
						spirit.Logger().
							WithField("api", api).
							WithField("delivery_id", delivery.Id()).
							Errorln("api not exist in request while delivery response")

					} else {
						apiResponse[api] = p.deliveryToApiResponse(delivery)
					}

					i = i - 1
				}
			case <-time.After(timeout):
				{
					break label_timeout_or_finished
				}
			}
		}

		// request timeout
		for _, api := range apiIds {
			if _, exist := apiResponse[api]; !exist {
				errCode := ErrRequestTimeout.New()
				apiResponse[api] = APIResponse{
					Code:           errCode.Code(),
					ErrorId:        errCode.Id(),
					ErrorNamespace: errCode.Namespace(),
					Message:        errCode.Error(),
					Result:         nil,
				}
			}
		}

		// render deliveries to json response
		// normal response: {"code": 0, "message": "", "result": null}
		// error response: {"code": 212, "error_namespace": "xxxx", "message": "something wrong", "result": null}

		isMultiCall := req.Header.Get(p.conf.HeaderDefines.MultiCallHeader) == "1" ||
			req.Header.Get(p.conf.HeaderDefines.MultiCallHeader) == "on" ||
			req.Header.Get(p.conf.HeaderDefines.MultiCallHeader) == "true"

		if renderedData, e := p.responseRenderer.Render(isMultiCall, apiResponse); e != nil {
			err := ErrRenderApiDataFailed.New(errors.Params{"err": e})
			resp := APIResponse{
				Code:           err.Code(),
				ErrorId:        err.Id(),
				ErrorNamespace: err.Namespace(),
				Message:        err.Error(),
				Result:         nil,
			}

			if errRespData, e := json.Marshal(resp); e != nil {
				strInternalErr := `{"code": 500, "message": "api server internal error", "result": null}`
				p.writeResponseWithStatusCode([]byte(strInternalErr), res, req, gohttp.StatusInternalServerError)
			} else {
				p.writeResponse(errRespData, res, req)
			}
		} else {
			p.writeResponse(renderedData, res, req)
		}
		return
	}(len(deliveries), apiIds, res, req, deliveryChan, done)

	return
}

func (p *JsonApiReceiver) toDeliveries(req *gohttp.Request) (deliveries []spirit.Delivery, apiIds map[string]string, err error) {
	isMultiCall := req.Header.Get(p.conf.HeaderDefines.MultiCallHeader) == "1" ||
		req.Header.Get(p.conf.HeaderDefines.MultiCallHeader) == "on" ||
		req.Header.Get(p.conf.HeaderDefines.MultiCallHeader) == "true"

	isForwarded := req.Header.Get(HeaderForwardedPayload) == "1" ||
		req.Header.Get(HeaderForwardedPayload) == "on" ||
		req.Header.Get(HeaderForwardedPayload) == "true"

	if isMultiCall == true && isForwarded == true {
		err = ErrNotSupportMultiCallForward.New()
		return
	}

	idMapping := make(map[string]string)

	var body []byte
	if body, err = ioutil.ReadAll(req.Body); err != nil {
		return
	}

	var apiDatas map[string]interface{} = make(map[string]interface{})

	if isMultiCall {
		if err = json.Unmarshal(body, &apiDatas); err != nil {
			return
		}
	} else {

		apiName := req.Header.Get(p.conf.HeaderDefines.ApiHeader)

		if apiName == "" {
			if p.conf.Path != req.RequestURI {
				apiName = strings.TrimPrefix(req.RequestURI, p.conf.Path)
				apiName = strings.TrimRight(apiName, "/")
			}
		}

		if apiName == "" {
			err = ErrApiNameIsEmpty
			return
		}

		if isForwarded {
			apiData := JsonPayload{}
			if json.Unmarshal(body, &apiData); err != nil {
				return
			}
			apiDatas[apiName] = apiData
		} else {
			var apiData map[string]interface{}
			if json.Unmarshal(body, &apiData); err != nil {
				return
			}
			apiDatas[apiName] = apiData
		}

	}

	var tmpDeliveries []spirit.Delivery
	for api, apiData := range apiDatas {

		payload := NewHttpJsonApiPayload()

		if isForwarded {
			if jsonPayload, ok := apiData.(JsonPayload); ok {
				payload.id = jsonPayload.Id
				payload.data = jsonPayload.Data
				payload.metadata = jsonPayload.Metadata
				payload.contexts = jsonPayload.Context

				if jsonPayload.Error != nil {
					errCode := errors.NewErrorCode(
						jsonPayload.Error.Id,
						jsonPayload.Error.Code,
						jsonPayload.Error.Namespace,
						jsonPayload.Error.Message,
						jsonPayload.Error.StackTrace,
						jsonPayload.Error.Context)

					payload.SetError(errCode)
				}
			}
		} else {
			payload.SetData(apiData)

			headerContexts := map[string]interface{}{}
			cookiesContexts := map[string]interface{}{}

			for _, key := range p.conf.ToContext.Headers {
				if req.Header.Get(key) != "" {
					headerContexts[key] = req.Header.Get(key)
				}
			}

			for _, key := range p.conf.ToContext.Cookies {
				if cookie, e := req.Cookie(key); e == nil {
					cookiesContexts[cookie.Name] = cookie
				}
			}

			if len(cookiesContexts) > 0 {
				payload.SetContext(CtxHttpCookies, cookiesContexts)
			}

			if len(headerContexts) > 0 {
				payload.SetContext(CtxHttpHeaders, headerContexts)
			}

			if len(p.conf.ToContext.Customs) > 0 {
				payload.SetContext(CtxHttpCustom, p.conf.ToContext.Customs)
			}
		}

		deliveryURN := ""
		if urn, exist := p.conf.ApiURN[api]; exist {
			deliveryURN = urn
		} else {
			deliveryURN = p.conf.BindURN
		}

		deliveryLabels := spirit.Labels{}
		if labels, exist := p.conf.ApiLabels[api]; exist {
			deliveryLabels = labels
		}

		de := &HttpJsonApiDelivery{
			id:        xid.New().String(),
			payload:   payload,
			urn:       deliveryURN,
			labels:    deliveryLabels,
			timestamp: time.Now(),
		}

		tmpDeliveries = append(tmpDeliveries, de)

		idMapping[de.id] = api
	}

	deliveries = tmpDeliveries
	apiIds = idMapping

	return
}

func (p *JsonApiReceiver) deliveryToApiResponse(delivery spirit.Delivery) (resp APIResponse) {

	var apiResp APIResponse

	toErrResponseFunc := func(err error) APIResponse {
		if errCode, ok := delivery.Payload().GetError().(errors.ErrCode); ok {
			return APIResponse{
				Code:           errCode.Code(),
				ErrorId:        errCode.Id(),
				ErrorNamespace: errCode.Namespace(),
				Message:        errCode.Error(),
				Result:         nil,
			}
		} else {
			errCode := ErrApiGenericError.New().Append(err)
			return APIResponse{
				Code:           errCode.Code(),
				ErrorId:        errCode.Id(),
				ErrorNamespace: errCode.Namespace(),
				Message:        errCode.Error(),
				Result:         nil,
			}
		}
	}

	if e := delivery.Payload().GetError(); e != nil {
		apiResp = toErrResponseFunc(e)
	} else {
		if data, e := delivery.Payload().GetData(); e != nil {
			apiResp = toErrResponseFunc(e)
		} else {
			apiResp = APIResponse{
				Code:   0,
				Result: data,
			}
		}
	}

	resp = apiResp

	return
}

func (p *JsonApiReceiver) writeResponse(data []byte, w gohttp.ResponseWriter, r *gohttp.Request) {
	p.writeResponseWithStatusCode(data, w, r, gohttp.StatusOK)
}

func (p *JsonApiReceiver) writeResponseWithStatusCode(data []byte, w gohttp.ResponseWriter, r *gohttp.Request, code int) {
	p.writeAccessHeaders(w, r)
	p.writeBasicHeaders(w, r)
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(code)
	w.Write(data)
}

func (p *JsonApiReceiver) writeAccessHeaders(w gohttp.ResponseWriter, r *gohttp.Request) {
	refer := r.Referer()
	if refer == "" {
		refer = r.Header.Get("Origin")
	}

	if origin, isAllowed := p.conf.AccessControl.ParseOrigin(refer); isAllowed {
		w.Header().Set("Access-Control-Allow-Origin", origin)
	}

	if p.conf.AccessControl.Credentials {
		w.Header().Set("Access-Control-Allow-Credentials", "true")
	}

	if len(p.conf.AccessControl.methods) > 0 {
		w.Header().Set("Access-Control-Allow-Methods", p.conf.AccessControl.methods)
	}

	if len(p.conf.AccessControl.headers) > 0 {
		w.Header().Set("Access-Control-Allow-Headers", p.conf.AccessControl.headers)
	}
}

func (p *JsonApiReceiver) writeBasicHeaders(w gohttp.ResponseWriter, r *gohttp.Request) {
	if p.conf.ResponseHeaders != nil {
		for key, value := range p.conf.ResponseHeaders {
			w.Header().Set(key, value)
		}
	}
}
<|endoftext|>


<commit_before>package negroni

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"runtime"
	"runtime/debug"
	"text/template"
)

const (
	panicText = "PANIC: %s\n%s"
	panicHTML = `
<title>PANIC: {{.RecoveredPanic}}</title>

Negroni - PANIC

{{.RequestDescription}}

	Runtime error: {{.RecoveredPanic}}

{{ if .Stack }}

Runtime Stack

{{.StackAsString}}

{{ end }}

`
	nilRequestMessage = "Request is nil"
)

var panicHTMLTemplate = template.Must(template.New("PanicPage").Parse(panicHTML))

// PanicInformation contains all
// elements for printing stack informations.
type PanicInformation struct {
	RecoveredPanic interface{}
	Stack          []byte
	Request        *http.Request
}

// StackAsString returns a printable version of the stack
func (p *PanicInformation) StackAsString() string {
	return string(p.Stack)
}

// RequestDescription returns a printable description of the url
func (p *PanicInformation) RequestDescription() string {

	if p.Request == nil {
		return nilRequestMessage
	}

	var queryOutput string
	if p.Request.URL.RawQuery != "" {
		queryOutput = "?" + p.Request.URL.RawQuery
	}
	return fmt.Sprintf("%s %s%s", p.Request.Method, p.Request.URL.Path, queryOutput)
}

// PanicFormatter is an interface on object can implement
// to be able to output the stack trace
type PanicFormatter interface {
	// FormatPanicError output the stack for a given answer/response.
	// In case the the middleware should not output the stack trace,
	// the field `Stack` of the passed `PanicInformation` instance equals `[]byte{}`.
	FormatPanicError(rw http.ResponseWriter, r *http.Request, infos *PanicInformation)
}

// TextPanicFormatter output the stack
// as simple text on os.Stdout. If no `Content-Type` is set,
// it will output the data as `text/plain; charset=utf-8`.
// Otherwise, the origin `Content-Type` is kept.
type TextPanicFormatter struct{}

func (t *TextPanicFormatter) FormatPanicError(rw http.ResponseWriter, r *http.Request, infos *PanicInformation) {
	if rw.Header().Get("Content-Type") == "" {
		rw.Header().Set("Content-Type", "text/plain; charset=utf-8")
	}
	fmt.Fprintf(rw, panicText, infos.RecoveredPanic, infos.Stack)
}

// HTMLPanicFormatter output the stack inside
// an HTML page. This has been largely inspired by
// https://github.com/go-martini/martini/pull/156/commits.
type HTMLPanicFormatter struct{}

func (t *HTMLPanicFormatter) FormatPanicError(rw http.ResponseWriter, r *http.Request, infos *PanicInformation) {
	if rw.Header().Get("Content-Type") == "" {
		rw.Header().Set("Content-Type", "text/html; charset=utf-8")
	}
	panicHTMLTemplate.Execute(rw, infos)
}

// Recovery is a Negroni middleware that recovers from any panics and writes a 500 if there was one.
type Recovery struct {
	Logger           ALogger
	PrintStack       bool
	PanicHandlerFunc func(*PanicInformation)
	StackAll         bool
	StackSize        int
	Formatter        PanicFormatter

	// Deprecated: Use PanicHandlerFunc instead to receive panic
	// error with additional information (see PanicInformation)
	ErrorHandlerFunc func(interface{})
}

// NewRecovery returns a new instance of Recovery
func NewRecovery() *Recovery {
	return &Recovery{
		Logger:     log.New(os.Stdout, "[negroni] ", 0),
		PrintStack: true,
		StackAll:   false,
		StackSize:  1024 * 8,
		Formatter:  &TextPanicFormatter{},
	}
}

func (rec *Recovery) ServeHTTP(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	defer func() {
		if err := recover(); err != nil {
			rw.WriteHeader(http.StatusInternalServerError)

			stack := make([]byte, rec.StackSize)
			stack = stack[:runtime.Stack(stack, rec.StackAll)]
			infos := &PanicInformation{RecoveredPanic: err, Request: r}

			if rec.PrintStack {
				infos.Stack = stack
			}
			rec.Logger.Printf(panicText, err, stack)
			rec.Formatter.FormatPanicError(rw, r, infos)

			if rec.ErrorHandlerFunc != nil {
				func() {
					defer func() {
						if err := recover(); err != nil {
							rec.Logger.Printf("provided ErrorHandlerFunc panic'd: %s, trace:\n%s", err, debug.Stack())
							rec.Logger.Printf("%s\n", debug.Stack())
						}
					}()
					rec.ErrorHandlerFunc(err)
				}()
			}
			if rec.PanicHandlerFunc != nil {
				func() {
					defer func() {
						if err := recover(); err != nil {
							rec.Logger.Printf("provided PanicHandlerFunc panic'd: %s, trace:\n%s", err, debug.Stack())
							rec.Logger.Printf("%s\n", debug.Stack())
						}
					}()
					rec.PanicHandlerFunc(infos)
				}()
			}
		}
	}()

	next(rw, r)
}
<commit_msg>make logging the stack trace optional<commit_after>package negroni

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"runtime"
	"runtime/debug"
	"text/template"
)

const (
	panicText = "PANIC: %s\n%s"
	panicHTML = `
<title>PANIC: {{.RecoveredPanic}}</title>

Negroni - PANIC

{{.RequestDescription}}

	Runtime error: {{.RecoveredPanic}}

{{ if .Stack }}

Runtime Stack

{{.StackAsString}}

{{ end }}

`
	nilRequestMessage = "Request is nil"
)

var panicHTMLTemplate = template.Must(template.New("PanicPage").Parse(panicHTML))

// PanicInformation contains all
// elements for printing stack informations.
type PanicInformation struct {
	RecoveredPanic interface{}
	Stack          []byte
	Request        *http.Request
}

// StackAsString returns a printable version of the stack
func (p *PanicInformation) StackAsString() string {
	return string(p.Stack)
}

// RequestDescription returns a printable description of the url
func (p *PanicInformation) RequestDescription() string {

	if p.Request == nil {
		return nilRequestMessage
	}

	var queryOutput string
	if p.Request.URL.RawQuery != "" {
		queryOutput = "?" + p.Request.URL.RawQuery
	}
	return fmt.Sprintf("%s %s%s", p.Request.Method, p.Request.URL.Path, queryOutput)
}

// PanicFormatter is an interface on object can implement
// to be able to output the stack trace
type PanicFormatter interface {
	// FormatPanicError output the stack for a given answer/response.
	// In case the the middleware should not output the stack trace,
	// the field `Stack` of the passed `PanicInformation` instance equals `[]byte{}`.
	FormatPanicError(rw http.ResponseWriter, r *http.Request, infos *PanicInformation)
}

// TextPanicFormatter output the stack
// as simple text on os.Stdout. If no `Content-Type` is set,
// it will output the data as `text/plain; charset=utf-8`.
// Otherwise, the origin `Content-Type` is kept.
type TextPanicFormatter struct{}

func (t *TextPanicFormatter) FormatPanicError(rw http.ResponseWriter, r *http.Request, infos *PanicInformation) {
	if rw.Header().Get("Content-Type") == "" {
		rw.Header().Set("Content-Type", "text/plain; charset=utf-8")
	}
	fmt.Fprintf(rw, panicText, infos.RecoveredPanic, infos.Stack)
}

// HTMLPanicFormatter output the stack inside
// an HTML page. This has been largely inspired by
// https://github.com/go-martini/martini/pull/156/commits.
type HTMLPanicFormatter struct{}

func (t *HTMLPanicFormatter) FormatPanicError(rw http.ResponseWriter, r *http.Request, infos *PanicInformation) {
	if rw.Header().Get("Content-Type") == "" {
		rw.Header().Set("Content-Type", "text/html; charset=utf-8")
	}
	panicHTMLTemplate.Execute(rw, infos)
}

// Recovery is a Negroni middleware that recovers from any panics and writes a 500 if there was one.
type Recovery struct {
	Logger           ALogger
	PrintStack       bool
	LogStack         bool
	PanicHandlerFunc func(*PanicInformation)
	StackAll         bool
	StackSize        int
	Formatter        PanicFormatter

	// Deprecated: Use PanicHandlerFunc instead to receive panic
	// error with additional information (see PanicInformation)
	ErrorHandlerFunc func(interface{})
}

// NewRecovery returns a new instance of Recovery
func NewRecovery() *Recovery {
	return &Recovery{
		Logger:     log.New(os.Stdout, "[negroni] ", 0),
		PrintStack: true,
		LogStack:   true,
		StackAll:   false,
		StackSize:  1024 * 8,
		Formatter:  &TextPanicFormatter{},
	}
}

func (rec *Recovery) ServeHTTP(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	defer func() {
		if err := recover(); err != nil {
			rw.WriteHeader(http.StatusInternalServerError)

			stack := make([]byte, rec.StackSize)
			stack = stack[:runtime.Stack(stack, rec.StackAll)]
			infos := &PanicInformation{RecoveredPanic: err, Request: r}

			if rec.PrintStack {
				infos.Stack = stack
			}

			if rec.LogStack {
				rec.Logger.Printf(panicText, err, stack)
			}

			rec.Formatter.FormatPanicError(rw, r, infos)

			if rec.ErrorHandlerFunc != nil {
				func() {
					defer func() {
						if err := recover(); err != nil {
							rec.Logger.Printf("provided ErrorHandlerFunc panic'd: %s, trace:\n%s", err, debug.Stack())
							rec.Logger.Printf("%s\n", debug.Stack())
						}
					}()
					rec.ErrorHandlerFunc(err)
				}()
			}
			if rec.PanicHandlerFunc != nil {
				func() {
					defer func() {
						if err := recover(); err != nil {
							rec.Logger.Printf("provided PanicHandlerFunc panic'd: %s, trace:\n%s", err, debug.Stack())
							rec.Logger.Printf("%s\n", debug.Stack())
						}
					}()
					rec.PanicHandlerFunc(infos)
				}()
			}
		}
	}()

	next(rw, r)
}
<|endoftext|>


<commit_before>package overcurrent

import (
	"errors"
	"sync"
	"time"

	"github.com/efritz/glock"
)

type (
	Registry interface {
		// Configure will register a new breaker instance under the given name using
		// the given configuration. A breaker config may not be changed after being
		// initialized. It is an error to register the same breaker twice, or try to
		// invoke Call or CallAsync with an unregistered breaker.
		Configure(name string, configs ...BreakerConfig) error

		// Call will invoke `Call` on the breaker configured with the given name. If
		// the breaker returns a non-nil error, the fallback function is invoked with
		// the error as the value. It may be the case that the fallback function is
		// invoked without the breaker function failing (e.g. circuit open).
		Call(name string, f BreakerFunc, fallback FallbackFunc) error

		// CallAsync will create a channel that receives the error value from an similar
		// invocation of Call. See the Breaker docs for more details.
		CallAsync(name string, f BreakerFunc, fallback FallbackFunc) <-chan error
	}

	registry struct {
		breakers map[string]*wrappedBreaker
		mutex    *sync.RWMutex
		clock    glock.Clock
	}

	wrappedBreaker struct {
		breaker   *circuitBreaker
		semaphore *semaphore
	}

	FallbackFunc func(error) error
)

var (
	ErrAlreadyConfigured   = errors.New("breaker is already configured")
	ErrBreakerUnconfigured = errors.New("breaker not configured")
	ErrMaxConcurrency      = errors.New("breaker is at max concurrency")
)

func NewRegistry() Registry {
	return newRegistryWithClock(glock.NewRealClock())
}

func newRegistryWithClock(clock glock.Clock) Registry {
	return &registry{
		breakers: map[string]*wrappedBreaker{},
		mutex:    &sync.RWMutex{},
		clock:    clock,
	}
}

func (r *registry) Configure(name string, configs ...BreakerConfig) error {
	r.mutex.Lock()
	defer r.mutex.Unlock()

	if _, ok := r.breakers[name]; ok {
		return ErrAlreadyConfigured
	}

	breaker := newCircuitBreaker(configs...)

	r.breakers[name] = &wrappedBreaker{
		breaker:   breaker,
		semaphore: newSemaphore(r.clock, breaker.maxConcurrency),
	}

	return nil
}

func (r *registry) Call(name string, f BreakerFunc, fallback FallbackFunc) error {
	wrapped, collector, err := r.getWrappedBreaker(name)
	if err != nil {
		return err
	}

	start := time.Now()
	err = r.call(wrapped, collector, f, fallback)
	elapsed := time.Now().Sub(start)

	collector.ReportDuration(EventTypeTotalDuration, elapsed)
	return err
}

func (r *registry) CallAsync(name string, f BreakerFunc, fallback FallbackFunc) <-chan error {
	return toErrChan(func() error { return r.Call(name, f, fallback) })
}

func (r *registry) getWrappedBreaker(name string) (*wrappedBreaker, MetricCollector, error) {
	r.mutex.RLock()
	defer r.mutex.RUnlock()

	wrapped, ok := r.breakers[name]
	if !ok {
		return nil, nil, ErrBreakerUnconfigured
	}

	return wrapped, wrapped.breaker.collector, nil
}

func (r *registry) call(wrapped *wrappedBreaker, collector MetricCollector, f BreakerFunc, fallback FallbackFunc) error {
	err := r.callWithSemaphore(wrapped.breaker, wrapped.semaphore, f)
	if err == nil {
		collector.Report(EventTypeSuccess)
		return nil
	}

	collector.Report(EventTypeFailure)

	if err == ErrMaxConcurrency {
		collector.Report(EventTypeRejection)
	}

	if fallback == nil {
		return err
	}

	if err := fallback(err); err != nil {
		collector.Report(EventTypeFallbackFailure)
		return err
	}

	collector.Report(EventTypeFallbackSuccess)
	return nil
}

func (r *registry) callWithSemaphore(breaker *circuitBreaker, semaphore *semaphore, f BreakerFunc) error {
	if !semaphore.wait(breaker.maxConcurrencyTimeout) {
		return ErrMaxConcurrency
	}

	defer semaphore.signal()
	return breaker.Call(f)
}
<commit_msg>Fill out additional stats in registry.<commit_after>package overcurrent

import (
	"errors"
	"sync"
	"time"

	"github.com/efritz/glock"
)

type (
	Registry interface {
		// Configure will register a new breaker instance under the given name using
		// the given configuration. A breaker config may not be changed after being
		// initialized. It is an error to register the same breaker twice, or try to
		// invoke Call or CallAsync with an unregistered breaker.
		Configure(name string, configs ...BreakerConfig) error

		// Call will invoke `Call` on the breaker configured with the given name. If
		// the breaker returns a non-nil error, the fallback function is invoked with
		// the error as the value. It may be the case that the fallback function is
		// invoked without the breaker function failing (e.g. circuit open).
		Call(name string, f BreakerFunc, fallback FallbackFunc) error

		// CallAsync will create a channel that receives the error value from an similar
		// invocation of Call. See the Breaker docs for more details.
		CallAsync(name string, f BreakerFunc, fallback FallbackFunc) <-chan error
	}

	registry struct {
		breakers map[string]*wrappedBreaker
		mutex    *sync.RWMutex
		clock    glock.Clock
	}

	wrappedBreaker struct {
		breaker   *circuitBreaker
		semaphore *semaphore
	}

	FallbackFunc func(error) error
)

var (
	ErrAlreadyConfigured   = errors.New("breaker is already configured")
	ErrBreakerUnconfigured = errors.New("breaker not configured")
	ErrMaxConcurrency      = errors.New("breaker is at max concurrency")
)

func NewRegistry() Registry {
	return newRegistryWithClock(glock.NewRealClock())
}

func newRegistryWithClock(clock glock.Clock) Registry {
	return &registry{
		breakers: map[string]*wrappedBreaker{},
		mutex:    &sync.RWMutex{},
		clock:    clock,
	}
}

func (r *registry) Configure(name string, configs ...BreakerConfig) error {
	r.mutex.Lock()
	defer r.mutex.Unlock()

	if _, ok := r.breakers[name]; ok {
		return ErrAlreadyConfigured
	}

	breaker := newCircuitBreaker(configs...)

	r.breakers[name] = &wrappedBreaker{
		breaker:   breaker,
		semaphore: newSemaphore(r.clock, breaker.maxConcurrency),
	}

	return nil
}

func (r *registry) Call(name string, f BreakerFunc, fallback FallbackFunc) error {
	wrapped, collector, err := r.getWrappedBreaker(name)
	if err != nil {
		return err
	}

	start := time.Now()
	err = r.call(wrapped, collector, f, fallback)
	elapsed := time.Now().Sub(start)

	collector.ReportDuration(EventTypeTotalDuration, elapsed)
	return err
}

func (r *registry) CallAsync(name string, f BreakerFunc, fallback FallbackFunc) <-chan error {
	return toErrChan(func() error {
		return r.Call(name, f, fallback)
	})
}

func (r *registry) getWrappedBreaker(name string) (*wrappedBreaker, MetricCollector, error) {
	r.mutex.RLock()
	defer r.mutex.RUnlock()

	wrapped, ok := r.breakers[name]
	if !ok {
		return nil, nil, ErrBreakerUnconfigured
	}

	return wrapped, wrapped.breaker.collector, nil
}

func (r *registry) call(wrapped *wrappedBreaker, collector MetricCollector, f BreakerFunc, fallback FallbackFunc) error {
	collector.Report(EventTypeAttempt)

	err := r.callWithSemaphore(wrapped.breaker, wrapped.semaphore, f)
	if err == nil {
		collector.Report(EventTypeSuccess)
		return nil
	}

	collector.Report(EventTypeFailure)

	if err == ErrMaxConcurrency {
		collector.Report(EventTypeRejection)
	}

	if fallback == nil {
		return err
	}

	if err := fallback(err); err != nil {
		collector.Report(EventTypeFallbackFailure)
		return err
	}

	collector.Report(EventTypeFallbackSuccess)
	return nil
}

func (r *registry) callWithSemaphore(breaker *circuitBreaker, semaphore *semaphore, f BreakerFunc) error {
	breaker.collector.Report(EventTypeSemaphoreQueued)
	val := semaphore.wait(breaker.maxConcurrencyTimeout)
	breaker.collector.Report(EventTypeSemaphoreDequeued)

	if !val {
		return ErrMaxConcurrency
	}

	defer func() {
		breaker.collector.Report(EventTypeSemaphoreReleased)
		semaphore.signal()
	}()

	breaker.collector.Report(EventTypeSemaphoreAcquired)
	return breaker.Call(f)
}
<|endoftext|>


<commit_before>// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pubsub

import (
	"context"

	"cloud.google.com/go/pubsub"
	vkit "cloud.google.com/go/pubsub/apiv1"
	gax "github.com/googleapis/gax-go"
	"go.chromium.org/luci/common/logging"
	pb "google.golang.org/genproto/googleapis/pubsub/v1"
)

// Publisher is a generic interface to something that can publish Pub/Sub
// messages.
//
// A Publisher should be Closed when finished with it.
type Publisher interface {
	Publish(c context.Context, msgs ...*pubsub.Message) ([]string, error)
	Close() error
}

// ClientFactory is passed into an UnbufferedPublisher to create or reset a client.
type ClientFactory interface {
	// Client returns the Pub/Sub publisher client to use.
	// Client will be closed when this UnbufferedPublisher is closed.
	Client(context.Context) (*vkit.PublisherClient, error)

	// RecreateClient is called if any publish calls fail.
	// This is used to tell the underlying service to maybe generate a new client.
	RecreateClient()
}

// UnbufferedPublisher directly instantiates a Pub/Sub client and publishes a
// message to it.
//
// The standard Pub/Sub library has several issues, especially when used from
// AppEngine:
//	- It uses an empty Context, discarding AppEngine context.
//	- It uses a buffer, which expects a lifecycle beyond that of a simple
//	  AppEngine Request.
type UnbufferedPublisher struct {
	// AECtx is the AppEngine context used to create a pubsub client.
	AECtx context.Context

	// Topic is the name of the Topic to publish to.
	Topic Topic

	// ClientFactory produces a client for the publisher.  This is called on each
	// and every publish request.  If a publish request fails, then RecreateClient is called.
	ClientFactory ClientFactory

	// CallOpts are arbitrary call options that will be passed to the Publish
	// call.
	CallOpts []gax.CallOption
}

var _ Publisher = (*UnbufferedPublisher)(nil)

// Publish publishes a message immediately, blocking until it completes.
//
// "c" must be an AppEngine context.
func (up *UnbufferedPublisher) Publish(c context.Context, msgs ...*pubsub.Message) ([]string, error) {
	if len(msgs) == 0 {
		return nil, nil
	}

	messages := make([]*pb.PubsubMessage, len(msgs))
	for i, msg := range msgs {
		messages[i] = &pb.PubsubMessage{
			Data:       msg.Data,
			Attributes: msg.Attributes,
		}
	}

	client, err := up.ClientFactory.Client(up.AECtx)
	if err != nil {
		return nil, err
	}

	resp, err := client.Publish(c, &pb.PublishRequest{
		Topic:    string(up.Topic),
		Messages: messages,
	}, up.CallOpts...)
	if err != nil {
		// Optimistically recreate the client.
		up.ClientFactory.RecreateClient()
		logging.Debugf(c, "Recreating a new PubSub client due to error")
		return nil, err
	}
	return resp.MessageIds, nil
}

// Close closes the UnbufferedPublisher, notably its Client.
func (up *UnbufferedPublisher) Close() error {
	client, err := up.ClientFactory.Client(up.AECtx)
	if err != nil {
		return err
	}
	return client.Close()
}
<commit_msg>update to use gax-go v2<commit_after>// Copyright 2017 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pubsub

import (
	"context"

	"cloud.google.com/go/pubsub"
	vkit "cloud.google.com/go/pubsub/apiv1"
	gax "github.com/googleapis/gax-go/v2"
	"go.chromium.org/luci/common/logging"
	pb "google.golang.org/genproto/googleapis/pubsub/v1"
)

// Publisher is a generic interface to something that can publish Pub/Sub
// messages.
//
// A Publisher should be Closed when finished with it.
type Publisher interface {
	Publish(c context.Context, msgs ...*pubsub.Message) ([]string, error)
	Close() error
}

// ClientFactory is passed into an UnbufferedPublisher to create or reset a client.
type ClientFactory interface {
	// Client returns the Pub/Sub publisher client to use.
	// Client will be closed when this UnbufferedPublisher is closed.
	Client(context.Context) (*vkit.PublisherClient, error)

	// RecreateClient is called if any publish calls fail.
	// This is used to tell the underlying service to maybe generate a new client.
	RecreateClient()
}

// UnbufferedPublisher directly instantiates a Pub/Sub client and publishes a
// message to it.
//
// The standard Pub/Sub library has several issues, especially when used from
// AppEngine:
//	- It uses an empty Context, discarding AppEngine context.
//	- It uses a buffer, which expects a lifecycle beyond that of a simple
//	  AppEngine Request.
type UnbufferedPublisher struct {
	// AECtx is the AppEngine context used to create a pubsub client.
	AECtx context.Context

	// Topic is the name of the Topic to publish to.
	Topic Topic

	// ClientFactory produces a client for the publisher.  This is called on each
	// and every publish request.  If a publish request fails, then RecreateClient is called.
	ClientFactory ClientFactory

	// CallOpts are arbitrary call options that will be passed to the Publish
	// call.
	CallOpts []gax.CallOption
}

var _ Publisher = (*UnbufferedPublisher)(nil)

// Publish publishes a message immediately, blocking until it completes.
//
// "c" must be an AppEngine context.
func (up *UnbufferedPublisher) Publish(c context.Context, msgs ...*pubsub.Message) ([]string, error) {
	if len(msgs) == 0 {
		return nil, nil
	}

	messages := make([]*pb.PubsubMessage, len(msgs))
	for i, msg := range msgs {
		messages[i] = &pb.PubsubMessage{
			Data:       msg.Data,
			Attributes: msg.Attributes,
		}
	}

	client, err := up.ClientFactory.Client(up.AECtx)
	if err != nil {
		return nil, err
	}

	resp, err := client.Publish(c, &pb.PublishRequest{
		Topic:    string(up.Topic),
		Messages: messages,
	}, up.CallOpts...)
	if err != nil {
		// Optimistically recreate the client.
		up.ClientFactory.RecreateClient()
		logging.Debugf(c, "Recreating a new PubSub client due to error")
		return nil, err
	}
	return resp.MessageIds, nil
}

// Close closes the UnbufferedPublisher, notably its Client.
func (up *UnbufferedPublisher) Close() error {
	client, err := up.ClientFactory.Client(up.AECtx)
	if err != nil {
		return err
	}
	return client.Close()
}
<|endoftext|>


<commit_before>package bamstats

import (
	"bytes"
	"fmt"
	"os"
	"sort"
	"sync"

	log "github.com/Sirupsen/logrus"
	"github.com/biogo/hts/bam"
	"github.com/biogo/hts/sam"
)

type TagMap map[int]int

type MappedReadsStats struct {
	Total    int    `json:"total,omitempty"`
	Mapped   TagMap `json:"mapped,omitempty"`
	Unmapped int    `json:"unmapped,omitempty"`
}

type MappingsStats struct {
	MappedReadsStats
	Mappings MultimapStats `json:"mappings"`
}

type MultimapStats struct {
	Count int     `json:"count"`
	Ratio float64 `json:"ratio"`
}

type GeneralStats struct {
	Reads MappingsStats    `json:"reads,omitempty"`
	Pairs MappedReadsStats `json:"pairs,omitempty"`
}

func (s *GeneralStats) Merge(others chan GeneralStats) {
	for other := range others {
		s.Update(other)
	}
}

func (s *GeneralStats) Update(other GeneralStats) {
	s.Reads.Update(other.Reads)
	s.Pairs.Update(other.Pairs)
	if len(s.Pairs.Mapped) > 0 {
		s.Pairs.Total = s.Reads.Total / 2
		s.Pairs.Unmapped = s.Pairs.Total - s.Pairs.Mapped.Total()
	}
}

func (s *MappedReadsStats) Update(other MappedReadsStats) {
	s.Total += other.Total
	s.Unmapped += other.Unmapped
	s.Mapped.Update(other.Mapped)
}

func (s *MappingsStats) Update(other MappingsStats) {
	s.MappedReadsStats.Update(other.MappedReadsStats)
	s.Mappings.Count += other.Mappings.Count
	s.UpdateMappingsRatio()
}

func (s *MappingsStats) UpdateMappingsRatio() {
	s.Mappings.Ratio = float64(s.Mappings.Count) / float64(s.Mapped.Total())
}

func (s *MappedReadsStats) Unique() int {
	return s.Mapped[1]
}

func (s TagMap) Update(other TagMap) {
	for k := range s {
		s[k] += other[k]
	}
	for k := range other {
		if _, ok := s[k]; !ok {
			s[k] += other[k]
		}
	}
}

func (s TagMap) Total() (sum int) {
	for _, v := range s {
		sum += v
	}
	return
}

func NewGeneralStats() *GeneralStats {
	ms := GeneralStats{}
	ms.Pairs = *NewMappedReadsStats()
	ms.Reads.MappedReadsStats = *NewMappedReadsStats()
	return &ms
}

func NewMappedReadsStats() *MappedReadsStats {
	s := MappedReadsStats{}
	s.Mapped = make(TagMap)
	return &s
}

func (tm TagMap) MarshalJSON() ([]byte, error) {
	buf := &bytes.Buffer{}
	buf.Write([]byte{'{', '\n'})
	var keys []int
	for k := range tm {
		keys = append(keys, int(k))
	}
	sort.Ints(keys)
	l := len(keys)
	for i, k := range keys {
		fmt.Fprintf(buf, "\t\"%d\": \"%v\"", k, tm[k])
		if i < l-1 {
			buf.WriteByte(',')
		}
		buf.WriteByte('\n')
	}
	buf.Write([]byte{'}', '\n'})
	return buf.Bytes(), nil
}

func (s *GeneralStats) Collect(r *sam.Record) {
	NH, hasNH := r.Tag([]byte("NH"))
	if !hasNH {
		NH, _ = sam.ParseAux([]byte("NH:i:0"))
	}
	NHKey := int(NH.Value().(uint8))
	if isUnmapped(r) {
		s.Reads.Total++
		s.Reads.Unmapped++
		return
	} else {
		s.Reads.Mappings.Count++
		if isPrimary(r) {
			s.Reads.Total++
			s.Reads.Mapped[NHKey]++
			if isFirstOfValidPair(r) {
				s.Pairs.Mapped[NHKey]++
			}
		}
	}
}

var wgg sync.WaitGroup

func gworker(in chan *sam.Record, out chan GeneralStats) {
	defer wgg.Done()
	stats := NewGeneralStats()
	for record := range in {
		stats.Collect(record)
	}
	log.Debug("Worker DONE!")
	out <- *stats
}

func cProc(br *bam.Reader, cpu int, maxBuf int, reads int) *GeneralStats {
	input := make([]chan *sam.Record, cpu)
	stats := make(chan GeneralStats, cpu)
	for i := 0; i < cpu; i++ {
		wgg.Add(1)
		input[i] = make(chan *sam.Record, maxBuf)
		go gworker(input[i], stats)
	}
	c := 0
	for {
		if reads > -1 && c == reads {
			break
		}
		record, err := br.Read()
		if err != nil {
			break
		}
		input[c%cpu] <- record
		c++
	}
	for i := 0; i < cpu; i++ {
		close(input[i])
	}
	go func() {
		wgg.Wait()
		close(stats)
	}()
	st := <-stats
	st.Merge(stats)
	return &st
}

func lProc(br *bam.Reader) *GeneralStats {
	st := NewGeneralStats()
	for {
		record, err := br.Read()
		if err != nil {
			break
		}
		st.Collect(record)
	}
	return st
}

func General(bamFile string, cpu int, maxBuf int, reads int) *GeneralStats {
	f, err := os.Open(bamFile)
	defer f.Close()
	check(err)
	br, err := bam.NewReader(f, cpu)
	defer br.Close()
	check(err)
	switch {
	case cpu <= 0:
		log.Panic("Number of cpus must be a positive number")
	case cpu == 1:
		return lProc(br)
	case cpu > 1:
		return cProc(br, cpu, maxBuf, reads)
	}
	return nil
}
<commit_msg>Add insert sizes map for paired-end reads<commit_after>package bamstats

import (
	"bytes"
	"fmt"
	"math"
	"os"
	"sort"
	"sync"

	log "github.com/Sirupsen/logrus"
	"github.com/biogo/hts/bam"
	"github.com/biogo/hts/sam"
)

type TagMap map[int]int

type MappedReadsStats struct {
	Total    int    `json:"total,omitempty"`
	Unmapped int    `json:"unmapped,omitempty"`
	Mapped   TagMap `json:"mapped,omitempty"`
}

type MappingsStats struct {
	MappedReadsStats
	Continuous int           `json:"continuous"`
	Split      int           `json:"split"`
	Mappings   MultimapStats `json:"mappings"`
}

type MappedPairsStats struct {
	MappedReadsStats
	InsertSizes TagMap `json:"insert_sizes,omitempty"`
}

type MultimapStats struct {
	Ratio float64 `json:"ratio"`
	Count int     `json:"count"`
}

type GeneralStats struct {
	Reads MappingsStats    `json:"reads,omitempty"`
	Pairs MappedPairsStats `json:"pairs,omitempty"`
}

func (s *GeneralStats) Merge(others chan GeneralStats) {
	for other := range others {
		s.Update(other)
	}
}

func (s *GeneralStats) Update(other GeneralStats) {
	s.Reads.Update(other.Reads)
	s.Pairs.Update(other.Pairs)
	if len(s.Pairs.Mapped) > 0 {
		s.Pairs.Total = s.Reads.Total / 2
		s.Pairs.Unmapped = s.Pairs.Total - s.Pairs.Mapped.Total()
	}
}

func (s *MappedReadsStats) Update(other MappedReadsStats) {
	s.Total += other.Total
	s.Unmapped += other.Unmapped
	s.Mapped.Update(other.Mapped)
}

func (s *MappingsStats) Update(other MappingsStats) {
	s.MappedReadsStats.Update(other.MappedReadsStats)
	s.Continuous += other.Continuous
	s.Split += other.Split
	s.Mappings.Count += other.Mappings.Count
	s.UpdateMappingsRatio()
}

func (s *MappedPairsStats) Update(other MappedPairsStats) {
	s.MappedReadsStats.Update(other.MappedReadsStats)
	s.InsertSizes.Update(other.InsertSizes)
}

func (s *MappedPairsStats) FilterInsertSizes(percent float64) {
	for k, v := range s.InsertSizes {
		if float64(v) < float64(s.Total)*(percent/100) {
			delete(s.InsertSizes, k)
		}
	}
}

func (s *MappingsStats) UpdateMappingsRatio() {
	s.Mappings.Ratio = float64(s.Mappings.Count) / float64(s.Mapped.Total())
}

func (s *MappedReadsStats) Unique() int {
	return s.Mapped[1]
}

func (s TagMap) Update(other TagMap) {
	for k := range s {
		s[k] += other[k]
	}
	for k := range other {
		if _, ok := s[k]; !ok {
			s[k] += other[k]
		}
	}
}

func (s TagMap) Total() (sum int) {
	for _, v := range s {
		sum += v
	}
	return
}

func NewGeneralStats() *GeneralStats {
	ms := GeneralStats{}
	ms.Pairs = *NewMappedPairsStats()
	ms.Reads.MappedReadsStats = *NewMappedReadsStats()
	return &ms
}

func NewMappedReadsStats() *MappedReadsStats {
	s := MappedReadsStats{}
	s.Mapped = make(TagMap)
	return &s
}

func NewMappedPairsStats() *MappedPairsStats {
	s := MappedPairsStats{}
	s.MappedReadsStats = *NewMappedReadsStats()
	s.InsertSizes = make(TagMap)
	return &s
}

func (tm TagMap) MarshalJSON() ([]byte, error) {
	buf := &bytes.Buffer{}
	buf.Write([]byte{'{', '\n'})
	var keys []int
	for k := range tm {
		keys = append(keys, int(k))
	}
	sort.Ints(keys)
	l := len(keys)
	for i, k := range keys {
		fmt.Fprintf(buf, "\t\"%d\": \"%v\"", k, tm[k])
		if i < l-1 {
			buf.WriteByte(',')
		}
		buf.WriteByte('\n')
	}
	buf.Write([]byte{'}', '\n'})
	return buf.Bytes(), nil
}

func (s *GeneralStats) Collect(r *sam.Record) {
	NH, hasNH := r.Tag([]byte("NH"))
	if !hasNH {
		NH, _ = sam.ParseAux([]byte("NH:i:0"))
	}
	NHKey := int(NH.Value().(uint8))
	if isUnmapped(r) {
		s.Reads.Total++
		s.Reads.Unmapped++
		return
	} else {
		s.Reads.Mappings.Count++
		if isPrimary(r) {
			s.Reads.Total++
			s.Reads.Mapped[NHKey]++
			if isSplit(r) {
				s.Reads.Split++
			} else {
				s.Reads.Continuous++
			}
			if isFirstOfValidPair(r) {
				s.Pairs.Mapped[NHKey]++
				isLen := int(math.Abs(float64(r.TempLen)))
				s.Pairs.InsertSizes[isLen]++
			}
		}
	}
}

var wgg sync.WaitGroup

func gworker(in chan *sam.Record, out chan GeneralStats) {
	defer wgg.Done()
	stats := NewGeneralStats()
	for record := range in {
		stats.Collect(record)
	}
	log.Debug("Worker DONE!")
	out <- *stats
}

func cProc(br *bam.Reader, cpu int, maxBuf int, reads int) *GeneralStats {
	input := make([]chan *sam.Record, cpu)
	stats := make(chan GeneralStats, cpu)
	for i := 0; i < cpu; i++ {
		wgg.Add(1)
		input[i] = make(chan *sam.Record, maxBuf)
		go gworker(input[i], stats)
	}
	c := 0
	for {
		if reads > -1 && c == reads {
			break
		}
		record, err := br.Read()
		if err != nil {
			break
		}
		input[c%cpu] <- record
		c++
	}
	for i := 0; i < cpu; i++ {
		close(input[i])
	}
	go func() {
		wgg.Wait()
		close(stats)
	}()
	st := <-stats
	st.Merge(stats)
	return &st
}

func lProc(br *bam.Reader) *GeneralStats {
	st := NewGeneralStats()
	for {
		record, err := br.Read()
		if err != nil {
			break
		}
		st.Collect(record)
	}
	return st
}

func General(bamFile string, cpu int, maxBuf int, reads int) *GeneralStats {
	f, err := os.Open(bamFile)
	defer f.Close()
	check(err)
	br, err := bam.NewReader(f, cpu)
	defer br.Close()
	check(err)
	switch {
	case cpu <= 0:
		log.Panic("Number of cpus must be a positive number")
	case cpu == 1:
		return lProc(br)
	case cpu > 1:
		return cProc(br, cpu, maxBuf, reads)
	}
	return nil
}
<|endoftext|>


<commit_before>package git

import (
	"errors"
	"log"
	"os"
	"os/exec"
)

var gitPath string

func init() {
	var err error
	gitPath, err = exec.LookPath("git")
	if err != nil {
		log.Fatal("git must be installed")
	}
}

// Construct an *exec.Cmd for `git {args}` with a workingDirectory
func Command(workingDirectory string, args ...string) *exec.Cmd {
	c := exec.Command(gitPath, args...)
	c.Dir = workingDirectory
	return c
}

// Execute `git init {directory}` in the current workingDirectory
func Init(directory string) error {
	wd, err := os.Getwd()
	if err != nil {
		return err
	}
	return Command(wd, "init", directory).Run()
}

// Execute `git status -s` in directory
// If there is output, the directory has is dirty
func IsClean(directory string) error {
	c := Command(directory, "status", "-s")

	o, err := c.Output()
	if err != nil {
		return err
	}

	if len(o) != 0 {
		return errors.New("directory is dirty")
	}

	return nil
}

// Execute `git add --all {filepath}` in workingDirectory
func AddFilepath(workingDirectory string, filepath string) error {
	return Command(workingDirectory, "add", "--all", filepath).Run()
}

// Execute `git commit -m {msg}` in workingDirectory
func CommitWithMessage(workingDirectory string, msg string) error {
	return Command(workingDirectory, "commit", "-m", msg).Run()
}

// Execute `git commit --allow-empty -m {msg}` in workingDirectory.
func CommitEmpty(workingDirectory string, msg string) error {
	return Command(workingDirectory, "commit", "--allow-empty", "-m", msg).Run()
}
<commit_msg>Added more information to `git [add|commit]` errors | Helped with debugging the fix package specs<commit_after>package git

import (
	"errors"
	"fmt"
	"log"
	"os"
	"os/exec"
)

var gitPath string

func init() {
	var err error
	gitPath, err = exec.LookPath("git")
	if err != nil {
		log.Fatal("git must be installed")
	}
}

// Construct an *exec.Cmd for `git {args}` with a workingDirectory
func Command(workingDirectory string, args ...string) *exec.Cmd {
	c := exec.Command(gitPath, args...)
	c.Dir = workingDirectory
	return c
}

// Execute `git init {directory}` in the current workingDirectory
func Init(directory string) error {
	wd, err := os.Getwd()
	if err != nil {
		return err
	}
	return Command(wd, "init", directory).Run()
}

// Execute `git status -s` in directory
// If there is output, the directory has is dirty
func IsClean(directory string) error {
	c := Command(directory, "status", "-s")

	o, err := c.Output()
	if err != nil {
		return err
	}

	if len(o) != 0 {
		return errors.New("directory is dirty")
	}

	return nil
}

// Execute `git add --all {filepath}` in workingDirectory
func AddFilepath(workingDirectory string, filepath string) error {
	o, err := Command(workingDirectory, "add", "--all", filepath).CombinedOutput()
	if err != nil {
		return errors.New(fmt.Sprintf("error during `git add`: %s\n%s", err.Error(), string(o)))
	}
	return nil
}

// Execute `git commit -m {msg}` in workingDirectory
func CommitWithMessage(workingDirectory string, msg string) error {
	o, err := Command(workingDirectory, "commit", "-m", msg).CombinedOutput()
	if err != nil {
		return errors.New(fmt.Sprintf("error during `git commit`: %s\n%s", err.Error(), string(o)))
	}

	return nil
}

// Execute `git commit --allow-empty -m {msg}` in workingDirectory.
func CommitEmpty(workingDirectory string, msg string) error {
	return Command(workingDirectory, "commit", "--allow-empty", "-m", msg).Run()
}
<|endoftext|>


<commit_before>package giturl

import (
	"fmt"
	str "strings"
)

var dotGit = ".git"

func Http2Ssh(httpUrl string) string {
	userNameRepoPair := str.Split(httpUrl, "https://")[1]

	tempPair := str.Split(userNameRepoPair, "/")
	host := tempPair[0]
	userName := tempPair[1]
	repoName := str.Split(tempPair[2], ".git")[0]

	return fmt.Sprintf("git@%s:%s/%s.git", host, userName, repoName)
}

func Ssh2Http(sshUrl string) string {
	// fmt.Printf("%s is http ? %t\n", sshUrl, str.Contains(sshUrl, http))

	hostRepoPair := str.Split(sshUrl, "@")[1]
	host := str.Split(hostRepoPair, ":")[0]
	userNameRepoPair := str.Split(hostRepoPair, ":")[1]
	userName := str.Split(userNameRepoPair, "/")[0]
	repoName := str.Split(userNameRepoPair, "/")[1]

	if str.Contains(sshUrl, dotGit) {
		return fmt.Sprintf("https://%s/%s/%s", host, userName, repoName)
	} else {
		return fmt.Sprintf("https://%s/%s/%s.git", host, userName, repoName)
	}
}

//func main() {
//	sshUrl := "git@github.com:yelinaung/git-url.git"
//	fmt.Println(Ssh2Http(sshUrl))
//
//	httpUrl := "https://bitbucket.org/myname/myrepo.git"
//	fmt.Println(Http2Ssh(httpUrl))
//}
<commit_msg>code clean up<commit_after>package giturl

import (
	"fmt"
	str "strings"
)

var dotGit = ".git"

func Http2Ssh(httpUrl string) string {
	userNameRepoPair := str.Split(httpUrl, "https://")[1]

	tempPair := str.Split(userNameRepoPair, "/")
	host := tempPair[0]
	userName := tempPair[1]
	repoName := str.Split(tempPair[2], ".git")[0]

	return fmt.Sprintf("git@%s:%s/%s.git", host, userName, repoName)
}

func Ssh2Http(sshUrl string) string {
	hostRepoPair := str.Split(sshUrl, "@")[1]
	host := str.Split(hostRepoPair, ":")[0]
	userNameRepoPair := str.Split(hostRepoPair, ":")[1]
	userName := str.Split(userNameRepoPair, "/")[0]
	repoName := str.Split(userNameRepoPair, "/")[1]

	if str.Contains(sshUrl, dotGit) {
		return fmt.Sprintf("https://%s/%s/%s", host, userName, repoName)
	} else {
		return fmt.Sprintf("https://%s/%s/%s.git", host, userName, repoName)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"github.com/cdale77/gitmine/Godeps/_workspace/src/github.com/melvinmt/firebase"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"
)

type Event struct {
	Type       string
	Created_at string
	Actor      EventActor
	Payload    EventPayload
}

type EventActor struct {
	Login      string
	Avatar_url string
}

type EventPayload struct {
	Size    int
	Commits []CommitCommit
}

type StoredCommit struct {
	Date    string
	Login   string
	Avatar  string
	Message string
	Url     string
}

type CommitCommit struct {
	Message string
	Url     string
}

func main() {
	fullDate := time.Now().AddDate(0, 0, -1).Format("2006-01-02")
	getData(fullDate)
}

func storeCommit(event Event, commitMessage string, commitUrl string) bool {
	fmt.Println("storing event:")
	fmt.Println(event)
	authToken := os.Getenv("FIREBASE_SECRET")

	url := os.Getenv("FIREBASE_URL")

	fireBase := firebase.NewReference(url).Auth(authToken)

	var storedCommit StoredCommit
	storedCommit.Date = event.Created_at
	storedCommit.Login = event.Actor.Login
	storedCommit.Avatar = event.Actor.Avatar_url
	storedCommit.Message = commitMessage
	storedCommit.Url = commitUrl

	err := fireBase.Push(storedCommit)
	if err != nil {
		fmt.Println("Firebase error")
		fmt.Println(err)
		return false
	} else {
		fmt.Println("Firebase success")
		return true
	}
}

// There must be a better way to do this. Probably sort cussWords alpha
// and use a lookup table.
func isDirty(message string) bool {

	result := false

	cussWords := []string{
		"fuck",
		"bitch",
		"stupid",
		"tits",
		"asshole",
		"cocksucker",
		"cunt",
		"hell",
		"douche",
		"testicle",
		"twat",
		"bastard",
		"sperm",
		"shit",
		"dildo",
		"wanker",
		"prick",
		"penis",
		"vagina",
		"whore"}

	messageWords := strings.Split(message, " ")

	for _, cussWord := range cussWords {
		for _, word := range messageWords {
			if word == cussWord {
				result = true
			}
		}
	}
	return result
}

func parseEvent(line string) {
	var event Event

	jsonErr := json.Unmarshal([]byte(line), &event)
	if jsonErr != nil {
		fmt.Println("Could not parse json.")
		fmt.Println(jsonErr)
	}

	if event.Type == "PushEvent" && event.Payload.Size > 0 {

		// An event can have multiple commits.
		commits := event.Payload.Commits
		for _, commit := range commits {
			if isDirty(commit.Message) {
				fmt.Println(commit.Message)
				htmlUrl := makeHtmlUrl(commit.Url)
				storeCommit(event, commit.Message, htmlUrl)
			}
		}
	}
}

func parseFile(fName string) {
	// https://groups.google.com/forum/#!topic/golang-nuts/GjIkryuCyAY
	// TODO: standardize use of file api
	// https://stackoverflow.com/questions/1821811/how-to-read-write-from-to-file
	fileOS, err := os.Open(fName)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Can't open %s: error: %s\n", fName, err)
		os.Exit(1)
	}

	//https://stackoverflow.com/questions/1821811/how-to-read-write-from-to-file
	// close fi on exit and check for its returned error
	defer func() {
		if err := fileOS.Close(); err != nil {
			panic(err)
		}
	}()

	fileGzip, err := gzip.NewReader(fileOS)
	if err != nil {
		fmt.Printf("The file %v is not in gzip format.\n", fName)
		os.Exit(1)
	}

	fileRead := bufio.NewReader(fileGzip)
	i := 0
	for {
		line, err := fileRead.ReadString('\n')
		if err != nil {
			fmt.Println("Error reading file.")
			fmt.Println(err)
			break
		}

		parseEvent(line)

		i++
	}

	os.Remove(fName)
}

func getData(fullDate string) {

	urls := makeUrlArray(fullDate)

	for i, value := range urls {

		fmt.Println("fetching url", value)

		resp, archiveErr := http.Get(value)

		if resp != nil {
			defer resp.Body.Close()
		}

		if archiveErr != nil {
			handleError("Error getting github archive", archiveErr)
		}

		contents, readErr := ioutil.ReadAll(resp.Body)

		if readErr != nil {
			handleError("Error converting response", readErr)
		}

		fname := makeFileName(fullDate, i)

		fileErr := ioutil.WriteFile(fname, contents, 0644)

		if fileErr != nil {
			handleError("Error writing response to file", fileErr)
		}

		parseFile(fname)

	}
}

func makeUrlArray(fullDate string) [24]string {

	baseUrl := makeUrlBase(fullDate)
	urlEnd := ".json.gz"

	var urls [24]string

	for i := 0; i < 24; i++ {

		var buffer bytes.Buffer
		buffer.WriteString(baseUrl)
		buffer.WriteString("-")
		buffer.WriteString(strconv.Itoa(i))
		buffer.WriteString(urlEnd)
		url := buffer.String()

		urls[i] = url
	}

	return urls
}

func makeUrlBase(fullDate string) string {
	split := strings.Split(fullDate, "-")

	var buffer bytes.Buffer
	buffer.WriteString("http://data.githubarchive.org/")
	buffer.WriteString(split[0]) //year
	buffer.WriteString("-")
	buffer.WriteString(split[1]) //month
	buffer.WriteString("-")
	buffer.WriteString(split[2]) //day

	return buffer.String()
}

func makeFileName(fullDate string, i int) string {
	var buffer bytes.Buffer
	buffer.WriteString("data-")
	buffer.WriteString(fullDate)
	buffer.WriteString("-")
	buffer.WriteString(strconv.Itoa(i))
	buffer.WriteString(".gz")

	return buffer.String()

}

// The data does not contain an url to make a proper html page. But we can
// deduce it from the supplied api url (which makes json)
func makeHtmlUrl(apiUrl string) string {
	newUrl1 := strings.Replace(apiUrl, "api.", "", 1)
	newUrl2 := strings.Replace(newUrl1, "repos/", "", 1)
	newUrl3 := strings.Replace(newUrl2, "commits", "commit", 1)
	return newUrl3
}

func handleError(message string, err error) {
	fmt.Println(message, err)
	os.Exit(1)
}
<commit_msg>concurrency!<commit_after>package main

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"github.com/cdale77/gitmine/Godeps/_workspace/src/github.com/melvinmt/firebase"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"
)

type Event struct {
	Type       string
	Created_at string
	Actor      EventActor
	Payload    EventPayload
}

type EventActor struct {
	Login      string
	Avatar_url string
}

type EventPayload struct {
	Size    int
	Commits []CommitCommit
}

type StoredCommit struct {
	Date    string
	Login   string
	Avatar  string
	Message string
	Url     string
}

type CommitCommit struct {
	Message string
	Url     string
}

func main() {
	fullDate := time.Now().AddDate(0, 0, -1).Format("2006-01-02")
	getData(fullDate)
}

func storeCommit(event Event, commitMessage string, commitUrl string) bool {
	fmt.Println("storing event:")
	fmt.Println(event)
	authToken := os.Getenv("FIREBASE_SECRET")

	url := os.Getenv("FIREBASE_URL")

	fireBase := firebase.NewReference(url).Auth(authToken)

	var storedCommit StoredCommit
	storedCommit.Date = event.Created_at
	storedCommit.Login = event.Actor.Login
	storedCommit.Avatar = event.Actor.Avatar_url
	storedCommit.Message = commitMessage
	storedCommit.Url = commitUrl

	err := fireBase.Push(storedCommit)
	if err != nil {
		fmt.Println("Firebase error")
		fmt.Println(err)
		return false
	} else {
		fmt.Println("Firebase success")
		return true
	}
}

// There must be a better way to do this. Probably sort cussWords alpha
// and use a lookup table.
func isDirty(message string) bool {

	result := false

	cussWords := []string{
		"fuck",
		"bitch",
		"stupid",
		"tits",
		"asshole",
		"cocksucker",
		"cunt",
		"hell",
		"douche",
		"testicle",
		"twat",
		"bastard",
		"sperm",
		"shit",
		"dildo",
		"wanker",
		"prick",
		"penis",
		"vagina",
		"whore"}

	messageWords := strings.Split(message, " ")

	for _, cussWord := range cussWords {
		for _, word := range messageWords {
			if word == cussWord {
				result = true
			}
		}
	}
	return result
}

func parseEvent(line string) {
	var event Event

	jsonErr := json.Unmarshal([]byte(line), &event)
	if jsonErr != nil {
		fmt.Println("Could not parse json.")
		fmt.Println(jsonErr)
	}

	if event.Type == "PushEvent" && event.Payload.Size > 0 {

		// An event can have multiple commits.
		commits := event.Payload.Commits
		for _, commit := range commits {
			if isDirty(commit.Message) {
				fmt.Println(commit.Message)
				htmlUrl := makeHtmlUrl(commit.Url)
				storeCommit(event, commit.Message, htmlUrl)
			}
		}
	}
}

func parseFile(fName string) {
	// https://groups.google.com/forum/#!topic/golang-nuts/GjIkryuCyAY
	// TODO: standardize use of file api
	// https://stackoverflow.com/questions/1821811/how-to-read-write-from-to-file
	fileOS, err := os.Open(fName)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Can't open %s: error: %s\n", fName, err)
		os.Exit(1)
	}

	//https://stackoverflow.com/questions/1821811/how-to-read-write-from-to-file
	// close fi on exit and check for its returned error
	defer func() {
		if err := fileOS.Close(); err != nil {
			panic(err)
		}
	}()

	fileGzip, err := gzip.NewReader(fileOS)
	if err != nil {
		fmt.Printf("The file %v is not in gzip format.\n", fName)
		os.Exit(1)
	}

	fileRead := bufio.NewReader(fileGzip)
	i := 0
	for {
		line, err := fileRead.ReadString('\n')
		if err != nil {
			fmt.Println("Error reading file.")
			fmt.Println(err)
			break
		}

		parseEvent(line)

		i++
	}

	os.Remove(fName)
}

func getData(fullDate string) {

	urls := makeUrlArray(fullDate)

	for i, value := range urls {

		fmt.Println("fetching url", value)

		resp, archiveErr := http.Get(value)

		if resp != nil {
			defer resp.Body.Close()
		}

		if archiveErr != nil {
			handleError("Error getting github archive", archiveErr)
		}

		contents, readErr := ioutil.ReadAll(resp.Body)

		if readErr != nil {
			handleError("Error converting response", readErr)
		}

		fname := makeFileName(fullDate, i)

		fileErr := ioutil.WriteFile(fname, contents, 0644)

		if fileErr != nil {
			handleError("Error writing response to file", fileErr)
		}

		go parseFile(fname)

	}
}

func makeUrlArray(fullDate string) [24]string {

	baseUrl := makeUrlBase(fullDate)
	urlEnd := ".json.gz"

	var urls [24]string

	for i := 0; i < 24; i++ {

		var buffer bytes.Buffer
		buffer.WriteString(baseUrl)
		buffer.WriteString("-")
		buffer.WriteString(strconv.Itoa(i))
		buffer.WriteString(urlEnd)
		url := buffer.String()

		urls[i] = url
	}

	return urls
}

func makeUrlBase(fullDate string) string {
	split := strings.Split(fullDate, "-")

	var buffer bytes.Buffer
	buffer.WriteString("http://data.githubarchive.org/")
	buffer.WriteString(split[0]) //year
	buffer.WriteString("-")
	buffer.WriteString(split[1]) //month
	buffer.WriteString("-")
	buffer.WriteString(split[2]) //day

	return buffer.String()
}

func makeFileName(fullDate string, i int) string {
	var buffer bytes.Buffer
	buffer.WriteString("data-")
	buffer.WriteString(fullDate)
	buffer.WriteString("-")
	buffer.WriteString(strconv.Itoa(i))
	buffer.WriteString(".gz")

	return buffer.String()

}

// The data does not contain an url to make a proper html page. But we can
// deduce it from the supplied api url (which makes json)
func makeHtmlUrl(apiUrl string) string {
	newUrl1 := strings.Replace(apiUrl, "api.", "", 1)
	newUrl2 := strings.Replace(newUrl1, "repos/", "", 1)
	newUrl3 := strings.Replace(newUrl2, "commits", "commit", 1)
	return newUrl3
}

func handleError(message string, err error) {
	fmt.Println(message, err)
	os.Exit(1)
}
<|endoftext|>