<commit_before>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Timothy Chen
// Author: Ben Darnell

package server

import (
	"sync"
	"time"

	"github.com/cockroachdb/cockroach/gossip"
	"github.com/cockroachdb/cockroach/multiraft"
	"github.com/cockroachdb/cockroach/roachpb"
	"github.com/cockroachdb/cockroach/rpc"
	"github.com/cockroachdb/cockroach/util"
	"github.com/cockroachdb/cockroach/util/log"
	"github.com/gogo/protobuf/proto"

	gorpc "net/rpc"
)

const (
	raftServiceName = "MultiRaft"
	raftMessageName = raftServiceName + ".RaftMessage"
	// Outgoing messages are queued on a per-node basis on a channel of
	// this size.
	raftSendBufferSize = 500
	// When no message has been sent to a Node for that duration, the
	// corresponding instance of processQueue will shut down.
	raftIdleTimeout = time.Minute
)

// rpcTransport handles the rpc messages for multiraft.
type rpcTransport struct {
	gossip     *gossip.Gossip
	rpcServer  *rpc.Server
	rpcContext *rpc.Context
	mu         sync.Mutex
	servers    map[roachpb.StoreID]multiraft.ServerInterface
	queues     map[roachpb.StoreID]chan *multiraft.RaftMessageRequest
}

// newRPCTransport creates a new rpcTransport with specified gossip and rpc server.
func newRPCTransport(gossip *gossip.Gossip, rpcServer *rpc.Server, rpcContext *rpc.Context) (
	multiraft.Transport, error) {
	t := &rpcTransport{
		gossip:     gossip,
		rpcServer:  rpcServer,
		rpcContext: rpcContext,
		servers:    make(map[roachpb.StoreID]multiraft.ServerInterface),
		queues:     make(map[roachpb.StoreID]chan *multiraft.RaftMessageRequest),
	}

	if t.rpcServer != nil {
		if err := t.rpcServer.RegisterAsync(raftMessageName, false, /*not public*/
			t.RaftMessage, &multiraft.RaftMessageRequest{}); err != nil {
			return nil, err
		}
	}

	return t, nil
}

// RaftMessage proxies the incoming request to the listening server interface.
func (t *rpcTransport) RaftMessage(args proto.Message, callback func(proto.Message, error)) {
	req := args.(*multiraft.RaftMessageRequest)

	t.mu.Lock()
	server, ok := t.servers[req.ToReplica.StoreID]
	t.mu.Unlock()

	if !ok {
		callback(nil, util.Errorf("Unable to proxy message to node: %d", req.Message.To))
		return
	}

	// Raft responses are empty so we don't actually need to convert
	// between multiraft's internal struct and the external proto
	// representation. In fact, we don't even need to wait for the
	// message to be processed to invoke the callback. We are just
	// (ab)using the async handler mechanism to get this (synchronous)
	// handler called in the RPC server's goroutine so we can preserve
	// order of incoming messages.
	resp, err := server.RaftMessage(req)
	callback(resp, err)
}

// Listen implements the multiraft.Transport interface by registering a ServerInterface
// to receive proxied messages.
func (t *rpcTransport) Listen(id roachpb.StoreID, server multiraft.ServerInterface) error {
	t.mu.Lock()
	t.servers[id] = server
	t.mu.Unlock()
	return nil
}

// Stop implements the multiraft.Transport interface by unregistering the server id.
func (t *rpcTransport) Stop(id roachpb.StoreID) {
	t.mu.Lock()
	defer t.mu.Unlock()
	delete(t.servers, id)
}

// processQueue creates a client and sends messages from its designated queue
// via that client, exiting when the client fails or when it idles out. All
// messages remaining in the queue at that point are lost and a new instance of
// processQueue should be started by the next message to be sent.
// TODO(tschottdorf) should let MultiRaft know if the node is down;
// need a feedback mechanism for that. Potentially easiest is to arrange for
// the next call to Send() to fail appropriately.
func (t *rpcTransport) processQueue(nodeID roachpb.NodeID, storeID roachpb.StoreID) {
	t.mu.Lock()
	ch, ok := t.queues[storeID]
	t.mu.Unlock()
	if !ok {
		return
	}
	// Clean-up when the loop below shuts down.
	defer func() {
		t.mu.Lock()
		delete(t.queues, storeID)
		t.mu.Unlock()
	}()

	addr, err := t.gossip.GetNodeIDAddress(nodeID)
	if err != nil {
		log.Errorf("could not get address for node %d: %s", nodeID, err)
		return
	}
	client := rpc.NewClient(addr, t.rpcContext)
	select {
	case <-t.rpcContext.Stopper.ShouldStop():
		return
	case <-client.Closed:
		log.Warningf("raft client for node %d was closed", nodeID)
		return
	case <-time.After(raftIdleTimeout):
		// Should never happen.
		log.Errorf("raft client for node %d stuck connecting", nodeID)
		return
	case <-client.Healthy():
	}

	done := make(chan *gorpc.Call, cap(ch))
	var req *multiraft.RaftMessageRequest
	protoResp := &multiraft.RaftMessageResponse{}
	for {
		select {
		case <-t.rpcContext.Stopper.ShouldStop():
			return
		case <-time.After(raftIdleTimeout):
			if log.V(1) {
				log.Infof("closing Raft transport to %d due to inactivity", nodeID)
			}
			return
		case <-client.Closed:
			log.Warningf("raft client for node %d closed", nodeID)
			return
		case call := <-done:
			if call.Error != nil {
				log.Errorf("raft message to node %d failed: %s", nodeID, call.Error)
			}
			continue
		case req = <-ch:
		}
		if req == nil {
			return
		}

		client.Go(raftMessageName, req, protoResp, done)
	}
}

// Send a message to the recipient specified in the request.
func (t *rpcTransport) Send(req *multiraft.RaftMessageRequest) error {
	t.mu.Lock()
	ch, ok := t.queues[req.ToReplica.StoreID]
	if !ok {
		ch = make(chan *multiraft.RaftMessageRequest, raftSendBufferSize)
		t.queues[req.ToReplica.StoreID] = ch
		go t.processQueue(req.ToReplica.NodeID, req.ToReplica.StoreID)
	}
	t.mu.Unlock()

	select {
	case ch <- req:
	default:
		return util.Errorf("queue for node %d is full", req.Message.To)
	}
	return nil
}

// Close shuts down an rpcTransport.
func (t *rpcTransport) Close() {
	// No-op since we share the global cache of client connections.
}
<commit_msg>downgrade a spammy log message<commit_after>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Timothy Chen
// Author: Ben Darnell

package server

import (
	"sync"
	"time"

	"github.com/cockroachdb/cockroach/gossip"
	"github.com/cockroachdb/cockroach/multiraft"
	"github.com/cockroachdb/cockroach/roachpb"
	"github.com/cockroachdb/cockroach/rpc"
	"github.com/cockroachdb/cockroach/util"
	"github.com/cockroachdb/cockroach/util/log"
	"github.com/gogo/protobuf/proto"

	gorpc "net/rpc"
)

const (
	raftServiceName = "MultiRaft"
	raftMessageName = raftServiceName + ".RaftMessage"
	// Outgoing messages are queued on a per-node basis on a channel of
	// this size.
	raftSendBufferSize = 500
	// When no message has been sent to a Node for that duration, the
	// corresponding instance of processQueue will shut down.
	raftIdleTimeout = time.Minute
)

// rpcTransport handles the rpc messages for multiraft.
type rpcTransport struct {
	gossip     *gossip.Gossip
	rpcServer  *rpc.Server
	rpcContext *rpc.Context
	mu         sync.Mutex
	servers    map[roachpb.StoreID]multiraft.ServerInterface
	queues     map[roachpb.StoreID]chan *multiraft.RaftMessageRequest
}

// newRPCTransport creates a new rpcTransport with specified gossip and rpc server.
func newRPCTransport(gossip *gossip.Gossip, rpcServer *rpc.Server, rpcContext *rpc.Context) (
	multiraft.Transport, error) {
	t := &rpcTransport{
		gossip:     gossip,
		rpcServer:  rpcServer,
		rpcContext: rpcContext,
		servers:    make(map[roachpb.StoreID]multiraft.ServerInterface),
		queues:     make(map[roachpb.StoreID]chan *multiraft.RaftMessageRequest),
	}

	if t.rpcServer != nil {
		if err := t.rpcServer.RegisterAsync(raftMessageName, false, /*not public*/
			t.RaftMessage, &multiraft.RaftMessageRequest{}); err != nil {
			return nil, err
		}
	}

	return t, nil
}

// RaftMessage proxies the incoming request to the listening server interface.
func (t *rpcTransport) RaftMessage(args proto.Message, callback func(proto.Message, error)) {
	req := args.(*multiraft.RaftMessageRequest)

	t.mu.Lock()
	server, ok := t.servers[req.ToReplica.StoreID]
	t.mu.Unlock()

	if !ok {
		callback(nil, util.Errorf("Unable to proxy message to node: %d", req.Message.To))
		return
	}

	// Raft responses are empty so we don't actually need to convert
	// between multiraft's internal struct and the external proto
	// representation. In fact, we don't even need to wait for the
	// message to be processed to invoke the callback. We are just
	// (ab)using the async handler mechanism to get this (synchronous)
	// handler called in the RPC server's goroutine so we can preserve
	// order of incoming messages.
	resp, err := server.RaftMessage(req)
	callback(resp, err)
}

// Listen implements the multiraft.Transport interface by registering a ServerInterface
// to receive proxied messages.
func (t *rpcTransport) Listen(id roachpb.StoreID, server multiraft.ServerInterface) error {
	t.mu.Lock()
	t.servers[id] = server
	t.mu.Unlock()
	return nil
}

// Stop implements the multiraft.Transport interface by unregistering the server id.
func (t *rpcTransport) Stop(id roachpb.StoreID) {
	t.mu.Lock()
	defer t.mu.Unlock()
	delete(t.servers, id)
}

// processQueue creates a client and sends messages from its designated queue
// via that client, exiting when the client fails or when it idles out. All
// messages remaining in the queue at that point are lost and a new instance of
// processQueue should be started by the next message to be sent.
// TODO(tschottdorf) should let MultiRaft know if the node is down;
// need a feedback mechanism for that. Potentially easiest is to arrange for
// the next call to Send() to fail appropriately.
func (t *rpcTransport) processQueue(nodeID roachpb.NodeID, storeID roachpb.StoreID) {
	t.mu.Lock()
	ch, ok := t.queues[storeID]
	t.mu.Unlock()
	if !ok {
		return
	}
	// Clean-up when the loop below shuts down.
	defer func() {
		t.mu.Lock()
		delete(t.queues, storeID)
		t.mu.Unlock()
	}()

	addr, err := t.gossip.GetNodeIDAddress(nodeID)
	if err != nil {
		if log.V(1) {
			log.Errorf("could not get address for node %d: %s", nodeID, err)
		}
		return
	}
	client := rpc.NewClient(addr, t.rpcContext)
	select {
	case <-t.rpcContext.Stopper.ShouldStop():
		return
	case <-client.Closed:
		log.Warningf("raft client for node %d was closed", nodeID)
		return
	case <-time.After(raftIdleTimeout):
		// Should never happen.
		log.Errorf("raft client for node %d stuck connecting", nodeID)
		return
	case <-client.Healthy():
	}

	done := make(chan *gorpc.Call, cap(ch))
	var req *multiraft.RaftMessageRequest
	protoResp := &multiraft.RaftMessageResponse{}
	for {
		select {
		case <-t.rpcContext.Stopper.ShouldStop():
			return
		case <-time.After(raftIdleTimeout):
			if log.V(1) {
				log.Infof("closing Raft transport to %d due to inactivity", nodeID)
			}
			return
		case <-client.Closed:
			log.Warningf("raft client for node %d closed", nodeID)
			return
		case call := <-done:
			if call.Error != nil {
				log.Errorf("raft message to node %d failed: %s", nodeID, call.Error)
			}
			continue
		case req = <-ch:
		}
		if req == nil {
			return
		}

		client.Go(raftMessageName, req, protoResp, done)
	}
}

// Send a message to the recipient specified in the request.
func (t *rpcTransport) Send(req *multiraft.RaftMessageRequest) error {
	t.mu.Lock()
	ch, ok := t.queues[req.ToReplica.StoreID]
	if !ok {
		ch = make(chan *multiraft.RaftMessageRequest, raftSendBufferSize)
		t.queues[req.ToReplica.StoreID] = ch
		go t.processQueue(req.ToReplica.NodeID, req.ToReplica.StoreID)
	}
	t.mu.Unlock()

	select {
	case ch <- req:
	default:
		return util.Errorf("queue for node %d is full", req.Message.To)
	}
	return nil
}

// Close shuts down an rpcTransport.
func (t *rpcTransport) Close() {
	// No-op since we share the global cache of client connections.
}
<|endoftext|>


<commit_before>package server

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
)

type RedditIndexerConfiguration struct {
	Database *Database
}

type RedditIndexer struct {
	configuration RedditIndexerConfiguration
	closeSignal   chan bool
}

func NewRedditIndexer(configuration RedditIndexerConfiguration) (*RedditIndexer, error) {
	ret := &RedditIndexer{
		configuration: configuration,
		closeSignal:   make(chan bool),
	}
	go ret.run()
	return ret, nil
}

func (indexer *RedditIndexer) Close() {
	indexer.closeSignal <- true
}

func (indexer *RedditIndexer) run() {
	log.Info("starting reddit indexer")

	users := []string{
		"chris_wilson", "Bex_GGG", "Negitivefrags", "Omnitect", "qarldev", "BrianWeissman_GGG",
		"Mark_GGG", "RhysGGG", "Dan_GGG", "Rory_Rackham", "Blake_GGG", "Fitzy_GGG", "Hartlin_GGG",
	}
	next := 0

	for {
		select {
		case <-indexer.closeSignal:
			return
		default:
			indexer.index(users[next])
			next += 1
			if next >= len(users) {
				next = 0
			}
			time.Sleep(time.Second * 2)
		}
	}
}

func ParseRedditActivity(b []byte) ([]Activity, string, error) {
	activity := []Activity(nil)

	var root struct {
		Data struct {
			After    string `json:"after"`
			Children []struct {
				Kind string `json:"kind"`
				Data struct {
					Id           string  `json:"id"`
					Author       string  `json:"author"`
					BodyHTML     string  `json:"body_html"`
					SelftextHTML string  `json:"selftext_html"`
					SubredditId  string  `json:"subreddit_id"`
					Permalink    string  `json:"permalink"`
					URL          string  `json:"url"`
					Title        string  `json:"title"`
					CreatedUTC   float64 `json:"created_utc"`
					LinkId       string  `json:"link_id"`
					LinkTitle    string  `json:"link_title"`
				} `json:"data"`
			} `json:"children"`
		} `json:"data"`
	}

	if err := json.Unmarshal(b, &root); err != nil {
		return nil, "", err
	}

	for _, thing := range root.Data.Children {
		if thing.Data.SubredditId != "t5_2sf6m" {
			continue
		}
		switch thing.Kind {
		case "t1":
			activity = append(activity, &RedditComment{
				Id:        thing.Data.Id,
				Author:    thing.Data.Author,
				BodyHTML:  thing.Data.BodyHTML,
				PostId:    strings.TrimPrefix(thing.Data.LinkId, "t3_"),
				PostTitle: thing.Data.LinkTitle,
				Time:      time.Unix(int64(thing.Data.CreatedUTC), 0),
			})
		case "t3":
			activity = append(activity, &RedditPost{
				Id:        thing.Data.Id,
				Author:    thing.Data.Author,
				BodyHTML:  thing.Data.SelftextHTML,
				Permalink: thing.Data.Permalink,
				Title:     thing.Data.Title,
				URL:       thing.Data.URL,
				Time:      time.Unix(int64(thing.Data.CreatedUTC), 0),
			})
		}
	}

	return activity, root.Data.After, nil
}

func (indexer *RedditIndexer) redditActivity(user string, page string) ([]Activity, string, error) {
	url := fmt.Sprintf("https://www.reddit.com/user/%v.json?count=25&after=%v&raw_json=1", user, page)
	client := http.Client{
		Timeout: time.Second * 10,
	}

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, "", err
	}
	req.Header.Add("User-Agent", "GGG Tracker (https://github.com/ccbrown/gggtracker) by /u/rz2yoj")

	resp, err := client.Do(req)
	if err != nil {
		return nil, "", err
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, "", err
	}

	return ParseRedditActivity(body)
}

func (indexer *RedditIndexer) index(user string) {
	logger := log.WithFields(log.Fields{
		"user": user,
	})

	cutoff := time.Now().Add(time.Hour * -12)
	activity := []Activity(nil)

	for page := ""; ; {
		things, next, err := indexer.redditActivity(user, page)
		page = next
		if err != nil {
			logger.WithError(err).Error("error requesting reddit activity")
		}

		done := len(things) == 0
		for _, thing := range things {
			if thing.ActivityTime().Before(cutoff) {
				done = true
			}
			activity = append(activity, thing)
		}

		logger.WithFields(log.Fields{
			"count": len(things),
		}).Info("received reddit activity")

		if done {
			break
		}
		time.Sleep(time.Second * 2)
	}

	if len(activity) > 0 {
		indexer.configuration.Database.AddActivity(activity)
	}
}
<commit_msg>add /u/Hrishi_GGG to reddit posters<commit_after>package server

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
)

type RedditIndexerConfiguration struct {
	Database *Database
}

type RedditIndexer struct {
	configuration RedditIndexerConfiguration
	closeSignal   chan bool
}

func NewRedditIndexer(configuration RedditIndexerConfiguration) (*RedditIndexer, error) {
	ret := &RedditIndexer{
		configuration: configuration,
		closeSignal:   make(chan bool),
	}
	go ret.run()
	return ret, nil
}

func (indexer *RedditIndexer) Close() {
	indexer.closeSignal <- true
}

func (indexer *RedditIndexer) run() {
	log.Info("starting reddit indexer")

	users := []string{
		"chris_wilson", "Bex_GGG", "Negitivefrags", "Omnitect", "qarldev", "BrianWeissman_GGG",
		"Mark_GGG", "RhysGGG", "Dan_GGG", "Rory_Rackham", "Blake_GGG", "Fitzy_GGG", "Hartlin_GGG",
		"Hrishi_GGG",
	}
	next := 0

	for {
		select {
		case <-indexer.closeSignal:
			return
		default:
			indexer.index(users[next])
			next += 1
			if next >= len(users) {
				next = 0
			}
			time.Sleep(time.Second * 2)
		}
	}
}

func ParseRedditActivity(b []byte) ([]Activity, string, error) {
	activity := []Activity(nil)

	var root struct {
		Data struct {
			After    string `json:"after"`
			Children []struct {
				Kind string `json:"kind"`
				Data struct {
					Id           string  `json:"id"`
					Author       string  `json:"author"`
					BodyHTML     string  `json:"body_html"`
					SelftextHTML string  `json:"selftext_html"`
					SubredditId  string  `json:"subreddit_id"`
					Permalink    string  `json:"permalink"`
					URL          string  `json:"url"`
					Title        string  `json:"title"`
					CreatedUTC   float64 `json:"created_utc"`
					LinkId       string  `json:"link_id"`
					LinkTitle    string  `json:"link_title"`
				} `json:"data"`
			} `json:"children"`
		} `json:"data"`
	}

	if err := json.Unmarshal(b, &root); err != nil {
		return nil, "", err
	}

	for _, thing := range root.Data.Children {
		if thing.Data.SubredditId != "t5_2sf6m" {
			continue
		}
		switch thing.Kind {
		case "t1":
			activity = append(activity, &RedditComment{
				Id:        thing.Data.Id,
				Author:    thing.Data.Author,
				BodyHTML:  thing.Data.BodyHTML,
				PostId:    strings.TrimPrefix(thing.Data.LinkId, "t3_"),
				PostTitle: thing.Data.LinkTitle,
				Time:      time.Unix(int64(thing.Data.CreatedUTC), 0),
			})
		case "t3":
			activity = append(activity, &RedditPost{
				Id:        thing.Data.Id,
				Author:    thing.Data.Author,
				BodyHTML:  thing.Data.SelftextHTML,
				Permalink: thing.Data.Permalink,
				Title:     thing.Data.Title,
				URL:       thing.Data.URL,
				Time:      time.Unix(int64(thing.Data.CreatedUTC), 0),
			})
		}
	}

	return activity, root.Data.After, nil
}

func (indexer *RedditIndexer) redditActivity(user string, page string) ([]Activity, string, error) {
	url := fmt.Sprintf("https://www.reddit.com/user/%v.json?count=25&after=%v&raw_json=1", user, page)
	client := http.Client{
		Timeout: time.Second * 10,
	}

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, "", err
	}
	req.Header.Add("User-Agent", "GGG Tracker (https://github.com/ccbrown/gggtracker) by /u/rz2yoj")

	resp, err := client.Do(req)
	if err != nil {
		return nil, "", err
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, "", err
	}

	return ParseRedditActivity(body)
}

func (indexer *RedditIndexer) index(user string) {
	logger := log.WithFields(log.Fields{
		"user": user,
	})

	cutoff := time.Now().Add(time.Hour * -12)
	activity := []Activity(nil)

	for page := ""; ; {
		things, next, err := indexer.redditActivity(user, page)
		page = next
		if err != nil {
			logger.WithError(err).Error("error requesting reddit activity")
		}

		done := len(things) == 0
		for _, thing := range things {
			if thing.ActivityTime().Before(cutoff) {
				done = true
			}
			activity = append(activity, thing)
		}

		logger.WithFields(log.Fields{
			"count": len(things),
		}).Info("received reddit activity")

		if done {
			break
		}
		time.Sleep(time.Second * 2)
	}

	if len(activity) > 0 {
		indexer.configuration.Database.AddActivity(activity)
	}
}
<|endoftext|>


<commit_before>package analysis

import (
	"bytes"
	"fmt"
	"github.com/balzaczyy/golucene/core/util"
	"io"
)

// analysis/Analyzer.java

/*
An Analyzer builds TokenStreams, which analyze text. It thus reprents a policy
for extracting index terms for text.

In order to define what analysis is done, subclass must define their
TokenStreamConents in CreateComponents(string, Reader). The components are
then reused in each call to TokenStream(string, Reader).

Also note that one should Clone() Analyzer for each Go routine if
default ReuseStrategy is used.
*/
type Analyzer interface {
	TokenStreamForReader(string, io.RuneReader) (TokenStream, error)
	// Returns a TokenStream suitable for fieldName, tokenizing the
	// contents of text.
	//
	// This method uses createComponents(string, Reader) to obtain an
	// instance of TokenStreamComponents. It returns the sink of the
	// components and stores the components internally. Subsequent
	// calls to this method will reuse the previously stored components
	// after resetting them through TokenStreamComponents.SetReader(Reader).
	//
	// NOTE: After calling this method, the consumer must follow the
	// workflow described in TokenStream to propperly consume its
	// contents. See the Analysis package documentation for some
	// examples demonstrating this.
	TokenStreamForString(fieldName, text string) (TokenStream, error)
	PositionIncrementGap(string) int
	OffsetGap(string) int
}

type AnalyzerSPI interface {
	// Creates a new TokenStreamComponents instance for this analyzer.
	CreateComponents(fieldName string, reader io.RuneReader) *TokenStreamComponents
	// Override this if you want to add a CharFilter chain.
	//
	// The default implementation returns reader unchanged.
	InitReader(fieldName string, reader io.RuneReader) io.RuneReader
}

type container struct {
	value interface{}
}

type AnalyzerImpl struct {
	Spi           AnalyzerSPI
	reuseStrategy ReuseStrategy
	version       *util.SetOnce
	// Since Go doesn't have ThreadLocal alternatives, to share
	// Analyzer, one must Clone() the Analyzer for each Go routine. It
	// also means the performance may not be competitive compared to
	// Lucene Java Analyzer.
	storedValue *container
}

/*
Create a new Analyzer, reusing the same set of components per-thread
across calls to TokenStream(string, Reader).
*/
func NewAnalyzer() *AnalyzerImpl {
	return NewAnalyzerWithStrategy(GLOBAL_REUSE_STRATEGY)
}

func NewAnalyzerWithStrategy(reuseStrategy ReuseStrategy) *AnalyzerImpl {
	ans := &AnalyzerImpl{
		reuseStrategy: reuseStrategy,
		version:       util.NewSetOnce(),
		storedValue:   &container{nil},
	}
	ans.Spi = ans
	return ans
}

func (a *AnalyzerImpl) CreateComponents(fieldName string, reader io.RuneReader) *TokenStreamComponents {
	panic("must be inherited and implemented")
}

func (a *AnalyzerImpl) TokenStreamForReader(fieldName string, reader io.RuneReader) (TokenStream, error) {
	components := a.reuseStrategy.ReusableComponents(a, fieldName)
	r := a.InitReader(fieldName, reader)
	if components == nil {
		panic("not implemented yet")
	} else {
		if err := components.SetReader(r); err != nil {
			return nil, err
		}
	}
	return components.TokenStream(), nil
}

func (a *AnalyzerImpl) TokenStreamForString(fieldName, text string) (TokenStream, error) {
	components := a.reuseStrategy.ReusableComponents(a, fieldName)
	var strReader *ReusableStringReader
	if components == nil || components.reusableStringReader == nil {
		strReader = new(ReusableStringReader)
	} else {
		strReader = components.reusableStringReader
	}
	strReader.setValue(text)
	r := a.InitReader(fieldName, strReader)
	if components == nil {
		components = a.Spi.CreateComponents(fieldName, r)
		a.reuseStrategy.SetReusableComponents(a, fieldName, components)
	} else {
		err := components.SetReader(r)
		if err != nil {
			return nil, err
		}
	}
	components.reusableStringReader = strReader
	return components.TokenStream(), nil
}

func (a *AnalyzerImpl) InitReader(fieldName string, reader io.RuneReader) io.RuneReader {
	return reader
}

func (a *AnalyzerImpl) PositionIncrementGap(fieldName string) int {
	return 0
}

func (a *AnalyzerImpl) OffsetGap(fieldName string) int {
	return 1
}

func (a *AnalyzerImpl) SetVersion(v util.Version) {
	a.version.Set(v)
}

type myTokenizer interface {
	SetReader(io.RuneReader) error
}

/*
This class encapsulates the outer components of a token stream. It
provides access to the source Tokenizer and the outer end (sink), an
instance of TokenFilter which also serves as the TokenStream returned
by Analyzer.tokenStream(string, Reader).
*/
type TokenStreamComponents struct {
	// Original source of tokens.
	source myTokenizer
	// Sink tokenStream, such as the outer tokenFilter decorating the
	// chain. This can be the source if there are no filters.
	sink TokenStream
	// Internal cache only used by Analyzer.TokenStreamForString().
	reusableStringReader *ReusableStringReader
	// Resets the encapculated components with the given reader. If the
	// components canno be reset, an error should be returned.
	SetReader func(io.RuneReader) error
}

func NewTokenStreamComponents(source myTokenizer, result TokenStream) *TokenStreamComponents {
	ans := &TokenStreamComponents{source: source, sink: result}
	ans.SetReader = func(reader io.RuneReader) error {
		return ans.source.SetReader(reader)
	}
	return ans
}

/* Returns the sink TokenStream */
func (cp *TokenStreamComponents) TokenStream() TokenStream {
	return cp.sink
}

// L329

// Strategy defining how TokenStreamComponents are reused per call to
// TokenStream(string, io.Reader)
type ReuseStrategy interface {
	// Gets the reusable TokenStreamComponents for the field with the
	// given name.
	ReusableComponents(*AnalyzerImpl, string) *TokenStreamComponents
	// Stores the given TokenStreamComponents as the reusable
	// components for the field with the given name.
	SetReusableComponents(*AnalyzerImpl, string, *TokenStreamComponents)
}

type ReuseStrategyImpl struct {
}

/* Returns the currently stored value */
func (rs *ReuseStrategyImpl) storedValue(a *AnalyzerImpl) interface{} {
	assert2(a.storedValue != nil, "this Analyzer is closed")
	return a.storedValue.value
}

/* Set the stored value. */
func (rs *ReuseStrategyImpl) setStoredValue(a *AnalyzerImpl, v interface{}) {
	assert2(a.storedValue != nil, "this Analyzer is closed")
	a.storedValue.value = v
}

func assert2(ok bool, msg string, args ...interface{}) {
	if !ok {
		panic(fmt.Sprintf(msg, args...))
	}
}

/* A predefined ReuseStrategy that reuses the same components for every field */
var GLOBAL_REUSE_STRATEGY = new(GlobalReuseStrategy)

type GlobalReuseStrategy struct {
	*ReuseStrategyImpl
}

func (rs *GlobalReuseStrategy) ReusableComponents(a *AnalyzerImpl, fieldName string) *TokenStreamComponents {
	if ans := rs.storedValue(a); ans != nil {
		return ans.(*TokenStreamComponents)
	}
	return nil
}

func (rs *GlobalReuseStrategy) SetReusableComponents(a *AnalyzerImpl, fieldName string, components *TokenStreamComponents) {
	rs.setStoredValue(a, components)
}

// L423
// A predefined ReuseStrategy that reuses components per-field by
// maintaining a Map of TokenStreamComponent per field name.
var PER_FIELD_REUSE_STRATEGY = &PerFieldReuseStrategy{}

// Implementation of ReuseStrategy that reuses components per-field by
// maintianing a Map of TokenStreamComponent per field name.
type PerFieldReuseStrategy struct {
}

func (rs *PerFieldReuseStrategy) ReusableComponents(a *AnalyzerImpl, fieldName string) *TokenStreamComponents {
	panic("not implemented yet")
}

func (rs *PerFieldReuseStrategy) SetReusableComponents(a *AnalyzerImpl, fieldName string, components *TokenStreamComponents) {
	panic("not implemneted yet")
}

// analysis/ReusableStringReader.java

/* Internal class to enale reuse of the string reader by Analyzer.TokenStreamForString() */
type ReusableStringReader struct {
	s *bytes.Buffer
}

func (r *ReusableStringReader) setValue(s string) {
	r.s = bytes.NewBufferString(s)
}

func (r *ReusableStringReader) Read(p []byte) (int, error) {
	return r.s.Read(p)
}

func (r *ReusableStringReader) ReadRune() (rune, int, error) {
	return r.s.ReadRune()
}

func (r *ReusableStringReader) Close() error {
	r.s = nil
	return nil
}
<commit_msg>add missing Getter for Analyzer.version<commit_after>package analysis

import (
	"bytes"
	"fmt"
	"github.com/balzaczyy/golucene/core/util"
	"io"
)

// analysis/Analyzer.java

/*
An Analyzer builds TokenStreams, which analyze text. It thus reprents a policy
for extracting index terms for text.

In order to define what analysis is done, subclass must define their
TokenStreamConents in CreateComponents(string, Reader). The components are
then reused in each call to TokenStream(string, Reader).

Also note that one should Clone() Analyzer for each Go routine if
default ReuseStrategy is used.
*/
type Analyzer interface {
	TokenStreamForReader(string, io.RuneReader) (TokenStream, error)
	// Returns a TokenStream suitable for fieldName, tokenizing the
	// contents of text.
	//
	// This method uses createComponents(string, Reader) to obtain an
	// instance of TokenStreamComponents. It returns the sink of the
	// components and stores the components internally. Subsequent
	// calls to this method will reuse the previously stored components
	// after resetting them through TokenStreamComponents.SetReader(Reader).
	//
	// NOTE: After calling this method, the consumer must follow the
	// workflow described in TokenStream to propperly consume its
	// contents. See the Analysis package documentation for some
	// examples demonstrating this.
	TokenStreamForString(fieldName, text string) (TokenStream, error)
	PositionIncrementGap(string) int
	OffsetGap(string) int
}

type AnalyzerSPI interface {
	// Creates a new TokenStreamComponents instance for this analyzer.
	CreateComponents(fieldName string, reader io.RuneReader) *TokenStreamComponents
	// Override this if you want to add a CharFilter chain.
	//
	// The default implementation returns reader unchanged.
	InitReader(fieldName string, reader io.RuneReader) io.RuneReader
}

type container struct {
	value interface{}
}

type AnalyzerImpl struct {
	Spi           AnalyzerSPI
	reuseStrategy ReuseStrategy
	version       *util.SetOnce
	// Since Go doesn't have ThreadLocal alternatives, to share
	// Analyzer, one must Clone() the Analyzer for each Go routine. It
	// also means the performance may not be competitive compared to
	// Lucene Java Analyzer.
	storedValue *container
}

/*
Create a new Analyzer, reusing the same set of components per-thread
across calls to TokenStream(string, Reader).
*/
func NewAnalyzer() *AnalyzerImpl {
	return NewAnalyzerWithStrategy(GLOBAL_REUSE_STRATEGY)
}

func NewAnalyzerWithStrategy(reuseStrategy ReuseStrategy) *AnalyzerImpl {
	ans := &AnalyzerImpl{
		reuseStrategy: reuseStrategy,
		version:       util.NewSetOnce(),
		storedValue:   &container{nil},
	}
	ans.Spi = ans
	return ans
}

func (a *AnalyzerImpl) CreateComponents(fieldName string, reader io.RuneReader) *TokenStreamComponents {
	panic("must be inherited and implemented")
}

func (a *AnalyzerImpl) TokenStreamForReader(fieldName string, reader io.RuneReader) (TokenStream, error) {
	components := a.reuseStrategy.ReusableComponents(a, fieldName)
	r := a.InitReader(fieldName, reader)
	if components == nil {
		panic("not implemented yet")
	} else {
		if err := components.SetReader(r); err != nil {
			return nil, err
		}
	}
	return components.TokenStream(), nil
}

func (a *AnalyzerImpl) TokenStreamForString(fieldName, text string) (TokenStream, error) {
	components := a.reuseStrategy.ReusableComponents(a, fieldName)
	var strReader *ReusableStringReader
	if components == nil || components.reusableStringReader == nil {
		strReader = new(ReusableStringReader)
	} else {
		strReader = components.reusableStringReader
	}
	strReader.setValue(text)
	r := a.InitReader(fieldName, strReader)
	if components == nil {
		components = a.Spi.CreateComponents(fieldName, r)
		a.reuseStrategy.SetReusableComponents(a, fieldName, components)
	} else {
		err := components.SetReader(r)
		if err != nil {
			return nil, err
		}
	}
	components.reusableStringReader = strReader
	return components.TokenStream(), nil
}

func (a *AnalyzerImpl) InitReader(fieldName string, reader io.RuneReader) io.RuneReader {
	return reader
}

func (a *AnalyzerImpl) PositionIncrementGap(fieldName string) int {
	return 0
}

func (a *AnalyzerImpl) OffsetGap(fieldName string) int {
	return 1
}

func (a *AnalyzerImpl) SetVersion(v util.Version) {
	a.version.Set(v)
}

func (a *AnalyzerImpl) Version() util.Version {
	return a.version.Get().(util.Version)
}

type myTokenizer interface {
	SetReader(io.RuneReader) error
}

/*
This class encapsulates the outer components of a token stream. It
provides access to the source Tokenizer and the outer end (sink), an
instance of TokenFilter which also serves as the TokenStream returned
by Analyzer.tokenStream(string, Reader).
*/
type TokenStreamComponents struct {
	// Original source of tokens.
	source myTokenizer
	// Sink tokenStream, such as the outer tokenFilter decorating the
	// chain. This can be the source if there are no filters.
	sink TokenStream
	// Internal cache only used by Analyzer.TokenStreamForString().
	reusableStringReader *ReusableStringReader
	// Resets the encapculated components with the given reader. If the
	// components canno be reset, an error should be returned.
	SetReader func(io.RuneReader) error
}

func NewTokenStreamComponents(source myTokenizer, result TokenStream) *TokenStreamComponents {
	ans := &TokenStreamComponents{source: source, sink: result}
	ans.SetReader = func(reader io.RuneReader) error {
		return ans.source.SetReader(reader)
	}
	return ans
}

/* Returns the sink TokenStream */
func (cp *TokenStreamComponents) TokenStream() TokenStream {
	return cp.sink
}

// L329

// Strategy defining how TokenStreamComponents are reused per call to
// TokenStream(string, io.Reader)
type ReuseStrategy interface {
	// Gets the reusable TokenStreamComponents for the field with the
	// given name.
	ReusableComponents(*AnalyzerImpl, string) *TokenStreamComponents
	// Stores the given TokenStreamComponents as the reusable
	// components for the field with the given name.
	SetReusableComponents(*AnalyzerImpl, string, *TokenStreamComponents)
}

type ReuseStrategyImpl struct {
}

/* Returns the currently stored value */
func (rs *ReuseStrategyImpl) storedValue(a *AnalyzerImpl) interface{} {
	assert2(a.storedValue != nil, "this Analyzer is closed")
	return a.storedValue.value
}

/* Set the stored value. */
func (rs *ReuseStrategyImpl) setStoredValue(a *AnalyzerImpl, v interface{}) {
	assert2(a.storedValue != nil, "this Analyzer is closed")
	a.storedValue.value = v
}

func assert2(ok bool, msg string, args ...interface{}) {
	if !ok {
		panic(fmt.Sprintf(msg, args...))
	}
}

/* A predefined ReuseStrategy that reuses the same components for every field */
var GLOBAL_REUSE_STRATEGY = new(GlobalReuseStrategy)

type GlobalReuseStrategy struct {
	*ReuseStrategyImpl
}

func (rs *GlobalReuseStrategy) ReusableComponents(a *AnalyzerImpl, fieldName string) *TokenStreamComponents {
	if ans := rs.storedValue(a); ans != nil {
		return ans.(*TokenStreamComponents)
	}
	return nil
}

func (rs *GlobalReuseStrategy) SetReusableComponents(a *AnalyzerImpl, fieldName string, components *TokenStreamComponents) {
	rs.setStoredValue(a, components)
}

// L423
// A predefined ReuseStrategy that reuses components per-field by
// maintaining a Map of TokenStreamComponent per field name.
var PER_FIELD_REUSE_STRATEGY = &PerFieldReuseStrategy{}

// Implementation of ReuseStrategy that reuses components per-field by
// maintianing a Map of TokenStreamComponent per field name.
type PerFieldReuseStrategy struct {
}

func (rs *PerFieldReuseStrategy) ReusableComponents(a *AnalyzerImpl, fieldName string) *TokenStreamComponents {
	panic("not implemented yet")
}

func (rs *PerFieldReuseStrategy) SetReusableComponents(a *AnalyzerImpl, fieldName string, components *TokenStreamComponents) {
	panic("not implemneted yet")
}

// analysis/ReusableStringReader.java

/* Internal class to enale reuse of the string reader by Analyzer.TokenStreamForString() */
type ReusableStringReader struct {
	s *bytes.Buffer
}

func (r *ReusableStringReader) setValue(s string) {
	r.s = bytes.NewBufferString(s)
}

func (r *ReusableStringReader) Read(p []byte) (int, error) {
	return r.s.Read(p)
}

func (r *ReusableStringReader) ReadRune() (rune, int, error) {
	return r.s.ReadRune()
}

func (r *ReusableStringReader) Close() error {
	r.s = nil
	return nil
}
<|endoftext|>


<commit_before>/* Copyright (c) 2014, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import (
	"compress/zlib"
	"errors"
	"flag"
	"fmt"
	"html/template"
	"io"
	"log"
	"math/rand"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"
)

const (
	indexTmpl = "index.html"
	formTmpl  = "form.html"
	chars     = "abcdefghijklmnopqrstuvwxyz0123456789"

	// GET error messages
	invalidId     = "Invalid paste id."
	pasteNotFound = "Paste doesn't exist."
	unknownError  = "Something went terribly wrong."
	// POST error messages
	missingForm = "Paste could not be found inside the posted form."
)

var (
	siteUrl, listen, dataDir string
	lifeTime                 time.Duration
	maxSizeStr               string
	idSize                   int
	maxSize                  ByteSize

	validId       *regexp.Regexp
	regexByteSize = regexp.MustCompile(`^([\d\.]+)\s*([KM]?B|[BKM])$`)
	indexTemplate *template.Template
	formTemplate  *template.Template
	pasteInfos    = make(map[Id]PasteInfo)
	customRand    *rand.Rand
)

func init() {
	flag.StringVar(&siteUrl, "u", "http://localhost:8080", "URL of the site")
	flag.StringVar(&listen, "l", "localhost:8080", "Host and port to listen to")
	flag.StringVar(&dataDir, "d", "data", "Directory to store all the pastes in")
	flag.DurationVar(&lifeTime, "t", 12*time.Hour, "Lifetime of the pastes (units: s,m,h)")
	flag.StringVar(&maxSizeStr, "s", "1M", "Maximum size of POSTs in bytes (units: B,K,M)")
	flag.IntVar(&idSize, "i", 8, "Size of the paste ids (between 6 and 256)")
	validId = regexp.MustCompile("^[a-zA-Z0-9]{" + strconv.Itoa(idSize) + "}$")
	customRand = rand.New(rand.NewSource(time.Now().UnixNano()))
}

type PasteInfo struct {
	ModTime time.Time
}

type Id string

func IdFromPath(idPath string) (Id, error) {
	parts := strings.Split(idPath, string(filepath.Separator))
	if len(parts) != 3 {
		return "", errors.New("Found invalid number of directories at " + idPath)
	}
	rawId := parts[0] + parts[1] + parts[2]
	if !validId.MatchString(rawId) {
		return "", errors.New("Found invalid id " + rawId)
	}
	return Id(rawId), nil
}

func RandomId() Id {
	s := make([]byte, idSize)
	var offset int = 0
MainLoop:
	for {
		r := customRand.Int63()
		for i := 0; i < 8; i++ {
			randbyte := int(r&0xff) % len(chars)
			s[offset] = chars[randbyte]
			offset++
			if offset == idSize {
				break MainLoop
			}
			r >>= 8
		}
	}
	return Id(s)
}

func (id Id) String() string {
	return string(id)
}

func (id Id) Path() string {
	return path.Join(string(id[0:2]), string(id[2:4]), string(id[4:]))
}

func (id Id) EndLife() {
	err := os.Remove(id.Path())
	if err == nil {
		delete(pasteInfos, id)
		log.Printf("Removed paste: %s", id)
	} else {
		log.Printf("Could not end the life of %s: %s", id, err)
		id.EndLifeAfter(2 * time.Minute)
	}
}

func (id Id) EndLifeAfter(duration time.Duration) {
	timer := time.NewTimer(duration)
	go func() {
		<-timer.C
		id.EndLife()
	}()
}

type ByteSize int64

const (
	B ByteSize = 1 << (10 * iota)
	KB
	MB
)

func parseByteSize(str string) (ByteSize, error) {
	if !regexByteSize.MatchString(str) {
		return 0, errors.New("Could not parse size in bytes")
	}
	parts := regexByteSize.FindStringSubmatch(str)
	size, _ := strconv.ParseFloat(string(parts[1]), 64)

	switch string(parts[2]) {
	case "KB", "K":
		size *= float64(KB)
	case "MB", "M":
		size *= float64(MB)
	}
	return ByteSize(size), nil
}

func (b ByteSize) String() string {
	switch {
	case b >= MB:
		return fmt.Sprintf("%.2f MB", float64(b)/float64(MB))
	case b >= KB:
		return fmt.Sprintf("%.2f KB", float64(b)/float64(KB))
	}
	return fmt.Sprintf("%d B", b)
}

func handler(w http.ResponseWriter, r *http.Request) {
	var err error
	switch r.Method {
	case "GET":
		switch r.URL.Path {
		case "/":
			indexTemplate.Execute(w, siteUrl)
			return
		case "/form":
			formTemplate.Execute(w, siteUrl)
			return
		}
		rawId := r.URL.Path[1:]
		if !validId.MatchString(rawId) {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", invalidId)
			return
		}
		id := Id(strings.ToLower(rawId))
		_, e := pasteInfos[id]
		if !e {
			w.WriteHeader(http.StatusNotFound)
			fmt.Fprintf(w, "%s\n", pasteNotFound)
			return
		}
		pastePath := id.Path()
		pasteFile, err := os.Open(pastePath)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		defer pasteFile.Close()
		compReader, err := zlib.NewReader(pasteFile)
		if err != nil {
			log.Printf("Could not open a compression reader for %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		defer compReader.Close()
		io.Copy(w, compReader)

	case "POST":
		r.Body = http.MaxBytesReader(w, r.Body, int64(maxSize))
		var id Id
		var content string
		found := false
		for i := 0; i < 10; i++ {
			id = RandomId()
			if _, e := pasteInfos[id]; !e {
				found = true
				break
			}
		}
		if !found {
			log.Printf("Gave up trying to find an unused random id")
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		if err = r.ParseMultipartForm(int64(maxSize)); err != nil {
			log.Printf("Could not parse POST multipart form: %s", err)
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", err)
			return
		}
		if vs, found := r.Form["paste"]; found {
			content = vs[0]
		} else {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", missingForm)
			return
		}
		pastePath := id.Path()
		dir, _ := path.Split(pastePath)
		if err = os.MkdirAll(dir, 0700); err != nil {
			log.Printf("Could not create directories leading to %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		id.EndLifeAfter(lifeTime)
		pasteFile, err := os.OpenFile(pastePath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
		if err != nil {
			log.Printf("Could not create new paste pasteFile %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		defer pasteFile.Close()
		compWriter := zlib.NewWriter(pasteFile)
		defer compWriter.Close()
		b, err := io.WriteString(compWriter, content)
		if err != nil {
			log.Printf("Could not write compressed data into %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		writtenSize := ByteSize(b)
		pasteInfos[id] = PasteInfo{ModTime: time.Now()}
		log.Printf("Created a new paste: %s (%s)", id, writtenSize)
		fmt.Fprintf(w, "%s/%s\n", siteUrl, id)
	}
}

func walkFunc(filePath string, fileInfo os.FileInfo, err error) error {
	if err != nil {
		return err
	}
	if fileInfo.IsDir() {
		return nil
	}
	id, err := IdFromPath(filePath)
	if err != nil {
		return err
	}
	modTime := fileInfo.ModTime()
	deathTime := modTime.Add(lifeTime)
	now := time.Now()
	if deathTime.Before(now) {
		go id.EndLife()
		return nil
	}
	var lifeLeft time.Duration
	if deathTime.After(now.Add(lifeTime)) {
		lifeLeft = lifeTime
	} else {
		lifeLeft = deathTime.Sub(now)
	}
	log.Printf("Recovered paste %s has %s left", id, lifeLeft)
	pasteInfos[id] = PasteInfo{ModTime: modTime}
	id.EndLifeAfter(lifeLeft)
	return nil
}

func main() {
	var err error
	log.Printf("idSize   = %d", idSize)
	log.Printf("maxSize  = %s", maxSize)
	log.Printf("siteUrl  = %s", siteUrl)
	log.Printf("listen   = %s", listen)
	log.Printf("dataDir  = %s", dataDir)
	log.Printf("lifeTime = %s", lifeTime)
	flag.Parse()
	if idSize < 6 || idSize > 256 {
		log.Fatalf("Provided id size %d is not between 6 and 256", idSize)
	}
	if maxSize, err = parseByteSize(maxSizeStr); err != nil {
		log.Fatalf("Invalid max size '%s': %s", maxSizeStr, err)
	}
	if indexTemplate, err = template.ParseFiles(indexTmpl); err != nil {
		log.Fatalf("Could not load template %s: %s", indexTmpl, err)
	}
	if formTemplate, err = template.ParseFiles(formTmpl); err != nil {
		log.Fatalf("Could not load template %s: %s", formTmpl, err)
	}
	if err = os.MkdirAll(dataDir, 0700); err != nil {
		log.Fatalf("Could not create data directory %s: %s", dataDir, err)
	}
	if err = os.Chdir(dataDir); err != nil {
		log.Fatalf("Could not enter data directory %s: %s", dataDir, err)
	}
	if err = filepath.Walk(".", walkFunc); err != nil {
		log.Fatalf("Could not recover data directory %s: %s", dataDir, err)
	}
	http.HandleFunc("/", handler)
	log.Fatal(http.ListenAndServe(listen, nil))
}
<commit_msg>Support ETags (currently only one in If-None-Match)<commit_after>/* Copyright (c) 2014, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import (
	"compress/zlib"
	"errors"
	"flag"
	"fmt"
	"html/template"
	"io"
	"log"
	"math/rand"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"
)

const (
	indexTmpl = "index.html"
	formTmpl  = "form.html"
	chars     = "abcdefghijklmnopqrstuvwxyz0123456789"

	// GET error messages
	invalidId     = "Invalid paste id."
	pasteNotFound = "Paste doesn't exist."
	unknownError  = "Something went terribly wrong."
	// POST error messages
	missingForm = "Paste could not be found inside the posted form."
)

var (
	siteUrl, listen, dataDir string
	lifeTime                 time.Duration
	maxSizeStr               string
	idSize                   int
	maxSize                  ByteSize

	validId       *regexp.Regexp
	regexByteSize = regexp.MustCompile(`^([\d\.]+)\s*([KM]?B|[BKM])$`)
	indexTemplate *template.Template
	formTemplate  *template.Template
	pasteInfos    = make(map[Id]PasteInfo)
	customRand    *rand.Rand
)

func init() {
	flag.StringVar(&siteUrl, "u", "http://localhost:8080", "URL of the site")
	flag.StringVar(&listen, "l", "localhost:8080", "Host and port to listen to")
	flag.StringVar(&dataDir, "d", "data", "Directory to store all the pastes in")
	flag.DurationVar(&lifeTime, "t", 12*time.Hour, "Lifetime of the pastes (units: s,m,h)")
	flag.StringVar(&maxSizeStr, "s", "1M", "Maximum size of POSTs in bytes (units: B,K,M)")
	flag.IntVar(&idSize, "i", 8, "Size of the paste ids (between 6 and 256)")
	validId = regexp.MustCompile("^[a-zA-Z0-9]{" + strconv.Itoa(idSize) + "}$")
	customRand = rand.New(rand.NewSource(time.Now().UnixNano()))
}

type PasteInfo struct {
	ModTime time.Time
}

type Id string

func IdFromPath(idPath string) (Id, error) {
	parts := strings.Split(idPath, string(filepath.Separator))
	if len(parts) != 3 {
		return "", errors.New("Found invalid number of directories at " + idPath)
	}
	rawId := parts[0] + parts[1] + parts[2]
	if !validId.MatchString(rawId) {
		return "", errors.New("Found invalid id " + rawId)
	}
	return Id(rawId), nil
}

func RandomId() Id {
	s := make([]byte, idSize)
	var offset int = 0
MainLoop:
	for {
		r := customRand.Int63()
		for i := 0; i < 8; i++ {
			randbyte := int(r&0xff) % len(chars)
			s[offset] = chars[randbyte]
			offset++
			if offset == idSize {
				break MainLoop
			}
			r >>= 8
		}
	}
	return Id(s)
}

func (id Id) String() string {
	return string(id)
}

func (id Id) Path() string {
	return path.Join(string(id[0:2]), string(id[2:4]), string(id[4:]))
}

func (id Id) EndLife() {
	err := os.Remove(id.Path())
	if err == nil {
		delete(pasteInfos, id)
		log.Printf("Removed paste: %s", id)
	} else {
		log.Printf("Could not end the life of %s: %s", id, err)
		id.EndLifeAfter(2 * time.Minute)
	}
}

func (id Id) EndLifeAfter(duration time.Duration) {
	timer := time.NewTimer(duration)
	go func() {
		<-timer.C
		id.EndLife()
	}()
}

type ByteSize int64

const (
	B ByteSize = 1 << (10 * iota)
	KB
	MB
)

func parseByteSize(str string) (ByteSize, error) {
	if !regexByteSize.MatchString(str) {
		return 0, errors.New("Could not parse size in bytes")
	}
	parts := regexByteSize.FindStringSubmatch(str)
	size, _ := strconv.ParseFloat(string(parts[1]), 64)

	switch string(parts[2]) {
	case "KB", "K":
		size *= float64(KB)
	case "MB", "M":
		size *= float64(MB)
	}
	return ByteSize(size), nil
}

func (b ByteSize) String() string {
	switch {
	case b >= MB:
		return fmt.Sprintf("%.2f MB", float64(b)/float64(MB))
	case b >= KB:
		return fmt.Sprintf("%.2f KB", float64(b)/float64(KB))
	}
	return fmt.Sprintf("%d B", b)
}

func handler(w http.ResponseWriter, r *http.Request) {
	var err error
	switch r.Method {
	case "GET":
		switch r.URL.Path {
		case "/":
			indexTemplate.Execute(w, siteUrl)
			return
		case "/form":
			formTemplate.Execute(w, siteUrl)
			return
		}
		rawId := r.URL.Path[1:]
		if !validId.MatchString(rawId) {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", invalidId)
			return
		}
		id := Id(strings.ToLower(rawId))
		pasteInfo, e := pasteInfos[id]
		if !e {
			w.WriteHeader(http.StatusNotFound)
			fmt.Fprintf(w, "%s\n", pasteNotFound)
			return
		}
		etag := fmt.Sprintf("%d-%s", pasteInfo.ModTime.Unix(), id)
		if inm := r.Header.Get("If-None-Match"); inm != "" {
			if etag == inm || inm == "*" {
				w.WriteHeader(http.StatusNotModified)
				return
			}
		}
		pastePath := id.Path()
		pasteFile, err := os.Open(pastePath)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		defer pasteFile.Close()
		w.Header().Set("Etag", etag)
		compReader, err := zlib.NewReader(pasteFile)
		if err != nil {
			log.Printf("Could not open a compression reader for %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		defer compReader.Close()
		io.Copy(w, compReader)

	case "POST":
		r.Body = http.MaxBytesReader(w, r.Body, int64(maxSize))
		var id Id
		var content string
		found := false
		for i := 0; i < 10; i++ {
			id = RandomId()
			if _, e := pasteInfos[id]; !e {
				found = true
				break
			}
		}
		if !found {
			log.Printf("Gave up trying to find an unused random id")
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		if err = r.ParseMultipartForm(int64(maxSize)); err != nil {
			log.Printf("Could not parse POST multipart form: %s", err)
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", err)
			return
		}
		if vs, found := r.Form["paste"]; found {
			content = vs[0]
		} else {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "%s\n", missingForm)
			return
		}
		pastePath := id.Path()
		dir, _ := path.Split(pastePath)
		if err = os.MkdirAll(dir, 0700); err != nil {
			log.Printf("Could not create directories leading to %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		id.EndLifeAfter(lifeTime)
		pasteFile, err := os.OpenFile(pastePath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
		if err != nil {
			log.Printf("Could not create new paste pasteFile %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		defer pasteFile.Close()
		compWriter := zlib.NewWriter(pasteFile)
		defer compWriter.Close()
		b, err := io.WriteString(compWriter, content)
		if err != nil {
			log.Printf("Could not write compressed data into %s: %s", pastePath, err)
			w.WriteHeader(http.StatusInternalServerError)
			fmt.Fprintf(w, "%s\n", unknownError)
			return
		}
		writtenSize := ByteSize(b)
		pasteInfos[id] = PasteInfo{ModTime: time.Now()}
		log.Printf("Created a new paste: %s (%s)", id, writtenSize)
		fmt.Fprintf(w, "%s/%s\n", siteUrl, id)
	}
}

func walkFunc(filePath string, fileInfo os.FileInfo, err error) error {
	if err != nil {
		return err
	}
	if fileInfo.IsDir() {
		return nil
	}
	id, err := IdFromPath(filePath)
	if err != nil {
		return err
	}
	modTime := fileInfo.ModTime()
	deathTime := modTime.Add(lifeTime)
	now := time.Now()
	if deathTime.Before(now) {
		go id.EndLife()
		return nil
	}
	var lifeLeft time.Duration
	if deathTime.After(now.Add(lifeTime)) {
		lifeLeft = lifeTime
	} else {
		lifeLeft = deathTime.Sub(now)
	}
	log.Printf("Recovered paste %s has %s left", id, lifeLeft)
	pasteInfos[id] = PasteInfo{ModTime: modTime}
	id.EndLifeAfter(lifeLeft)
	return nil
}

func main() {
	var err error
	log.Printf("idSize   = %d", idSize)
	log.Printf("maxSize  = %s", maxSize)
	log.Printf("siteUrl  = %s", siteUrl)
	log.Printf("listen   = %s", listen)
	log.Printf("dataDir  = %s", dataDir)
	log.Printf("lifeTime = %s", lifeTime)
	flag.Parse()
	if idSize < 6 || idSize > 256 {
		log.Fatalf("Provided id size %d is not between 6 and 256", idSize)
	}
	if maxSize, err = parseByteSize(maxSizeStr); err != nil {
		log.Fatalf("Invalid max size '%s': %s", maxSizeStr, err)
	}
	if indexTemplate, err = template.ParseFiles(indexTmpl); err != nil {
		log.Fatalf("Could not load template %s: %s", indexTmpl, err)
	}
	if formTemplate, err = template.ParseFiles(formTmpl); err != nil {
		log.Fatalf("Could not load template %s: %s", formTmpl, err)
	}
	if err = os.MkdirAll(dataDir, 0700); err != nil {
		log.Fatalf("Could not create data directory %s: %s", dataDir, err)
	}
	if err = os.Chdir(dataDir); err != nil {
		log.Fatalf("Could not enter data directory %s: %s", dataDir, err)
	}
	if err = filepath.Walk(".", walkFunc); err != nil {
		log.Fatalf("Could not recover data directory %s: %s", dataDir, err)
	}
	http.HandleFunc("/", handler)
	log.Fatal(http.ListenAndServe(listen, nil))
}
<|endoftext|>


<commit_before>package serf

import (
	"github.com/hashicorp/serf/serf"

	"github.com/stretchr/testify/mock"
)

type MockMemberEventHandler struct {
	mock.Mock
}

func (m *MockMemberEventHandler) HandleMemberEvent(e serf.MemberEvent) {
	m.Called(e)
	return
}

type MockUserEventHandler struct {
	mock.Mock
}

func (m *MockUserEventHandler) HandleUserEvent(e serf.UserEvent) {
	m.Called(e)
	return
}

type MockReconciler struct {
	mock.Mock
}

func (m *MockReconciler) Reconcile(e serf.MemberEvent) {
	m.Called(e)
	return
}
<commit_msg>Add more mock objects<commit_after>package serf

import (
	"github.com/hashicorp/serf/serf"

	"github.com/stretchr/testify/mock"
)

// MockMemberEventHandler mocks MemberEvent handlers.
type MockMemberEventHandler struct {
	mock.Mock
}

// HandleMemberEvent processes member events.
func (m *MockMemberEventHandler) HandleMemberEvent(e serf.MemberEvent) {
	m.Called(e)
	return
}

// MockUserEventHandler mocks UserEvent handlers.
type MockUserEventHandler struct {
	mock.Mock
}

// HandleUserEvent processes UserEvents.
func (m *MockUserEventHandler) HandleUserEvent(e serf.UserEvent) {
	m.Called(e)
	return
}

// MockReconciler mocks a Reconciler.
type MockReconciler struct {
	mock.Mock
}

// Reconcile processes MemberEvents.
func (m *MockReconciler) Reconcile(e serf.MemberEvent) {
	m.Called(e)
	return
}

// MockQueryEventHandler mocks QueryEvent handlers.
type MockQueryEventHandler struct {
	mock.Mock
}

// HandleQueryEvent processes QueryEvents.
func (m *MockQueryEventHandler) HandleQueryEvent(e serf.Query) {
	m.Called(e)
	return
}

// MockEvent
type MockEvent struct {
	mock.Mock

	Type serf.EventType
	Name string
}

// EventType returns the EventType
func (m *MockEvent) EventType() serf.EventType {
	m.Called()
	return m.Type
}

// String returns the EventType name
func (m *MockEvent) String() string {
	return m.Name
}
<|endoftext|>


<commit_before>package webhooks

import (
	"time"
)

type PayloadBase struct {
	Sender     *User       `json:"sender"`
	Repository *Repository `json:"repository"`
}

type User struct {
	Login             string `json:"login"`
	Id                int64  `json:"id"`
	AvatarUrl         string `json:"avatar_url"`
	GravatarId        string `json:"gravatar_id"`
	Url               string `json:"url"`
	HtmlUrl           string `json:"html_url"`
	FollowersUrl      string `json:"followers_url"`
	FollowingUrl      string `json:"following_url"`
	GistsUrl          string `json:"gists_url"`
	StarredUrl        string `json:"starred_url"`
	SubscriptionsUrl  string `json:"subscriptions_url"`
	OrganizationsUrl  string `json:"organizations_url"`
	ReposUrl          string `json:"repos_url"`
	EventsUrl         string `json:"events_url"`
	ReceivedEventsUrl string `json:"received_events_url"`
	Type              string `json:"type"`
	SiteAdmin         bool   `json:"site_admin"`
}

type Repository struct {
	Id               int64     `json:"id"`
	Name             string    `json:"name"`
	FullName         string    `json:"full_name"`
	Owner            *User     `json:"owner"`
	Private          bool      `json:"private"`
	HtmlUrl          string    `json:"html_url"`
	Descriprion      string    `json:"description"`
	Fork             bool      `json:"fork"`
	Url              string    `json:"url"`
	ForksUrl         string    `json:"forks_url"`
	KeysUrl          string    `json:"keys_url"`
	CollaboratorsUrl string    `json:"collaborators_url"`
	TeamsUrl         string    `json:"teams_url"`
	HooksUrl         string    `json:"hooks_url"`
	IssueEventsUrl   string    `json:"issue_events_url"`
	EventsUrl        string    `json:"events_url"`
	AssigneesUrl     string    `json:"assignees_url"`
	BranchesUrl      string    `json:"branches_url"`
	TagsUrl          string    `json:"tags_url"`
	BlobsUrl         string    `json:"blobs_url"`
	GitTagsUrl       string    `json:"git_tags_url"`
	GitRefsUrl       string    `json:"git_refs_url"`
	TreesUrl         string    `json:"trees_url"`
	StatusesUrl      string    `json:"statuses_url"`
	LanguagesUrl     string    `json:"languages_url"`
	StargazersUrl    string    `json:"stargazers_url"`
	ContributorsUrl  string    `json:"contributors_url"`
	SubscribersUrl   string    `json:"subscribers_url"`
	SubscriptionUrl  string    `json:"subscription_url"`
	CommitsUrl       string    `json:"commits_url"`
	GitCommitsUrl    string    `json:"git_commits_url"`
	CommentsUrl      string    `json:"comments_url"`
	IssueCommentUrl  string    `json:"issue_comment_url"`
	ContentsUrl      string    `json:"contents_url"`
	CompareUrl       string    `json:"compare_url"`
	MergesUrl        string    `json:"merges_url"`
	ArchiveUrl       string    `json:"archive_url"`
	DownloadsUrl     string    `json:"downloads_url"`
	IssuesUrl        string    `json:"issues_url"`
	PullsUrl         string    `json:"pulls_url"`
	MilestonesUrl    string    `json:"milestones_url"`
	NotificationsUrl string    `json:"notifications_url"`
	LabelsUrl        string    `json:"labels_url"`
	ReleasesUrl      string    `json:"releases_url"`
	CreatedAt        time.Time `json:"created_at"`
	UpdatedAt        time.Time `json:"updated_at"`
	PushedAt         time.Time `json:"pushed_at"`
	GitUrl           string    `json:"git_url"`
	SshUrl           string    `json:"ssh_url"`
	CloneUrl         string    `json:"clone_url"`
	SvnUrl           string    `json:"svn_url"`
	Homepage         string    `json:"homepage"`
	Size             int       `json:"size"`
	StargazersCount  int       `json:"stargazers_count"`
	WatchersCount    int       `json:"watchers_count"`
	Language         string    `json:"language"`
	HasIssues        bool      `json:"has_issues"`
	HasDownloads     bool      `json:"has_downloads"`
	HasWiki          bool      `json:"has_wiki"`
	HasPages         bool      `json:"has_pages"`
	ForksCount       int       `json:"forks_count"`
	MirrorUrl        string    `json:"mirror_url"`
	OpenIssuesCount  int       `json:"open_issues_count"`
	Forks            int       `json:"forks"`
	OpenIssues       int       `json:"open_issues"`
	DefaultBranch    string    `json:"default_branch"`
}

type Comment struct {
	HtmlUrl   string    `json:"html_url"`
	Url       string    `json:"url"`
	Id        int64     `json:"id"`
	Body      string    `json:"body"`
	Path      string    `json:"path"`
	Position  int       `json:"position"`
	Line      int       `json:"line"`
	CommitId  string    `json:"commit_id"`
	User      *User     `json:"user"`
	CreatedAt time.Time `json:"created_at"`
	UpdatedAt time.Time `json:"updated_at"`
}

type Commit struct {
	Id        string     `json:"id"`
	Distinct  bool       `json:"distinct"`
	Message   string     `json:"message"`
	Timestamp time.Time  `json:"timestamp"`
	Url       string     `json:"url"`
	Author    *Committer `json:"author"`
	Committer *Committer `json:"committer"`
	Added     []string   `json:"added"`
	Removed   []string   `json:"removed"`
	Modified  []string   `json:"modified"`
}

type Hook struct {
	Id        int64     `json:"id"`
	Url       string    `json:"url"`
	TestUrl   string    `json:"test_url"`
	PingUrl   string    `json:"ping_url"`
	Name      string    `json:"name"`
	Events    []string  `json:"events"`
	Active    bool      `json:"active"`
	CreatedAt time.Time `json:"created_at"`
	UpdatedAt time.Time `json:"updated_at"`
	Config    struct {
		Url         string `json:"url"`
		ContentType string `json:"content_type"`
	} `json:"config"`
}

type Label struct {
	Url   string `json:"url"`
	Name  string `json:"name"`
	Color string `json:"color"`
}

type Issue struct {
	Url          string      `json:"url"`
	LabelsUrl    string      `json:"labels_url"`
	CommentsUrl  string      `json:"comments_url"`
	EventsUrl    string      `json:"events_url"`
	HtmlUrl      string      `json:"html_url"`
	Id           int64       `json:"id"`
	Number       int64       `json:"number"`
	Title        string      `json:"title"`
	User         *User       `json:"user"`
	Labels       []*Label    `json:"labels"`
	State        string      `json:"state"`
	Locked       bool        `json:"locked"`
	Assignee     *User       `json:"assignee"`
	Milestone    interface{} `json:"milestone"`
	CommentCount int64       `json:"comments"`
	CreatedAt    time.Time   `json:"created_at"`
	UpdatedAt    time.Time   `json:"updated_at"`
	ClosedAt     time.Time   `json:"closed_at"`
	Body         string      `json:"body"`
}

type PullRequest struct {
	Issue
	DiffUrl            string      `json:"diff_url"`
	PatchUrl           string      `json:"patch_url"`
	IssueUrl           string      `json:"issue_url"`
	MergedAt           time.Time   `json:"merged_at"`
	MergeCommitSha     string      `json:"merge_commit_sha"`
	CommitsUrl         string      `json:"commits_url"`
	ReviewCommentsUrl  string      `json:"review_comments_url"`
	ReviewCommentUrl   string      `json:"review_comment_url"`
	StatusesUrl        string      `json:"statuses_url"`
	Head               interface{} `json:"head"` // not sure what this type is quite yet
	Base               interface{} `json:"base"` // same type as Head
	Merged             bool        `json:"merged"`
	Mergeable          bool        `json:"mergable"`
	MergableState      string      `json:"mergable_state"`
	MergedBy           *User       `json:"merged_by"`
	ReviewCommentCount int64       `json:"review_comments"`
	CommitCount        int64       `json:"commits"`
	Additions          int64       `json:"additions"`
	Deletions          int64       `json:"deletions"`
	ChangedFiles       int64       `json:"changed_files"`
}

type Committer struct {
	Name     string `json:"name"`
	Email    string `json:"email"`
	Username string `json:"username"`
}

type CommitCommentEvent struct {
	PayloadBase
	Comment *Comment `json:"comment"`
}

type CreateEvent struct {
	PayloadBase
	RefType      string `json:"ref_type"`
	Ref          string `json:"ref"`
	MasterBranch string `json:"master_branch"`
	Description  string `json:"description"`
}

type DeleteEvent struct {
	PayloadBase
	RefType string `json:"ref_type"`
	Ref     string `json:"ref"`
}

type DeploymentEvent struct {
	PayloadBase
	Deployment struct {
		Url           string      `json:"url"`
		Id            int64       `json:"id"`
		Sha           string      `json:"sha"`
		Ref           string      `json:"ref"`
		Task          string      `json:"task"`
		Payload       interface{} `json:"payload"`
		Environment   string      `json:"environment"`
		Description   string      `json:"description"`
		Creator       *User       `json:"creator"`
		CreatedAt     time.Time   `json:"created_at"`
		UpdatedAt     time.Time   `json:"updated_at"`
		StatusesUrl   string      `json:"statuses_url"`
		RepositoryUrl string      `json:"repository_url"`
	} `json:"deployment"`
}

type IssueCommentEvent struct {
	PayloadBase
	Action  *Issue `json:"issue"`
	Comment string `json:"comment"`
}

type PingEvent struct {
	PayloadBase
	Zen    string `json:"zen"`
	HookId int64  `json:"hook_id"`
	Hook   *Hook  `json:"hook"`
}

type PullRequestEvent struct {
	PayloadBase
	Action      string       `json:"action"`
	Number      int64        `json:"number"`
	PullRequest *PullRequest `json:"pull_request"`
}

type PushEvent struct {
	PayloadBase
	Ref        string     `json:"ref"`
	Before     string     `json:"before"`
	After      string     `json:"after"`
	Created    bool       `json:"created"`
	Deleted    bool       `json:"deleted"`
	Forced     bool       `json:"forced"`
	BaseRef    string     `json:"base_ref"`
	Compare    string     `json:"compare"`
	Commits    []*Commit  `json:"commits"`
	HeadCommit *Commit    `json:"head_commit"`
	Pusher     *Committer `json:"pusher"`
}
<commit_msg>nulls suck to deserialize<commit_after>package webhooks

import (
	"time"
)

type PayloadBase struct {
	Sender     *User       `json:"sender"`
	Repository *Repository `json:"repository"`
}

type User struct {
	Login             string `json:"login"`
	Id                int64  `json:"id"`
	AvatarUrl         string `json:"avatar_url"`
	GravatarId        string `json:"gravatar_id"`
	Url               string `json:"url"`
	HtmlUrl           string `json:"html_url"`
	FollowersUrl      string `json:"followers_url"`
	FollowingUrl      string `json:"following_url"`
	GistsUrl          string `json:"gists_url"`
	StarredUrl        string `json:"starred_url"`
	SubscriptionsUrl  string `json:"subscriptions_url"`
	OrganizationsUrl  string `json:"organizations_url"`
	ReposUrl          string `json:"repos_url"`
	EventsUrl         string `json:"events_url"`
	ReceivedEventsUrl string `json:"received_events_url"`
	Type              string `json:"type"`
	SiteAdmin         bool   `json:"site_admin"`
}

type Repository struct {
	Id               int64     `json:"id"`
	Name             string    `json:"name"`
	FullName         string    `json:"full_name"`
	Owner            *User     `json:"owner"`
	Private          bool      `json:"private"`
	HtmlUrl          string    `json:"html_url"`
	Descriprion      string    `json:"description"`
	Fork             bool      `json:"fork"`
	Url              string    `json:"url"`
	ForksUrl         string    `json:"forks_url"`
	KeysUrl          string    `json:"keys_url"`
	CollaboratorsUrl string    `json:"collaborators_url"`
	TeamsUrl         string    `json:"teams_url"`
	HooksUrl         string    `json:"hooks_url"`
	IssueEventsUrl   string    `json:"issue_events_url"`
	EventsUrl        string    `json:"events_url"`
	AssigneesUrl     string    `json:"assignees_url"`
	BranchesUrl      string    `json:"branches_url"`
	TagsUrl          string    `json:"tags_url"`
	BlobsUrl         string    `json:"blobs_url"`
	GitTagsUrl       string    `json:"git_tags_url"`
	GitRefsUrl       string    `json:"git_refs_url"`
	TreesUrl         string    `json:"trees_url"`
	StatusesUrl      string    `json:"statuses_url"`
	LanguagesUrl     string    `json:"languages_url"`
	StargazersUrl    string    `json:"stargazers_url"`
	ContributorsUrl  string    `json:"contributors_url"`
	SubscribersUrl   string    `json:"subscribers_url"`
	SubscriptionUrl  string    `json:"subscription_url"`
	CommitsUrl       string    `json:"commits_url"`
	GitCommitsUrl    string    `json:"git_commits_url"`
	CommentsUrl      string    `json:"comments_url"`
	IssueCommentUrl  string    `json:"issue_comment_url"`
	ContentsUrl      string    `json:"contents_url"`
	CompareUrl       string    `json:"compare_url"`
	MergesUrl        string    `json:"merges_url"`
	ArchiveUrl       string    `json:"archive_url"`
	DownloadsUrl     string    `json:"downloads_url"`
	IssuesUrl        string    `json:"issues_url"`
	PullsUrl         string    `json:"pulls_url"`
	MilestonesUrl    string    `json:"milestones_url"`
	NotificationsUrl string    `json:"notifications_url"`
	LabelsUrl        string    `json:"labels_url"`
	ReleasesUrl      string    `json:"releases_url"`
	CreatedAt        time.Time `json:"created_at"`
	UpdatedAt        time.Time `json:"updated_at"`
	PushedAt         time.Time `json:"pushed_at"`
	GitUrl           string    `json:"git_url"`
	SshUrl           string    `json:"ssh_url"`
	CloneUrl         string    `json:"clone_url"`
	SvnUrl           string    `json:"svn_url"`
	Homepage         string    `json:"homepage"`
	Size             int       `json:"size"`
	StargazersCount  int       `json:"stargazers_count"`
	WatchersCount    int       `json:"watchers_count"`
	Language         string    `json:"language"`
	HasIssues        bool      `json:"has_issues"`
	HasDownloads     bool      `json:"has_downloads"`
	HasWiki          bool      `json:"has_wiki"`
	HasPages         bool      `json:"has_pages"`
	ForksCount       int       `json:"forks_count"`
	MirrorUrl        string    `json:"mirror_url"`
	OpenIssuesCount  int       `json:"open_issues_count"`
	Forks            int       `json:"forks"`
	OpenIssues       int       `json:"open_issues"`
	DefaultBranch    string    `json:"default_branch"`
}

type Comment struct {
	HtmlUrl   string    `json:"html_url"`
	Url       string    `json:"url"`
	Id        int64     `json:"id"`
	Body      string    `json:"body"`
	Path      string    `json:"path"`
	Position  int       `json:"position"`
	Line      int       `json:"line"`
	CommitId  string    `json:"commit_id"`
	User      *User     `json:"user"`
	CreatedAt time.Time `json:"created_at"`
	UpdatedAt time.Time `json:"updated_at"`
}

type Commit struct {
	Id        string     `json:"id"`
	Distinct  bool       `json:"distinct"`
	Message   string     `json:"message"`
	Timestamp time.Time  `json:"timestamp"`
	Url       string     `json:"url"`
	Author    *Committer `json:"author"`
	Committer *Committer `json:"committer"`
	Added     []string   `json:"added"`
	Removed   []string   `json:"removed"`
	Modified  []string   `json:"modified"`
}

type Hook struct {
	Id        int64     `json:"id"`
	Url       string    `json:"url"`
	TestUrl   string    `json:"test_url"`
	PingUrl   string    `json:"ping_url"`
	Name      string    `json:"name"`
	Events    []string  `json:"events"`
	Active    bool      `json:"active"`
	CreatedAt time.Time `json:"created_at"`
	UpdatedAt time.Time `json:"updated_at"`
	Config    struct {
		Url         string `json:"url"`
		ContentType string `json:"content_type"`
	} `json:"config"`
}

type Label struct {
	Url   string `json:"url"`
	Name  string `json:"name"`
	Color string `json:"color"`
}

type Issue struct {
	Url          string      `json:"url"`
	LabelsUrl    string      `json:"labels_url"`
	CommentsUrl  string      `json:"comments_url"`
	EventsUrl    string      `json:"events_url"`
	HtmlUrl      string      `json:"html_url"`
	Id           int64       `json:"id"`
	Number       int64       `json:"number"`
	Title        string      `json:"title"`
	User         *User       `json:"user"`
	Labels       []*Label    `json:"labels"`
	State        string      `json:"state"`
	Locked       bool        `json:"locked"`
	Assignee     *User       `json:"assignee"`
	Milestone    interface{} `json:"milestone"`
	CommentCount int64       `json:"comments"`
	CreatedAt    time.Time   `json:"created_at"`
	UpdatedAt    time.Time   `json:"updated_at"`
	ClosedAt     *time.Time  `json:"closed_at"`
	Body         string      `json:"body"`
}

type PullRequest struct {
	Issue
	DiffUrl            string      `json:"diff_url"`
	PatchUrl           string      `json:"patch_url"`
	IssueUrl           string      `json:"issue_url"`
	MergedAt           *time.Time  `json:"merged_at"`
	MergeCommitSha     string      `json:"merge_commit_sha"`
	CommitsUrl         string      `json:"commits_url"`
	ReviewCommentsUrl  string      `json:"review_comments_url"`
	ReviewCommentUrl   string      `json:"review_comment_url"`
	StatusesUrl        string      `json:"statuses_url"`
	Head               interface{} `json:"head"` // not sure what this type is quite yet
	Base               interface{} `json:"base"` // same type as Head
	Merged             bool        `json:"merged"`
	Mergeable          bool        `json:"mergable"`
	MergableState      string      `json:"mergable_state"`
	MergedBy           *User       `json:"merged_by"`
	ReviewCommentCount int64       `json:"review_comments"`
	CommitCount        int64       `json:"commits"`
	Additions          int64       `json:"additions"`
	Deletions          int64       `json:"deletions"`
	ChangedFiles       int64       `json:"changed_files"`
}

type Committer struct {
	Name     string `json:"name"`
	Email    string `json:"email"`
	Username string `json:"username"`
}

type CommitCommentEvent struct {
	PayloadBase
	Comment *Comment `json:"comment"`
}

type CreateEvent struct {
	PayloadBase
	RefType      string `json:"ref_type"`
	Ref          string `json:"ref"`
	MasterBranch string `json:"master_branch"`
	Description  string `json:"description"`
}

type DeleteEvent struct {
	PayloadBase
	RefType string `json:"ref_type"`
	Ref     string `json:"ref"`
}

type DeploymentEvent struct {
	PayloadBase
	Deployment struct {
		Url           string      `json:"url"`
		Id            int64       `json:"id"`
		Sha           string      `json:"sha"`
		Ref           string      `json:"ref"`
		Task          string      `json:"task"`
		Payload       interface{} `json:"payload"`
		Environment   string      `json:"environment"`
		Description   string      `json:"description"`
		Creator       *User       `json:"creator"`
		CreatedAt     time.Time   `json:"created_at"`
		UpdatedAt     time.Time   `json:"updated_at"`
		StatusesUrl   string      `json:"statuses_url"`
		RepositoryUrl string      `json:"repository_url"`
	} `json:"deployment"`
}

type IssueCommentEvent struct {
	PayloadBase
	Action  *Issue `json:"issue"`
	Comment string `json:"comment"`
}

type PingEvent struct {
	PayloadBase
	Zen    string `json:"zen"`
	HookId int64  `json:"hook_id"`
	Hook   *Hook  `json:"hook"`
}

type PullRequestEvent struct {
	PayloadBase
	Action      string       `json:"action"`
	Number      int64        `json:"number"`
	PullRequest *PullRequest `json:"pull_request"`
}

type PushEvent struct {
	PayloadBase
	Ref        string     `json:"ref"`
	Before     string     `json:"before"`
	After      string     `json:"after"`
	Created    bool       `json:"created"`
	Deleted    bool       `json:"deleted"`
	Forced     bool       `json:"forced"`
	BaseRef    string     `json:"base_ref"`
	Compare    string     `json:"compare"`
	Commits    []*Commit  `json:"commits"`
	HeadCommit *Commit    `json:"head_commit"`
	Pusher     *Committer `json:"pusher"`
}
<|endoftext|>


<commit_before>package paystack

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"time"

	"github.com/mitchellh/mapstructure"
)

const (
	// library version
	version = "0.1.0"

	// defaultHTTPTimeout is the default timeout on the http client
	defaultHTTPTimeout = 60 * time.Second

	// base URL for all Paystack API requests
	baseURL = "https://api.paystack.co"

	// User agent used when communicating with the Paystack API.
	// userAgent = "paystack-go/" + version
	userAgent = "Mozilla/5.0 (Unknown; Linux) AppleWebKit/538.1 (KHTML, like Gecko) Chrome/v1.0.0 Safari/538.1"
)

type service struct {
	client *Client
}

// Client manages communication with the Paystack API
type Client struct {
	common service      // Reuse a single struct instead of allocating one for each service on the heap.
	client *http.Client // HTTP client used to communicate with the API.

	// the API Key used to authenticate all Paystack API requests
	key string

	baseURL *url.URL

	logger Logger
	// Services supported by the Paystack API.
	// Miscellaneous actions are directly implemented on the Client object
	Customer     *CustomerService
	Transaction  *TransactionService
	SubAccount   *SubAccountService
	Plan         *PlanService
	Subscription *SubscriptionService
	Page         *PageService
	Settlement   *SettlementService
	Transfer     *TransferService
	Charge       *ChargeService
	Bank         *BankService
	BulkCharge   *BulkChargeService

	LoggingEnabled bool
	Log            Logger
}

// Logger interface for custom loggers
type Logger interface {
	Printf(format string, v ...interface{})
}

// Metadata is an key-value pairs added to Paystack API requests
type Metadata map[string]interface{}

// Response represents arbitrary response data
type Response map[string]interface{}

// RequestValues aliased to url.Values as a workaround
type RequestValues url.Values

// MarshalJSON to handle custom JSON decoding for RequestValues
func (v RequestValues) MarshalJSON() ([]byte, error) {
	m := make(map[string]interface{}, 3)
	for k, val := range v {
		m[k] = val[0]
	}
	return json.Marshal(m)
}

// ListMeta is pagination metadata for paginated responses from the Paystack API
type ListMeta struct {
	Total     int `json:"total"`
	Skipped   int `json:"skipped"`
	PerPage   int `json:"perPage"`
	Page      int `json:"page"`
	PageCount int `json:"pageCount"`
}

// NewClient creates a new Paystack API client with the given API key
// and HTTP client, allowing overriding of the HTTP client to use.
// This is useful if you're running in a Google AppEngine environment
// where the http.DefaultClient is not available.
func NewClient(key string, httpClient *http.Client) *Client {
	if httpClient == nil {
		httpClient = &http.Client{Timeout: defaultHTTPTimeout}
	}

	u, _ := url.Parse(baseURL)
	c := &Client{
		client:         httpClient,
		key:            key,
		baseURL:        u,
		LoggingEnabled: true,
		Log:            log.New(os.Stderr, "", log.LstdFlags),
	}

	c.common.client = c
	c.Customer = (*CustomerService)(&c.common)
	c.Transaction = (*TransactionService)(&c.common)
	c.SubAccount = (*SubAccountService)(&c.common)
	c.Plan = (*PlanService)(&c.common)
	c.Subscription = (*SubscriptionService)(&c.common)
	c.Page = (*PageService)(&c.common)
	c.Settlement = (*SettlementService)(&c.common)
	c.Transfer = (*TransferService)(&c.common)
	c.Charge = (*ChargeService)(&c.common)
	c.Bank = (*BankService)(&c.common)
	c.BulkCharge = (*BulkChargeService)(&c.common)

	return c
}

// Call actually does the HTTP request to Paystack API
func (c *Client) Call(method, path string, body, v interface{}) error {
	var buf io.ReadWriter
	if body != nil {
		buf = new(bytes.Buffer)
		err := json.NewEncoder(buf).Encode(body)
		if err != nil {
			return err
		}
	}
	u, _ := c.baseURL.Parse(path)
	req, err := http.NewRequest(method, u.String(), buf)

	if err != nil {
		if c.LoggingEnabled {
			c.Log.Printf("Cannot create Paystack request: %v\n", err)
		}
		return err
	}

	if body != nil {
		req.Header.Set("Content-Type", "application/json")
	}
	req.Header.Set("Authorization", "Bearer "+c.key)
	req.Header.Set("User-Agent", userAgent)

	if c.LoggingEnabled {
		c.Log.Printf("Requesting %v %v%v\n", req.Method, req.URL.Host, req.URL.Path)
		c.Log.Printf("POST request data %v\n", buf)
	}

	start := time.Now()

	resp, err := c.client.Do(req)
	if err != nil {
		return err
	}

	if c.LoggingEnabled {
		c.Log.Printf("Completed in %v\n", time.Since(start))
	}

	defer resp.Body.Close()
	return c.decodeResponse(resp, v)
}

// ResolveCardBIN docs https://developers.paystack.co/v1.0/reference#resolve-card-bin
func (c *Client) ResolveCardBIN(bin int) (Response, error) {
	u := fmt.Sprintf("/decision/bin/%d", bin)
	resp := Response{}
	err := c.Call("GET", u, nil, &resp)

	return resp, err
}

// CheckBalance docs https://developers.paystack.co/v1.0/reference#resolve-card-bin
func (c *Client) CheckBalance() (Response, error) {
	resp := Response{}
	err := c.Call("GET", "balance", nil, &resp)
	// check balance 'data' node is an array
	resp2 := resp["data"].([]interface{})[0].(map[string]interface{})
	return resp2, err
}

// GetSessionTimeout fetches payment session timeout
func (c *Client) GetSessionTimeout() (Response, error) {
	resp := Response{}
	err := c.Call("GET", "/integration/payment_session_timeout", nil, &resp)
	return resp, err
}

// UpdateSessionTimeout updates payment session timeout
func (c *Client) UpdateSessionTimeout(timeout int) (Response, error) {
	data := url.Values{}
	data.Add("timeout", string(timeout))
	resp := Response{}
	u := "/integration/payment_session_timeout"
	err := c.Call("PUT", u, data, &resp)
	return resp, err
}

// INTERNALS
func paginateURL(path string, count, offset int) string {
	return fmt.Sprintf("%s?perPage=%d&page=%d", path, count, offset)
}

func mapstruct(data interface{}, v interface{}) error {
	config := &mapstructure.DecoderConfig{
		Result:           v,
		TagName:          "json",
		WeaklyTypedInput: true,
	}
	decoder, err := mapstructure.NewDecoder(config)
	if err != nil {
		return err
	}
	err = decoder.Decode(data)
	return err
}

func mustGetTestKey() string {
	key := os.Getenv("PAYSTACK_KEY")

	if len(key) == 0 {
		panic("PAYSTACK_KEY environment variable is not set\n")
	}

	return key
}

// decodeResponse decodes the JSON response from the Twitter API.
// The actual response will be written to the `v` parameter
func (c *Client) decodeResponse(httpResp *http.Response, v interface{}) error {
	var resp Response
	respBody, err := ioutil.ReadAll(httpResp.Body)
	json.Unmarshal(respBody, &resp)

	if status, _ := resp["status"].(bool); !status || httpResp.StatusCode >= 400 {
		if c.LoggingEnabled {
			c.Log.Printf("Paystack error: %+v", err)
		}
		return newAPIError(httpResp)
	}

	if c.LoggingEnabled {
		c.Log.Printf("Paystack response: %v\n", resp)
	}

	if data, ok := resp["data"]; ok {
		switch t := resp["data"].(type) {
		case map[string]interface{}:
			return mapstruct(data, v)
		default:
			_ = t
			return mapstruct(resp, v)
		}
	}
	// if response data does not contain data key, map entire response to v
	return mapstruct(resp, v)
}
<commit_msg>Fix integer to string conversion<commit_after>package paystack

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"time"

	"github.com/mitchellh/mapstructure"
)

const (
	// library version
	version = "0.1.0"

	// defaultHTTPTimeout is the default timeout on the http client
	defaultHTTPTimeout = 60 * time.Second

	// base URL for all Paystack API requests
	baseURL = "https://api.paystack.co"

	// User agent used when communicating with the Paystack API.
	// userAgent = "paystack-go/" + version
	userAgent = "Mozilla/5.0 (Unknown; Linux) AppleWebKit/538.1 (KHTML, like Gecko) Chrome/v1.0.0 Safari/538.1"
)

type service struct {
	client *Client
}

// Client manages communication with the Paystack API
type Client struct {
	common service      // Reuse a single struct instead of allocating one for each service on the heap.
	client *http.Client // HTTP client used to communicate with the API.

	// the API Key used to authenticate all Paystack API requests
	key string

	baseURL *url.URL

	logger Logger
	// Services supported by the Paystack API.
	// Miscellaneous actions are directly implemented on the Client object
	Customer     *CustomerService
	Transaction  *TransactionService
	SubAccount   *SubAccountService
	Plan         *PlanService
	Subscription *SubscriptionService
	Page         *PageService
	Settlement   *SettlementService
	Transfer     *TransferService
	Charge       *ChargeService
	Bank         *BankService
	BulkCharge   *BulkChargeService

	LoggingEnabled bool
	Log            Logger
}

// Logger interface for custom loggers
type Logger interface {
	Printf(format string, v ...interface{})
}

// Metadata is an key-value pairs added to Paystack API requests
type Metadata map[string]interface{}

// Response represents arbitrary response data
type Response map[string]interface{}

// RequestValues aliased to url.Values as a workaround
type RequestValues url.Values

// MarshalJSON to handle custom JSON decoding for RequestValues
func (v RequestValues) MarshalJSON() ([]byte, error) {
	m := make(map[string]interface{}, 3)
	for k, val := range v {
		m[k] = val[0]
	}
	return json.Marshal(m)
}

// ListMeta is pagination metadata for paginated responses from the Paystack API
type ListMeta struct {
	Total     int `json:"total"`
	Skipped   int `json:"skipped"`
	PerPage   int `json:"perPage"`
	Page      int `json:"page"`
	PageCount int `json:"pageCount"`
}

// NewClient creates a new Paystack API client with the given API key
// and HTTP client, allowing overriding of the HTTP client to use.
// This is useful if you're running in a Google AppEngine environment
// where the http.DefaultClient is not available.
func NewClient(key string, httpClient *http.Client) *Client {
	if httpClient == nil {
		httpClient = &http.Client{Timeout: defaultHTTPTimeout}
	}

	u, _ := url.Parse(baseURL)
	c := &Client{
		client:         httpClient,
		key:            key,
		baseURL:        u,
		LoggingEnabled: true,
		Log:            log.New(os.Stderr, "", log.LstdFlags),
	}

	c.common.client = c
	c.Customer = (*CustomerService)(&c.common)
	c.Transaction = (*TransactionService)(&c.common)
	c.SubAccount = (*SubAccountService)(&c.common)
	c.Plan = (*PlanService)(&c.common)
	c.Subscription = (*SubscriptionService)(&c.common)
	c.Page = (*PageService)(&c.common)
	c.Settlement = (*SettlementService)(&c.common)
	c.Transfer = (*TransferService)(&c.common)
	c.Charge = (*ChargeService)(&c.common)
	c.Bank = (*BankService)(&c.common)
	c.BulkCharge = (*BulkChargeService)(&c.common)

	return c
}

// Call actually does the HTTP request to Paystack API
func (c *Client) Call(method, path string, body, v interface{}) error {
	var buf io.ReadWriter
	if body != nil {
		buf = new(bytes.Buffer)
		err := json.NewEncoder(buf).Encode(body)
		if err != nil {
			return err
		}
	}
	u, _ := c.baseURL.Parse(path)
	req, err := http.NewRequest(method, u.String(), buf)

	if err != nil {
		if c.LoggingEnabled {
			c.Log.Printf("Cannot create Paystack request: %v\n", err)
		}
		return err
	}

	if body != nil {
		req.Header.Set("Content-Type", "application/json")
	}
	req.Header.Set("Authorization", "Bearer "+c.key)
	req.Header.Set("User-Agent", userAgent)

	if c.LoggingEnabled {
		c.Log.Printf("Requesting %v %v%v\n", req.Method, req.URL.Host, req.URL.Path)
		c.Log.Printf("POST request data %v\n", buf)
	}

	start := time.Now()

	resp, err := c.client.Do(req)
	if err != nil {
		return err
	}

	if c.LoggingEnabled {
		c.Log.Printf("Completed in %v\n", time.Since(start))
	}

	defer resp.Body.Close()
	return c.decodeResponse(resp, v)
}

// ResolveCardBIN docs https://developers.paystack.co/v1.0/reference#resolve-card-bin
func (c *Client) ResolveCardBIN(bin int) (Response, error) {
	u := fmt.Sprintf("/decision/bin/%d", bin)
	resp := Response{}
	err := c.Call("GET", u, nil, &resp)

	return resp, err
}

// CheckBalance docs https://developers.paystack.co/v1.0/reference#resolve-card-bin
func (c *Client) CheckBalance() (Response, error) {
	resp := Response{}
	err := c.Call("GET", "balance", nil, &resp)
	// check balance 'data' node is an array
	resp2 := resp["data"].([]interface{})[0].(map[string]interface{})
	return resp2, err
}

// GetSessionTimeout fetches payment session timeout
func (c *Client) GetSessionTimeout() (Response, error) {
	resp := Response{}
	err := c.Call("GET", "/integration/payment_session_timeout", nil, &resp)
	return resp, err
}

// UpdateSessionTimeout updates payment session timeout
func (c *Client) UpdateSessionTimeout(timeout int) (Response, error) {
	data := url.Values{}
	data.Add("timeout", strconv.Itoa(timeout))
	resp := Response{}
	u := "/integration/payment_session_timeout"
	err := c.Call("PUT", u, data, &resp)
	return resp, err
}

// INTERNALS
func paginateURL(path string, count, offset int) string {
	return fmt.Sprintf("%s?perPage=%d&page=%d", path, count, offset)
}

func mapstruct(data interface{}, v interface{}) error {
	config := &mapstructure.DecoderConfig{
		Result:           v,
		TagName:          "json",
		WeaklyTypedInput: true,
	}
	decoder, err := mapstructure.NewDecoder(config)
	if err != nil {
		return err
	}
	err = decoder.Decode(data)
	return err
}

func mustGetTestKey() string {
	key := os.Getenv("PAYSTACK_KEY")

	if len(key) == 0 {
		panic("PAYSTACK_KEY environment variable is not set\n")
	}

	return key
}

// decodeResponse decodes the JSON response from the Twitter API.
// The actual response will be written to the `v` parameter
func (c *Client) decodeResponse(httpResp *http.Response, v interface{}) error {
	var resp Response
	respBody, err := ioutil.ReadAll(httpResp.Body)
	json.Unmarshal(respBody, &resp)

	if status, _ := resp["status"].(bool); !status || httpResp.StatusCode >= 400 {
		if c.LoggingEnabled {
			c.Log.Printf("Paystack error: %+v", err)
		}
		return newAPIError(httpResp)
	}

	if c.LoggingEnabled {
		c.Log.Printf("Paystack response: %v\n", resp)
	}

	if data, ok := resp["data"]; ok {
		switch t := resp["data"].(type) {
		case map[string]interface{}:
			return mapstruct(data, v)
		default:
			_ = t
			return mapstruct(resp, v)
		}
	}
	// if response data does not contain data key, map entire response to v
	return mapstruct(resp, v)
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/codegangsta/cli"
)

func exitOnError(err error) {
	log.SetFlags(0)
	if err != nil {
		log.Fatalln(err)
	}
}

//Parse table to copy to from given filename or passed flags
func parseTableName(c *cli.Context, filename string) string {
	tableName := c.GlobalString("table")
	if tableName == "" {
		if filename == "" {
			// if no filename is not set, we reading stdin
			filename = "stdin"
		}
		base := filepath.Base(filename)
		ext := filepath.Ext(filename)
		tableName = strings.TrimSuffix(base, ext)
	}
	return postgresify(tableName)
}

func getDataType(c *cli.Context) string {
	dataType := "json"
	if c.GlobalBool("jsonb") {
		dataType = "jsonb"
	}

	return dataType
}

func main() {
	app := cli.NewApp()
	app.Name = "pgfutter"
	app.Version = "1.1"
	app.Usage = "Import JSON and CSV into PostgreSQL the easy way"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "dbname, db",
			Value:  "postgres",
			Usage:  "database to connect to",
			EnvVar: "DB_NAME",
		},
		cli.StringFlag{
			Name:   "host",
			Value:  "localhost",
			Usage:  "host name",
			EnvVar: "DB_HOST",
		},
		cli.StringFlag{
			Name:   "port",
			Value:  "5432",
			Usage:  "port",
			EnvVar: "DB_PORT",
		},
		cli.StringFlag{
			Name:   "username, user",
			Value:  "postgres",
			Usage:  "username",
			EnvVar: "DB_USER",
		},
		cli.BoolFlag{
			Name:  "ssl",
			Usage: "require ssl mode",
		},
		cli.StringFlag{
			Name:   "pass, pw",
			Value:  "",
			Usage:  "password",
			EnvVar: "DB_PASS",
		},
		cli.StringFlag{
			Name:   "schema",
			Value:  "import",
			Usage:  "database schema",
			EnvVar: "DB_SCHEMA",
		},
		cli.StringFlag{
			Name:   "table",
			Usage:  "destination table",
			EnvVar: "DB_TABLE",
		},
		cli.BoolFlag{
			Name:  "jsonb",
			Usage: "use JSONB data type",
		},
		cli.BoolFlag{
			Name:  "ignore-errors",
			Usage: "halt transaction on inconsistencies",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:  "json",
			Usage: "Import newline-delimited JSON objects into database",
			Action: func(c *cli.Context) error {
				cli.CommandHelpTemplate = strings.Replace(cli.CommandHelpTemplate, "[arguments...]", "<json-file>", -1)

				filename := c.Args().First()

				ignoreErrors := c.GlobalBool("ignore-errors")
				schema := c.GlobalString("schema")
				tableName := parseTableName(c, filename)
				dataType := getDataType(c)

				connStr := parseConnStr(c)
				err := importJSON(filename, connStr, schema, tableName, ignoreErrors, dataType)
				return err
			},
		},
		{
			Name:  "csv",
			Usage: "Import CSV into database",
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "excel",
					Usage: "support problematic Excel 2008 and Excel 2011 csv line endings",
				},
				cli.BoolFlag{
					Name:  "skip-header",
					Usage: "skip header row",
				},
				cli.StringFlag{
					Name:  "fields",
					Usage: "comma separated field names if no header row",
				},
				cli.StringFlag{
					Name:  "delimiter, d",
					Value: ",",
					Usage: "field delimiter",
				},
				cli.BoolFlag{
					Name:  "skip-parse-delimiter",
					Usage: "skip parsing escape sequences in the given delimiter",
				},
			},
			Action: func(c *cli.Context) error {
				cli.CommandHelpTemplate = strings.Replace(cli.CommandHelpTemplate, "[arguments...]", "<csv-file>", -1)

				filename := c.Args().First()

				ignoreErrors := c.GlobalBool("ignore-errors")
				schema := c.GlobalString("schema")
				tableName := parseTableName(c, filename)

				skipHeader := c.Bool("skip-header")
				fields := c.String("fields")
				skipParseheader := c.Bool("skip-parse-delimiter")
				excel := c.Bool("excel")
				delimiter := parseDelimiter(c.String("delimiter"), skipParseheader)
				connStr := parseConnStr(c)
				err := importCSV(filename, connStr, schema, tableName, ignoreErrors, skipHeader, fields, delimiter, excel)
				return err
			},
		},
	}

	app.Run(os.Args)
}
<commit_msg>Set v1.2<commit_after>package main

import (
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/codegangsta/cli"
)

func exitOnError(err error) {
	log.SetFlags(0)
	if err != nil {
		log.Fatalln(err)
	}
}

//Parse table to copy to from given filename or passed flags
func parseTableName(c *cli.Context, filename string) string {
	tableName := c.GlobalString("table")
	if tableName == "" {
		if filename == "" {
			// if no filename is not set, we reading stdin
			filename = "stdin"
		}
		base := filepath.Base(filename)
		ext := filepath.Ext(filename)
		tableName = strings.TrimSuffix(base, ext)
	}
	return postgresify(tableName)
}

func getDataType(c *cli.Context) string {
	dataType := "json"
	if c.GlobalBool("jsonb") {
		dataType = "jsonb"
	}

	return dataType
}

func main() {
	app := cli.NewApp()
	app.Name = "pgfutter"
	app.Version = "1.2"
	app.Usage = "Import JSON and CSV into PostgreSQL the easy way"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "dbname, db",
			Value:  "postgres",
			Usage:  "database to connect to",
			EnvVar: "DB_NAME",
		},
		cli.StringFlag{
			Name:   "host",
			Value:  "localhost",
			Usage:  "host name",
			EnvVar: "DB_HOST",
		},
		cli.StringFlag{
			Name:   "port",
			Value:  "5432",
			Usage:  "port",
			EnvVar: "DB_PORT",
		},
		cli.StringFlag{
			Name:   "username, user",
			Value:  "postgres",
			Usage:  "username",
			EnvVar: "DB_USER",
		},
		cli.BoolFlag{
			Name:  "ssl",
			Usage: "require ssl mode",
		},
		cli.StringFlag{
			Name:   "pass, pw",
			Value:  "",
			Usage:  "password",
			EnvVar: "DB_PASS",
		},
		cli.StringFlag{
			Name:   "schema",
			Value:  "import",
			Usage:  "database schema",
			EnvVar: "DB_SCHEMA",
		},
		cli.StringFlag{
			Name:   "table",
			Usage:  "destination table",
			EnvVar: "DB_TABLE",
		},
		cli.BoolFlag{
			Name:  "jsonb",
			Usage: "use JSONB data type",
		},
		cli.BoolFlag{
			Name:  "ignore-errors",
			Usage: "halt transaction on inconsistencies",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:  "json",
			Usage: "Import newline-delimited JSON objects into database",
			Action: func(c *cli.Context) error {
				cli.CommandHelpTemplate = strings.Replace(cli.CommandHelpTemplate, "[arguments...]", "<json-file>", -1)

				filename := c.Args().First()

				ignoreErrors := c.GlobalBool("ignore-errors")
				schema := c.GlobalString("schema")
				tableName := parseTableName(c, filename)
				dataType := getDataType(c)

				connStr := parseConnStr(c)
				err := importJSON(filename, connStr, schema, tableName, ignoreErrors, dataType)
				return err
			},
		},
		{
			Name:  "csv",
			Usage: "Import CSV into database",
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "excel",
					Usage: "support problematic Excel 2008 and Excel 2011 csv line endings",
				},
				cli.BoolFlag{
					Name:  "skip-header",
					Usage: "skip header row",
				},
				cli.StringFlag{
					Name:  "fields",
					Usage: "comma separated field names if no header row",
				},
				cli.StringFlag{
					Name:  "delimiter, d",
					Value: ",",
					Usage: "field delimiter",
				},
				cli.BoolFlag{
					Name:  "skip-parse-delimiter",
					Usage: "skip parsing escape sequences in the given delimiter",
				},
			},
			Action: func(c *cli.Context) error {
				cli.CommandHelpTemplate = strings.Replace(cli.CommandHelpTemplate, "[arguments...]", "<csv-file>", -1)

				filename := c.Args().First()

				ignoreErrors := c.GlobalBool("ignore-errors")
				schema := c.GlobalString("schema")
				tableName := parseTableName(c, filename)

				skipHeader := c.Bool("skip-header")
				fields := c.String("fields")
				skipParseheader := c.Bool("skip-parse-delimiter")
				excel := c.Bool("excel")
				delimiter := parseDelimiter(c.String("delimiter"), skipParseheader)
				connStr := parseConnStr(c)
				err := importCSV(filename, connStr, schema, tableName, ignoreErrors, skipHeader, fields, delimiter, excel)
				return err
			},
		},
	}

	app.Run(os.Args)
}
<|endoftext|>