<commit_before>// Copyright 2013 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package handlers is a collection of handlers for use with Go's net/http package.
*/
package handlers

import (
	"fmt"
	"io"
	"net/http"
	"sort"
	"strings"
	"time"
)

// MethodHandler is an http.Handler that dispatches to a handler whose key in the MethodHandler's
// map matches the name of the HTTP request's method, eg: GET
//
// If the request's method is OPTIONS and OPTIONS is not a key in the map then the handler
// responds with a status of 200 and sets the Allow header to a comma-separated list of
// available methods.
//
// If the request's method doesn't match any of its keys the handler responds with
// a status of 406, Method not allowed and sets the Allow header to a comma-separated list
// of available methods.
type MethodHandler map[string]http.Handler

func (h MethodHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	if handler, ok := h[req.Method]; ok {
		handler.ServeHTTP(w, req)
	} else {
		allow := []string{}
		for k := range h {
			allow = append(allow, k)
		}
		sort.Strings(allow)
		w.Header().Set("Allow", strings.Join(allow, ", "))
		if req.Method == "OPTIONS" {
			w.WriteHeader(http.StatusOK)
		} else {
			http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
		}
	}
}

// loggingHandler is the http.Handler implementation for LoggingHandlerTo and its friends
type loggingHandler struct {
	writer  io.Writer
	handler http.Handler
}

// combinedLoggingHandler is the http.Handler implementation for LoggingHandlerTo and its friends
type combinedLoggingHandler struct {
	writer  io.Writer
	handler http.Handler
}

func (h loggingHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	t := time.Now()
	logger := responseLogger{w: w}
	h.handler.ServeHTTP(&logger, req)
	writeLog(h.writer, req, t, logger.status, logger.size)
}

func (h combinedLoggingHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	t := time.Now()
	logger := responseLogger{w: w}
	h.handler.ServeHTTP(&logger, req)
	writeCombinedLog(h.writer, req, t, logger.status, logger.size)
}

// responseLogger is wrapper of http.ResponseWriter that keeps track of its HTTP status
// code and body size
type responseLogger struct {
	w      http.ResponseWriter
	status int
	size   int
}

func (l *responseLogger) Header() http.Header {
	return l.w.Header()
}

func (l *responseLogger) Write(b []byte) (int, error) {
	if l.status == 0 {
		// The status will be StatusOK if WriteHeader has not been called yet
		l.status = http.StatusOK
	}
	size, err := l.w.Write(b)
	l.size += size
	return size, err
}

func (l *responseLogger) WriteHeader(s int) {
	l.w.WriteHeader(s)
	l.status = s
}

// buildCommonLogLine builds a log entry for req in Apache Common Log Format.
// ts is the timestamp with which the entry should be logged.
// status and size are used to provide the response HTTP status and size.
func buildCommonLogLine(req *http.Request, ts time.Time, status int, size int) string {
	username := "-"
	if req.URL.User != nil {
		if name := req.URL.User.Username(); name != "" {
			username = name
		}
	}

	return fmt.Sprintf("%s - %s [%s] \"%s %s %s\" %d %d",
		strings.Split(req.RemoteAddr, ":")[0],
		username,
		ts.Format("02/Jan/2006:15:04:05 -0700"),
		req.Method,
		req.URL.RequestURI(),
		req.Proto,
		status,
		size,
	)
}

// writeLog writes a log entry for req to w in Apache Common Log Format.
// ts is the timestamp with which the entry should be logged.
// status and size are used to provide the response HTTP status and size.
func writeLog(w io.Writer, req *http.Request, ts time.Time, status, size int) {
	line := buildCommonLogLine(req, ts, status, size) + "\n"
	fmt.Fprint(w, line)
}

// writeCombinedLog writes a log entry for req to w in Apache Combined Log Format.
// ts is the timestamp with which the entry should be logged.
// status and size are used to provide the response HTTP status and size.
func writeCombinedLog(w io.Writer, req *http.Request, ts time.Time, status, size int) {
	line := buildCommonLogLine(req, ts, status, size)
	combinedLine := fmt.Sprintf("%s \"%s\" \"%s\"\n", line, req.Referer(), req.UserAgent())
	fmt.Fprint(w, combinedLine)
}

// CombinedLoggingHandler return a http.Handler that wraps h and logs requests to out in
// Apache Combined Log Format.
//
// See http://httpd.apache.org/docs/2.2/logs.html#combined for a description of this format.
//
// LoggingHandler always sets the ident field of the log to -
func CombinedLoggingHandler(out io.Writer, h http.Handler) http.Handler {
	return combinedLoggingHandler{out, h}
}

// LoggingHandler return a http.Handler that wraps h and logs requests to out in
// Apache Common Log Format (CLF).
//
// See http://httpd.apache.org/docs/2.2/logs.html#common for a description of this format.
//
// LoggingHandler always sets the ident field of the log to -
func LoggingHandler(out io.Writer, h http.Handler) http.Handler {
	return loggingHandler{out, h}
}
<commit_msg>Correctly parse remote address<commit_after>// Copyright 2013 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package handlers is a collection of handlers for use with Go's net/http package.
*/
package handlers

import (
	"fmt"
	"io"
	"net"
	"net/http"
	"sort"
	"strings"
	"time"
)

// MethodHandler is an http.Handler that dispatches to a handler whose key in the MethodHandler's
// map matches the name of the HTTP request's method, eg: GET
//
// If the request's method is OPTIONS and OPTIONS is not a key in the map then the handler
// responds with a status of 200 and sets the Allow header to a comma-separated list of
// available methods.
//
// If the request's method doesn't match any of its keys the handler responds with
// a status of 406, Method not allowed and sets the Allow header to a comma-separated list
// of available methods.
type MethodHandler map[string]http.Handler

func (h MethodHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	if handler, ok := h[req.Method]; ok {
		handler.ServeHTTP(w, req)
	} else {
		allow := []string{}
		for k := range h {
			allow = append(allow, k)
		}
		sort.Strings(allow)
		w.Header().Set("Allow", strings.Join(allow, ", "))
		if req.Method == "OPTIONS" {
			w.WriteHeader(http.StatusOK)
		} else {
			http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
		}
	}
}

// loggingHandler is the http.Handler implementation for LoggingHandlerTo and its friends
type loggingHandler struct {
	writer  io.Writer
	handler http.Handler
}

// combinedLoggingHandler is the http.Handler implementation for LoggingHandlerTo and its friends
type combinedLoggingHandler struct {
	writer  io.Writer
	handler http.Handler
}

func (h loggingHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	t := time.Now()
	logger := responseLogger{w: w}
	h.handler.ServeHTTP(&logger, req)
	writeLog(h.writer, req, t, logger.status, logger.size)
}

func (h combinedLoggingHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	t := time.Now()
	logger := responseLogger{w: w}
	h.handler.ServeHTTP(&logger, req)
	writeCombinedLog(h.writer, req, t, logger.status, logger.size)
}

// responseLogger is wrapper of http.ResponseWriter that keeps track of its HTTP status
// code and body size
type responseLogger struct {
	w      http.ResponseWriter
	status int
	size   int
}

func (l *responseLogger) Header() http.Header {
	return l.w.Header()
}

func (l *responseLogger) Write(b []byte) (int, error) {
	if l.status == 0 {
		// The status will be StatusOK if WriteHeader has not been called yet
		l.status = http.StatusOK
	}
	size, err := l.w.Write(b)
	l.size += size
	return size, err
}

func (l *responseLogger) WriteHeader(s int) {
	l.w.WriteHeader(s)
	l.status = s
}

// buildCommonLogLine builds a log entry for req in Apache Common Log Format.
// ts is the timestamp with which the entry should be logged.
// status and size are used to provide the response HTTP status and size.
func buildCommonLogLine(req *http.Request, ts time.Time, status int, size int) string {
	username := "-"
	if req.URL.User != nil {
		if name := req.URL.User.Username(); name != "" {
			username = name
		}
	}

	host, _, err := net.SplitHostPort(req.RemoteAddr)

	if err != nil {
		host = "-"
	}

	return fmt.Sprintf("%s - %s [%s] \"%s %s %s\" %d %d",
		host,
		username,
		ts.Format("02/Jan/2006:15:04:05 -0700"),
		req.Method,
		req.URL.RequestURI(),
		req.Proto,
		status,
		size,
	)
}

// writeLog writes a log entry for req to w in Apache Common Log Format.
// ts is the timestamp with which the entry should be logged.
// status and size are used to provide the response HTTP status and size.
func writeLog(w io.Writer, req *http.Request, ts time.Time, status, size int) {
	line := buildCommonLogLine(req, ts, status, size) + "\n"
	fmt.Fprint(w, line)
}

// writeCombinedLog writes a log entry for req to w in Apache Combined Log Format.
// ts is the timestamp with which the entry should be logged.
// status and size are used to provide the response HTTP status and size.
func writeCombinedLog(w io.Writer, req *http.Request, ts time.Time, status, size int) {
	line := buildCommonLogLine(req, ts, status, size)
	combinedLine := fmt.Sprintf("%s \"%s\" \"%s\"\n", line, req.Referer(), req.UserAgent())
	fmt.Fprint(w, combinedLine)
}

// CombinedLoggingHandler return a http.Handler that wraps h and logs requests to out in
// Apache Combined Log Format.
//
// See http://httpd.apache.org/docs/2.2/logs.html#combined for a description of this format.
//
// LoggingHandler always sets the ident field of the log to -
func CombinedLoggingHandler(out io.Writer, h http.Handler) http.Handler {
	return combinedLoggingHandler{out, h}
}

// LoggingHandler return a http.Handler that wraps h and logs requests to out in
// Apache Common Log Format (CLF).
//
// See http://httpd.apache.org/docs/2.2/logs.html#common for a description of this format.
//
// LoggingHandler always sets the ident field of the log to -
func LoggingHandler(out io.Writer, h http.Handler) http.Handler {
	return loggingHandler{out, h}
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"net/http"

	"github.com/cloudykit/jet"
	"github.com/generationtux/brizo/database"
)

var views = jet.NewHTMLSet("./views")

// uiHandler for requests to Javascript app
func uiHandler(rw http.ResponseWriter, request *http.Request) {
	view, err := views.GetTemplate("index.jet")

	if err != nil {
		log.Println("Unexpected template err:", err.Error())
	}

	view.Execute(rw, nil, nil)
}

// healthzHandler for health check requests
func healthzHandler(rw http.ResponseWriter, request *http.Request) {
	db, err := database.Connect()
	defer db.Close()

	if err != nil {
		log.Println(err)
		rw.WriteHeader(500)
		rw.Write([]byte("error"))
		return
	}

	rw.Write([]byte("ok"))
}
<commit_msg>fix import<commit_after>package main

import (
	"log"
	"net/http"

	"github.com/CloudyKit/jet"
	"github.com/generationtux/brizo/database"
)

var views = jet.NewHTMLSet("./views")

// uiHandler for requests to Javascript app
func uiHandler(rw http.ResponseWriter, request *http.Request) {
	view, err := views.GetTemplate("index.jet")

	if err != nil {
		log.Println("Unexpected template err:", err.Error())
	}

	view.Execute(rw, nil, nil)
}

// healthzHandler for health check requests
func healthzHandler(rw http.ResponseWriter, request *http.Request) {
	db, err := database.Connect()
	defer db.Close()

	if err != nil {
		log.Println(err)
		rw.WriteHeader(500)
		rw.Write([]byte("error"))
		return
	}

	rw.Write([]byte("ok"))
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"fmt"
	"log"
	"math/rand"
	"net/http"
	"strings"
	"time"

	"github.com/mozilla-services/go-bouncer/bouncer"
)

const DefaultLang = "en-US"
const DefaultOS = "win"

// BouncerHandler is the primary handler for this application
type BouncerHandler struct {
	db *bouncer.DB

	CacheTime time.Duration
}

func randomMirror(mirrors []bouncer.MirrorsResult) *bouncer.MirrorsResult {
	totalRatings := 0
	for _, m := range mirrors {
		totalRatings += m.Rating
	}
	for _, m := range mirrors {
		// Intn(x) returns from [0,x) and we need [1,x], so adding 1
		rand := rand.Intn(totalRatings) + 1
		if rand <= m.Rating {
			return &m
		}
		totalRatings -= m.Rating
	}

	// This shouldn't happen
	if len(mirrors) == 0 {
		return nil
	}
	return &mirrors[0]
}

// URL returns the final redirect URL given a lang, os and product
// if the string is == "", no mirror or location was found
func (b *BouncerHandler) URL(lang, os, product string) (string, error) {
	product, err := b.db.AliasFor(product)
	if err != nil {
		return "", err
	}

	osID, err := b.db.OSID(os)
	switch {
	case err == sql.ErrNoRows:
		return "", nil
	case err != nil:
		return "", err
	}

	productID, sslOnly, err := b.db.ProductForLanguage(product, lang)
	switch {
	case err == sql.ErrNoRows:
		return "", nil
	case err != nil:
		return "", err
	}

	locationID, locationPath, err := b.db.Location(productID, osID)
	switch {
	case err == sql.ErrNoRows:
		return "", nil
	case err != nil:
		return "", err
	}

	mirrors, err := b.db.Mirrors(sslOnly, lang, locationID, true)
	if err != nil {
		return "", err
	}

	if len(mirrors) == 0 {
		// try again, looking for unhealthy mirrors
		mirrors, err = b.db.Mirrors(sslOnly, lang, locationID, false)
		if err != nil {
			return "", err
		}
	}

	if len(mirrors) == 0 {
		return "", nil
	}

	mirror := randomMirror(mirrors)
	if mirror == nil {
		return "", nil
	}

	locationPath = strings.Replace(locationPath, ":lang", lang, -1)

	return mirror.BaseURL + locationPath, nil
}

func (b *BouncerHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	queryVals := req.URL.Query()

	os := queryVals.Get("os")
	product := queryVals.Get("product")
	lang := queryVals.Get("lang")

	if product == "" {
		http.Redirect(w, req, "http://www.mozilla.org/", 302)
		return
	}
	if os == "" {
		os = DefaultOS
	}
	if lang == "" {
		lang = DefaultLang
	}

	product = strings.TrimSpace(strings.ToLower(product))
	os = strings.TrimSpace(strings.ToLower(os))

	url, err := b.URL(lang, os, product)
	if err != nil {
		http.Error(w, "Internal Server Error.", http.StatusInternalServerError)
		log.Println(err)
		return
	}
	if url == "" {
		http.NotFound(w, req)
		return
	}

	if b.CacheTime.Seconds() > 0 {
		w.Header().Set("Cache-Control", fmt.Sprintf("max-age=%d", b.CacheTime.Seconds()))
	}

	http.Redirect(w, req, url, 302)
}
<commit_msg>fix Cache-Control header<commit_after>package main

import (
	"database/sql"
	"fmt"
	"log"
	"math/rand"
	"net/http"
	"strings"
	"time"

	"github.com/mozilla-services/go-bouncer/bouncer"
)

const DefaultLang = "en-US"
const DefaultOS = "win"

// BouncerHandler is the primary handler for this application
type BouncerHandler struct {
	db *bouncer.DB

	CacheTime time.Duration
}

func randomMirror(mirrors []bouncer.MirrorsResult) *bouncer.MirrorsResult {
	totalRatings := 0
	for _, m := range mirrors {
		totalRatings += m.Rating
	}
	for _, m := range mirrors {
		// Intn(x) returns from [0,x) and we need [1,x], so adding 1
		rand := rand.Intn(totalRatings) + 1
		if rand <= m.Rating {
			return &m
		}
		totalRatings -= m.Rating
	}

	// This shouldn't happen
	if len(mirrors) == 0 {
		return nil
	}
	return &mirrors[0]
}

// URL returns the final redirect URL given a lang, os and product
// if the string is == "", no mirror or location was found
func (b *BouncerHandler) URL(lang, os, product string) (string, error) {
	product, err := b.db.AliasFor(product)
	if err != nil {
		return "", err
	}

	osID, err := b.db.OSID(os)
	switch {
	case err == sql.ErrNoRows:
		return "", nil
	case err != nil:
		return "", err
	}

	productID, sslOnly, err := b.db.ProductForLanguage(product, lang)
	switch {
	case err == sql.ErrNoRows:
		return "", nil
	case err != nil:
		return "", err
	}

	locationID, locationPath, err := b.db.Location(productID, osID)
	switch {
	case err == sql.ErrNoRows:
		return "", nil
	case err != nil:
		return "", err
	}

	mirrors, err := b.db.Mirrors(sslOnly, lang, locationID, true)
	if err != nil {
		return "", err
	}

	if len(mirrors) == 0 {
		// try again, looking for unhealthy mirrors
		mirrors, err = b.db.Mirrors(sslOnly, lang, locationID, false)
		if err != nil {
			return "", err
		}
	}

	if len(mirrors) == 0 {
		return "", nil
	}

	mirror := randomMirror(mirrors)
	if mirror == nil {
		return "", nil
	}

	locationPath = strings.Replace(locationPath, ":lang", lang, -1)

	return mirror.BaseURL + locationPath, nil
}

func (b *BouncerHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	queryVals := req.URL.Query()

	os := queryVals.Get("os")
	product := queryVals.Get("product")
	lang := queryVals.Get("lang")

	if product == "" {
		http.Redirect(w, req, "http://www.mozilla.org/", 302)
		return
	}
	if os == "" {
		os = DefaultOS
	}
	if lang == "" {
		lang = DefaultLang
	}

	product = strings.TrimSpace(strings.ToLower(product))
	os = strings.TrimSpace(strings.ToLower(os))

	url, err := b.URL(lang, os, product)
	if err != nil {
		http.Error(w, "Internal Server Error.", http.StatusInternalServerError)
		log.Println(err)
		return
	}
	if url == "" {
		http.NotFound(w, req)
		return
	}

	if b.CacheTime > 0 {
		w.Header().Set("Cache-Control", fmt.Sprintf("max-age=%d", b.CacheTime/time.Second))
	}

	http.Redirect(w, req, url, 302)
}
<|endoftext|>


<commit_before>package goinsta

import (
	"encoding/json"
	"fmt"
)

type Hashtag struct {
	inst *Instagram
	err  error

	Name string `json:"name"`

	Sections []struct {
		LayoutType    string `json:"layout_type"`
		LayoutContent struct {
			// F*ck you instagram.
			// Why you do this f*cking horribly structure?!?
			// Media []Media IS EASY. CHECK IT!
			Medias []struct {
				Item Item `json:"media"`
			} `json:"medias"`
		} `json:"layout_content"`
		FeedType        string `json:"feed_type"`
		ExploreItemInfo struct {
			NumColumns      int  `json:"num_columns"`
			TotalNumColumns int  `json:"total_num_columns"`
			AspectRatio     int  `json:"aspect_ratio"`
			Autoplay        bool `json:"autoplay"`
		} `json:"explore_item_info"`
	} `json:"sections"`
	MediaCount          int     `json:"media_count"`
	ID                  int64   `json:"id"`
	MoreAvailable       bool    `json:"more_available"`
	NextID              string  `json:"next_max_id"`
	NextPage            int     `json:"next_page"`
	NextMediaIds        []int64 `json:"next_media_ids"`
	AutoLoadMoreEnabled bool    `json:"auto_load_more_enabled"`
	Status              string  `json:"status"`
}

// NewHashtag returns initialised hashtag structure
func (inst *Instagram) NewHashtag() *Hashtag {
	return &Hashtag{
		inst: inst,
	}
}

// Sync updates Hashtag information preparing it to Next call.
func (h *Hashtag) Sync() error {
	insta := h.inst

	body, err := insta.sendSimpleRequest(urlTagSync, h.Name)
	if err == nil {
		var resp struct {
			Name       string `json:"name"`
			ID         int64  `json:"id"`
			MediaCount int    `json:"media_count"`
		}
		err = json.Unmarshal(body, &resp)
		if err == nil {
			h.Name = resp.Name
			h.ID = resp.ID
			h.MediaCount = resp.MediaCount
		}
	}
	return err
}

// Next paginates over hashtag pages (xd).
func (h *Hashtag) Next() bool {
	if h.err != nil {
		return false
	}
	insta := h.inst
	name := h.Name
	body, err := insta.sendRequest(
		&reqOptions{
			Query: map[string]string{
				"max_id":     h.NextID,
				"rank_token": insta.rankToken,
				"page":       fmt.Sprintf("%d", h.NextPage),
			},
			Endpoint: fmt.Sprintf(urlTagContent, name),
			IsPost:   false,
		},
	)
	if err == nil {
		ht := Hashtag{}
		err = json.Unmarshal(body, &ht)
		if err == nil {
			*h = ht
			h.inst = insta
			h.Name = name
			if !h.MoreAvailable {
				h.err = ErrNoMore
			}
			return true
		}
	}
	h.err = err
	return false
}

func (h *Hashtag) Error() error {
	return h.err
}

// TODO: func (h *Hashtag) Stories()
<commit_msg>Added Name parameter to NewHashtag<commit_after>package goinsta

import (
	"encoding/json"
	"fmt"
)

type Hashtag struct {
	inst *Instagram
	err  error

	Name string `json:"name"`

	Sections []struct {
		LayoutType    string `json:"layout_type"`
		LayoutContent struct {
			// F*ck you instagram.
			// Why you do this f*cking horribly structure?!?
			// Media []Media IS EASY. CHECK IT!
			Medias []struct {
				Item Item `json:"media"`
			} `json:"medias"`
		} `json:"layout_content"`
		FeedType        string `json:"feed_type"`
		ExploreItemInfo struct {
			NumColumns      int  `json:"num_columns"`
			TotalNumColumns int  `json:"total_num_columns"`
			AspectRatio     int  `json:"aspect_ratio"`
			Autoplay        bool `json:"autoplay"`
		} `json:"explore_item_info"`
	} `json:"sections"`
	MediaCount          int     `json:"media_count"`
	ID                  int64   `json:"id"`
	MoreAvailable       bool    `json:"more_available"`
	NextID              string  `json:"next_max_id"`
	NextPage            int     `json:"next_page"`
	NextMediaIds        []int64 `json:"next_media_ids"`
	AutoLoadMoreEnabled bool    `json:"auto_load_more_enabled"`
	Status              string  `json:"status"`
}

// NewHashtag returns initialised hashtag structure
// Name parameter is hashtag name
func (inst *Instagram) NewHashtag(name string) *Hashtag {
	return &Hashtag{
		inst: inst,
		Name: name,
	}
}

// Sync updates Hashtag information preparing it to Next call.
func (h *Hashtag) Sync() error {
	insta := h.inst

	body, err := insta.sendSimpleRequest(urlTagSync, h.Name)
	if err == nil {
		var resp struct {
			Name       string `json:"name"`
			ID         int64  `json:"id"`
			MediaCount int    `json:"media_count"`
		}
		err = json.Unmarshal(body, &resp)
		if err == nil {
			h.Name = resp.Name
			h.ID = resp.ID
			h.MediaCount = resp.MediaCount
		}
	}
	return err
}

// Next paginates over hashtag pages (xd).
func (h *Hashtag) Next() bool {
	if h.err != nil {
		return false
	}
	insta := h.inst
	name := h.Name
	body, err := insta.sendRequest(
		&reqOptions{
			Query: map[string]string{
				"max_id":     h.NextID,
				"rank_token": insta.rankToken,
				"page":       fmt.Sprintf("%d", h.NextPage),
			},
			Endpoint: fmt.Sprintf(urlTagContent, name),
			IsPost:   false,
		},
	)
	if err == nil {
		ht := Hashtag{}
		err = json.Unmarshal(body, &ht)
		if err == nil {
			*h = ht
			h.inst = insta
			h.Name = name
			if !h.MoreAvailable {
				h.err = ErrNoMore
			}
			return true
		}
	}
	h.err = err
	return false
}

func (h *Hashtag) Error() error {
	return h.err
}

// TODO: func (h *Hashtag) Stories()
<|endoftext|>


<commit_before>// Copyright 2016 The rkt Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build host coreos src kvm

package main

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net"
	"os"
	"path/filepath"
	"testing"
	"time"

	sd_dbus "github.com/coreos/go-systemd/dbus"
	sd_util "github.com/coreos/go-systemd/util"
	"github.com/coreos/rkt/tests/testutils"
	"github.com/vishvananda/netlink"
)

func TestSocketProxyd(t *testing.T) {
	if !sd_util.IsRunningSystemd() {
		t.Skip("Systemd is not running on the host.")
	}

	socketProxydPath := "/lib/systemd/systemd-socket-proxyd"
	if _, err := os.Stat(socketProxydPath); os.IsNotExist(err) {
		t.Skip("systemd-socket-proxyd is not installed.")
	}

	ctx := testutils.NewRktRunCtx()
	defer ctx.Cleanup()

	iface, _, err := testutils.GetNonLoIfaceWithAddrs(netlink.FAMILY_V4)
	if err != nil {
		t.Fatalf("Error while getting non-lo host interface: %v\n", err)
	}
	if iface.Name == "" {
		t.Skipf("Cannot run test without non-lo host interface")
	}

	nt := networkTemplateT{
		Name:   "ptp0",
		Type:   "ptp",
		IpMasq: true,
		Master: iface.Name,
		Ipam: &ipamTemplateT{
			Type:   "host-local",
			Subnet: "192.168.0.0/24",
			Routes: []map[string]string{
				{"dst": "0.0.0.0/0"},
			},
		},
	}

	netDir := prepareTestNet(t, ctx, nt)
	defer os.RemoveAll(netDir)

	port, err := randomFreePort(t)
	if err != nil {
		t.Fatal(err)
	}

	echoImage := patchTestACI("rkt-inspect-echo.aci",
		"--exec=/echo-socket-activated",
		"--ports=test-port,protocol=tcp,port=80,socketActivated=true")
	defer os.Remove(echoImage)

	conn, err := sd_dbus.New()
	if err != nil {
		t.Fatal(err)
	}

	rktTestingEchoService := `
	[Unit]
	Description=Socket-activated echo server

	[Service]
	ExecStart=%s
	KillMode=process
	`

	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	rnd := r.Int()

	// Write unit files directly to runtime system units directory
	// (/run/systemd/system) to avoid calling LinkUnitFiles - it is buggy in
	// systemd v219 as it does not work with absolute paths.
	unitsDir := "/run/systemd/system"
	containerIP := "192.168.0.101"

	cmd := fmt.Sprintf("%s --insecure-options=image --debug run --net=%s:IP=%s --port=test-port:%d --mds-register=false %s",
		ctx.Cmd(), nt.Name, containerIP, port, echoImage)

	serviceContent := fmt.Sprintf(rktTestingEchoService, cmd)
	serviceTargetBase := fmt.Sprintf("rkt-testing-socket-activation-%d.service", rnd)
	serviceTarget := filepath.Join(unitsDir, serviceTargetBase)

	if err := ioutil.WriteFile(serviceTarget, []byte(serviceContent), 0666); err != nil {
		t.Fatal(err)
	}
	defer os.Remove(serviceTarget)

	rktTestingEchoSocket := `
	[Unit]
	Description=Socket-activated netcat server socket

	[Socket]
	ListenStream=%d

	[Install]
	WantedBy=sockets.target
	`

	socketContent := fmt.Sprintf(rktTestingEchoSocket, port)
	socketTargetBase := fmt.Sprintf("proxy-to-rkt-testing-socket-activation-%d.socket", rnd)
	socketTarget := filepath.Join(unitsDir, socketTargetBase)

	if err := ioutil.WriteFile(socketTarget, []byte(socketContent), 0666); err != nil {
		t.Fatal(err)
	}
	defer os.Remove(socketTarget)

	proxyToRktTestingEchoService := `
	[Unit]
	Requires=%s
	After=%s

	[Service]
	ExecStart=%s %s:%d
	`

	proxyContent := fmt.Sprintf(proxyToRktTestingEchoService, serviceTargetBase, serviceTargetBase,
		socketProxydPath, containerIP, port)
	proxyContentBase := fmt.Sprintf("proxy-to-rkt-testing-socket-activation-%d.service", rnd)
	proxyTarget := filepath.Join(unitsDir, proxyContentBase)

	if err := ioutil.WriteFile(proxyTarget, []byte(proxyContent), 0666); err != nil {
		t.Fatal(err)
	}
	defer os.Remove(proxyTarget)

	reschan := make(chan string)
	doJob := func() {
		job := <-reschan
		if job != "done" {
			t.Fatal("Job is not done:", job)
		}
	}

	if _, err := conn.StartUnit(socketTargetBase, "replace", reschan); err != nil {
		t.Fatal(err)
	}
	doJob()

	defer func() {
		if _, err := conn.StopUnit(socketTargetBase, "replace", reschan); err != nil {
			t.Fatal(err)
		}
		doJob()

		if _, err := conn.StopUnit(serviceTargetBase, "replace", reschan); err != nil {
			t.Fatal(err)
		}
		doJob()

		if _, err := conn.StopUnit(proxyContentBase, "replace", reschan); err != nil {
			t.Fatal(err)
		}
		doJob()
	}()

	expected := "HELO\n"
	sockConn, err := net.Dial("tcp", fmt.Sprintf("localhost:%d", port))
	if err != nil {
		t.Fatal(err)
	}

	if _, err := fmt.Fprintf(sockConn, expected); err != nil {
		t.Fatal(err)
	}

	answer, err := bufio.NewReader(sockConn).ReadString('\n')
	if err != nil {
		t.Fatal(err)
	}

	if answer != expected {
		t.Fatalf("Expected %q, Got %q", expected, answer)
	}

	return
}
<commit_msg>functional tests: skip TestSocketProxyd<commit_after>// Copyright 2016 The rkt Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build host coreos src kvm

package main

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net"
	"os"
	"path/filepath"
	"testing"
	"time"

	sd_dbus "github.com/coreos/go-systemd/dbus"
	sd_util "github.com/coreos/go-systemd/util"
	"github.com/coreos/rkt/tests/testutils"
	"github.com/vishvananda/netlink"
)

func TestSocketProxyd(t *testing.T) {
	// Skip the test for now. See
	// https://github.com/coreos/rkt/issues/2432#issuecomment-238858840 for
	// details.
	t.Skip("this test is racy, let's skip it until we fix it")

	if !sd_util.IsRunningSystemd() {
		t.Skip("Systemd is not running on the host.")
	}

	socketProxydPath := "/lib/systemd/systemd-socket-proxyd"
	if _, err := os.Stat(socketProxydPath); os.IsNotExist(err) {
		t.Skip("systemd-socket-proxyd is not installed.")
	}

	ctx := testutils.NewRktRunCtx()
	defer ctx.Cleanup()

	iface, _, err := testutils.GetNonLoIfaceWithAddrs(netlink.FAMILY_V4)
	if err != nil {
		t.Fatalf("Error while getting non-lo host interface: %v\n", err)
	}
	if iface.Name == "" {
		t.Skipf("Cannot run test without non-lo host interface")
	}

	nt := networkTemplateT{
		Name:   "ptp0",
		Type:   "ptp",
		IpMasq: true,
		Master: iface.Name,
		Ipam: &ipamTemplateT{
			Type:   "host-local",
			Subnet: "192.168.0.0/24",
			Routes: []map[string]string{
				{"dst": "0.0.0.0/0"},
			},
		},
	}

	netDir := prepareTestNet(t, ctx, nt)
	defer os.RemoveAll(netDir)

	port, err := randomFreePort(t)
	if err != nil {
		t.Fatal(err)
	}

	echoImage := patchTestACI("rkt-inspect-echo.aci",
		"--exec=/echo-socket-activated",
		"--ports=test-port,protocol=tcp,port=80,socketActivated=true")
	defer os.Remove(echoImage)

	conn, err := sd_dbus.New()
	if err != nil {
		t.Fatal(err)
	}

	rktTestingEchoService := `
	[Unit]
	Description=Socket-activated echo server

	[Service]
	ExecStart=%s
	KillMode=process
	`

	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	rnd := r.Int()

	// Write unit files directly to runtime system units directory
	// (/run/systemd/system) to avoid calling LinkUnitFiles - it is buggy in
	// systemd v219 as it does not work with absolute paths.
	unitsDir := "/run/systemd/system"
	containerIP := "192.168.0.101"

	cmd := fmt.Sprintf("%s --insecure-options=image --debug run --net=%s:IP=%s --port=test-port:%d --mds-register=false %s",
		ctx.Cmd(), nt.Name, containerIP, port, echoImage)

	serviceContent := fmt.Sprintf(rktTestingEchoService, cmd)
	serviceTargetBase := fmt.Sprintf("rkt-testing-socket-activation-%d.service", rnd)
	serviceTarget := filepath.Join(unitsDir, serviceTargetBase)

	if err := ioutil.WriteFile(serviceTarget, []byte(serviceContent), 0666); err != nil {
		t.Fatal(err)
	}
	defer os.Remove(serviceTarget)

	rktTestingEchoSocket := `
	[Unit]
	Description=Socket-activated netcat server socket

	[Socket]
	ListenStream=%d

	[Install]
	WantedBy=sockets.target
	`

	socketContent := fmt.Sprintf(rktTestingEchoSocket, port)
	socketTargetBase := fmt.Sprintf("proxy-to-rkt-testing-socket-activation-%d.socket", rnd)
	socketTarget := filepath.Join(unitsDir, socketTargetBase)

	if err := ioutil.WriteFile(socketTarget, []byte(socketContent), 0666); err != nil {
		t.Fatal(err)
	}
	defer os.Remove(socketTarget)

	proxyToRktTestingEchoService := `
	[Unit]
	Requires=%s
	After=%s

	[Service]
	ExecStart=%s %s:%d
	`

	proxyContent := fmt.Sprintf(proxyToRktTestingEchoService, serviceTargetBase, serviceTargetBase,
		socketProxydPath, containerIP, port)
	proxyContentBase := fmt.Sprintf("proxy-to-rkt-testing-socket-activation-%d.service", rnd)
	proxyTarget := filepath.Join(unitsDir, proxyContentBase)

	if err := ioutil.WriteFile(proxyTarget, []byte(proxyContent), 0666); err != nil {
		t.Fatal(err)
	}
	defer os.Remove(proxyTarget)

	reschan := make(chan string)
	doJob := func() {
		job := <-reschan
		if job != "done" {
			t.Fatal("Job is not done:", job)
		}
	}

	if _, err := conn.StartUnit(socketTargetBase, "replace", reschan); err != nil {
		t.Fatal(err)
	}
	doJob()

	defer func() {
		if _, err := conn.StopUnit(socketTargetBase, "replace", reschan); err != nil {
			t.Fatal(err)
		}
		doJob()

		if _, err := conn.StopUnit(serviceTargetBase, "replace", reschan); err != nil {
			t.Fatal(err)
		}
		doJob()

		if _, err := conn.StopUnit(proxyContentBase, "replace", reschan); err != nil {
			t.Fatal(err)
		}
		doJob()
	}()

	expected := "HELO\n"
	sockConn, err := net.Dial("tcp", fmt.Sprintf("localhost:%d", port))
	if err != nil {
		t.Fatal(err)
	}

	if _, err := fmt.Fprintf(sockConn, expected); err != nil {
		t.Fatal(err)
	}

	answer, err := bufio.NewReader(sockConn).ReadString('\n')
	if err != nil {
		t.Fatal(err)
	}

	if answer != expected {
		t.Fatalf("Expected %q, Got %q", expected, answer)
	}

	return
}
<|endoftext|>


<commit_before>// Copyright 2014 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package queue

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/adeven/redismq"
	"github.com/globocom/config"
	"github.com/globocom/tsuru/log"
	"io"
	"net"
	"strings"
	"time"
)

type redismqQ struct {
	name     string
	queue    *redismq.Queue
	consumer *redismq.Consumer
}

func (r *redismqQ) Put(m *Message, delay time.Duration) error {
	var buf bytes.Buffer
	err := json.NewEncoder(&buf).Encode(m)
	if err != nil {
		return err
	}
	if delay > 0 {
		go func() {
			time.Sleep(delay)
			r.queue.Put(buf.String())
		}()
		return nil
	} else {
		return r.queue.Put(buf.String())
	}
}

func (r *redismqQ) Get(timeout time.Duration) (*Message, error) {
	packChan := make(chan *redismq.Package)
	errChan := make(chan error)
	quit := make(chan int)
	go func() {
		var pack *redismq.Package
		var err error
		for pack == nil {
			select {
			case <-quit:
				return
			default:
				pack, err = r.consumer.NoWaitGet()
				if err != nil {
					errChan <- err
					return
				}
			}
		}
		packChan <- pack
	}()
	var pack *redismq.Package
	select {
	case pack = <-packChan:
	case err := <-errChan:
		return nil, err
	case <-time.After(timeout):
		close(quit)
		return nil, &timeoutError{timeout: timeout}
	}
	defer pack.Ack()
	reader := strings.NewReader(pack.Payload)
	var msg Message
	if err := json.NewDecoder(reader).Decode(&msg); err != nil && err != io.EOF {
		return nil, fmt.Errorf("Invalid message: %q", pack.Payload)
	}
	return &msg, nil
}

type redismqQFactory struct{}

func (factory redismqQFactory) Get(name string) (Q, error) {
	return factory.get(name, "factory")
}

func (redismqQFactory) get(name, consumerName string) (*redismqQ, error) {
	host, err := config.GetString("queue:redis-host")
	if err != nil {
		host = "localhost"
	}
	port, err := config.GetString("queue:redis-port")
	if err != nil {
		if nport, err := config.GetInt("queue:redis-port"); err != nil {
			port = "6379"
		} else {
			port = fmt.Sprintf("%d", nport)
		}
	}
	password, _ := config.GetString("queue:redis-password")
	db, err := config.GetInt("queue:redis-db")
	if err != nil {
		db = 3
	}
	queue := redismq.CreateQueue(host, port, password, int64(db), name)
	consumer, err := queue.AddConsumer(consumerName)
	if err != nil {
		return nil, err
	}
	return &redismqQ{name: name, queue: queue, consumer: consumer}, nil
}

func (factory redismqQFactory) Handler(f func(*Message), names ...string) (Handler, error) {
	name := "default"
	if len(names) > 0 {
		name = names[0]
	}
	consumerName := fmt.Sprintf("handler-%d", time.Now().UnixNano())
	queue, err := factory.get(name, consumerName)
	if err != nil {
		return nil, err
	}
	return &executor{
		inner: func() {
			if message, err := queue.Get(5e9); err == nil {
				log.Debugf("Dispatching %q message to handler function.", message.Action)
				go func(m *Message) {
					f(m)
					if m.fail {
						queue.Put(m, 0)
					}
				}(message)
			} else {
				log.Debugf("Failed to get message from the queue: %s. Trying again...", err)
				if e, ok := err.(*net.OpError); ok && e.Op == "dial" {
					time.Sleep(5e9)
				}
			}
		},
	}, nil
}
<commit_msg>queue/redis: change configuration parameter<commit_after>// Copyright 2014 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package queue

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/adeven/redismq"
	"github.com/globocom/config"
	"github.com/globocom/tsuru/log"
	"io"
	"net"
	"strings"
	"time"
)

type redismqQ struct {
	name     string
	queue    *redismq.Queue
	consumer *redismq.Consumer
}

func (r *redismqQ) Put(m *Message, delay time.Duration) error {
	var buf bytes.Buffer
	err := json.NewEncoder(&buf).Encode(m)
	if err != nil {
		return err
	}
	if delay > 0 {
		go func() {
			time.Sleep(delay)
			r.queue.Put(buf.String())
		}()
		return nil
	} else {
		return r.queue.Put(buf.String())
	}
}

func (r *redismqQ) Get(timeout time.Duration) (*Message, error) {
	packChan := make(chan *redismq.Package)
	errChan := make(chan error)
	quit := make(chan int)
	go func() {
		var pack *redismq.Package
		var err error
		for pack == nil {
			select {
			case <-quit:
				return
			default:
				pack, err = r.consumer.NoWaitGet()
				if err != nil {
					errChan <- err
					return
				}
			}
		}
		packChan <- pack
	}()
	var pack *redismq.Package
	select {
	case pack = <-packChan:
	case err := <-errChan:
		return nil, err
	case <-time.After(timeout):
		close(quit)
		return nil, &timeoutError{timeout: timeout}
	}
	defer pack.Ack()
	reader := strings.NewReader(pack.Payload)
	var msg Message
	if err := json.NewDecoder(reader).Decode(&msg); err != nil && err != io.EOF {
		return nil, fmt.Errorf("Invalid message: %q", pack.Payload)
	}
	return &msg, nil
}

type redismqQFactory struct{}

func (factory redismqQFactory) Get(name string) (Q, error) {
	return factory.get(name, "factory")
}

func (redismqQFactory) get(name, consumerName string) (*redismqQ, error) {
	host, err := config.GetString("redis-queue:host")
	if err != nil {
		host = "localhost"
	}
	port, err := config.GetString("redis-queue:port")
	if err != nil {
		if nport, err := config.GetInt("redis-queue:port"); err != nil {
			port = "6379"
		} else {
			port = fmt.Sprintf("%d", nport)
		}
	}
	password, _ := config.GetString("redis-queue:password")
	db, err := config.GetInt("queue:redis-db")
	if err != nil {
		db = 3
	}
	queue := redismq.CreateQueue(host, port, password, int64(db), name)
	consumer, err := queue.AddConsumer(consumerName)
	if err != nil {
		return nil, err
	}
	return &redismqQ{name: name, queue: queue, consumer: consumer}, nil
}

func (factory redismqQFactory) Handler(f func(*Message), names ...string) (Handler, error) {
	name := "default"
	if len(names) > 0 {
		name = names[0]
	}
	consumerName := fmt.Sprintf("handler-%d", time.Now().UnixNano())
	queue, err := factory.get(name, consumerName)
	if err != nil {
		return nil, err
	}
	return &executor{
		inner: func() {
			if message, err := queue.Get(5e9); err == nil {
				log.Debugf("Dispatching %q message to handler function.", message.Action)
				go func(m *Message) {
					f(m)
					if m.fail {
						queue.Put(m, 0)
					}
				}(message)
			} else {
				log.Debugf("Failed to get message from the queue: %s. Trying again...", err)
				if e, ok := err.(*net.OpError); ok && e.Op == "dial" {
					time.Sleep(5e9)
				}
			}
		},
	}, nil
}
<|endoftext|>


<commit_before>// Copyright 2011, Bryan Matsuo. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
/*
 *  Filename:    queue.go
 *  Package:     dispatch
 *  Author:      Bryan Matsuo <bmatsuo@soe.ucsc.edu>
 *  Created:     Wed Jul  6 17:30:20 PDT 2011
 *  Description:
 */

//  Package queues defines the Queue interface used in package dispatch,
//  and several Queue implementations.
package queues
import (
    "sort"
)

//  A Task is the interface satisfied by objects passed to a Dispatch.
type Task interface {
    SetFunc(func (id int64))
    Func() func (id int64)
    Type() string                 // Used mostly for debugging
}
//  A Task given to a Dispatch is given a unique id and becomes a
//  RegisteredTask.
type RegisteredTask interface {
    Task() Task
    Func() func (id int64)
    Id()   int64
}

func registeredTaskSearch(rts []RegisteredTask, less func(t RegisteredTask)bool) int {
    return sort.Search(len(rts), func(i int)bool{less(rts[i])})
    var (
        low  = 0
        high = len(rts)
        mid  = (high-low)/2
    )
    if high == 0 || !less(rts[0]){
        return 0
    }
    if less(rts[high-1]) {
        return high
    }
    for low < high {
        switch less(rts[mid]) {
        case true:
            low = mid+1
        case false:
            high = mid
        }
        mid = low + (high-low)/2
    }
    return low
}

//  A Queue is a queue for RegisteredTasks, used by a Dispatch.
type Queue interface {
    Enqueue(task RegisteredTask)  // Insert a DispatchTask
    Dequeue() RegisteredTask      // Remove the next task.
    Len() int                     // Number of items to be processed.
    SetKey(int64, float64)        // Set a task's key (priority queues).
}

//  A naive First In First Out (FIFO) Queue.
type FIFO struct {
    head, tail  int
    length      int
    circ        []RegisteredTask
}
//  Create a new FIFO.
func NewFIFO() *FIFO {
    var q = new(FIFO)
    q.circ = make([]RegisteredTask, 10)
    q.head = 0
    q.tail = 0
    q.length = 0
    return q
}

//  See Queue.
func (dq *FIFO) Len() int {
    return dq.length
}
//  See Queue.
func (dq *FIFO) Enqueue(task RegisteredTask) {
    var n = len(dq.circ)
    if dq.length == len(dq.circ) {
        // Copy the circular slice into a new slice with twice the length.
        var tmp = dq.circ
        dq.circ = make([]RegisteredTask, 2*n)
        for i := 0 ; i < n ; i++ {
            var j = (dq.head+i)%n
            dq.circ[i] = tmp[j]
            tmp[j] = nil
        }
        dq.head = 0
        dq.tail = n
    }
    dq.circ[dq.tail] = task
    dq.tail = (dq.tail+1)%n
    dq.length++
}
//  See Queue.
func (dq *FIFO) Dequeue() RegisteredTask {
    if dq.length == 0 {
        panic("empty")
    }
    var task = dq.circ[dq.head]
    dq.head = (dq.head+1)%dq.length
    dq.length--
    return task
}
//  Does nothing. See Queue.
func (dq *FIFO) SetKey(id int64, k float64) { }

//  A naive Last In First Out (LIFO) Queue (also known as a stack).
type LIFO struct {
    top    int
    stack   []RegisteredTask
}
//  Create a new LIFO.
func NewLIFO() *LIFO {
    var q = new(LIFO)
    q.stack = make([]RegisteredTask, 10)
    q.top = 0
    return q
}

//  See Queue.
func (dq *LIFO) Len() int {
    return dq.top
}
//  See Queue.
func (dq *LIFO) Enqueue(task RegisteredTask) {
    var n = len(dq.stack)
    if dq.top == n {
        var tmpstack = dq.stack
        dq.stack = make([]RegisteredTask, 2*n)
        copy(dq.stack, tmpstack)
    }
    dq.stack[dq.top] = task
    dq.top++
}
//  See Queue.
func (dq *LIFO) Dequeue() RegisteredTask {
    if dq.top == 0 {
        panic("empty")
    }
    dq.top--
    var task = dq.stack[dq.top]
    dq.stack[dq.top] = nil
    return task
}
//  Does nothing. See Queue.
func (dq *LIFO) SetKey(id int64, k float64) { }
<commit_msg>Fix a bug in the sort.Search less function.<commit_after>// Copyright 2011, Bryan Matsuo. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
/*
 *  Filename:    queue.go
 *  Package:     dispatch
 *  Author:      Bryan Matsuo <bmatsuo@soe.ucsc.edu>
 *  Created:     Wed Jul  6 17:30:20 PDT 2011
 *  Description:
 */

//  Package queues defines the Queue interface used in package dispatch,
//  and several Queue implementations.
package queues
import (
    "sort"
)

//  A Task is the interface satisfied by objects passed to a Dispatch.
type Task interface {
    SetFunc(func (id int64))
    Func() func (id int64)
    Type() string                 // Used mostly for debugging
}
//  A Task given to a Dispatch is given a unique id and becomes a
//  RegisteredTask.
type RegisteredTask interface {
    Task() Task
    Func() func (id int64)
    Id()   int64
}

func registeredTaskSearch(rts []RegisteredTask, less func(t RegisteredTask)bool) int {
    return sort.Search(len(rts), func(i int)bool{return less(rts[i])})
}

//  A Queue is a queue for RegisteredTasks, used by a Dispatch.
type Queue interface {
    Enqueue(task RegisteredTask)  // Insert a DispatchTask
    Dequeue() RegisteredTask      // Remove the next task.
    Len() int                     // Number of items to be processed.
    SetKey(int64, float64)        // Set a task's key (priority queues).
}

//  A naive First In First Out (FIFO) Queue.
type FIFO struct {
    head, tail  int
    length      int
    circ        []RegisteredTask
}
//  Create a new FIFO.
func NewFIFO() *FIFO {
    var q = new(FIFO)
    q.circ = make([]RegisteredTask, 10)
    q.head = 0
    q.tail = 0
    q.length = 0
    return q
}

//  See Queue.
func (dq *FIFO) Len() int {
    return dq.length
}
//  See Queue.
func (dq *FIFO) Enqueue(task RegisteredTask) {
    var n = len(dq.circ)
    if dq.length == len(dq.circ) {
        // Copy the circular slice into a new slice with twice the length.
        var tmp = dq.circ
        dq.circ = make([]RegisteredTask, 2*n)
        for i := 0 ; i < n ; i++ {
            var j = (dq.head+i)%n
            dq.circ[i] = tmp[j]
            tmp[j] = nil
        }
        dq.head = 0
        dq.tail = n
    }
    dq.circ[dq.tail] = task
    dq.tail = (dq.tail+1)%n
    dq.length++
}
//  See Queue.
func (dq *FIFO) Dequeue() RegisteredTask {
    if dq.length == 0 {
        panic("empty")
    }
    var task = dq.circ[dq.head]
    dq.head = (dq.head+1)%dq.length
    dq.length--
    return task
}
//  Does nothing. See Queue.
func (dq *FIFO) SetKey(id int64, k float64) { }

//  A naive Last In First Out (LIFO) Queue (also known as a stack).
type LIFO struct {
    top    int
    stack   []RegisteredTask
}
//  Create a new LIFO.
func NewLIFO() *LIFO {
    var q = new(LIFO)
    q.stack = make([]RegisteredTask, 10)
    q.top = 0
    return q
}

//  See Queue.
func (dq *LIFO) Len() int {
    return dq.top
}
//  See Queue.
func (dq *LIFO) Enqueue(task RegisteredTask) {
    var n = len(dq.stack)
    if dq.top == n {
        var tmpstack = dq.stack
        dq.stack = make([]RegisteredTask, 2*n)
        copy(dq.stack, tmpstack)
    }
    dq.stack[dq.top] = task
    dq.top++
}
//  See Queue.
func (dq *LIFO) Dequeue() RegisteredTask {
    if dq.top == 0 {
        panic("empty")
    }
    dq.top--
    var task = dq.stack[dq.top]
    dq.stack[dq.top] = nil
    return task
}
//  Does nothing. See Queue.
func (dq *LIFO) SetKey(id int64, k float64) { }
<|endoftext|>


<commit_before>// Copyright 2013,2014 The go-logger Authors. All rights reserved.
// This code is MIT licensed. See the LICENSE file for more info.

package log

import (
	"bytes"
	"fmt"
	"io"
	"math/rand"
	"os"
	"path/filepath"
	"runtime"
	"testing"
	"time"
)

func TestStream(t *testing.T) {
	var buf bytes.Buffer
	logr := New(LEVEL_CRITICAL, os.Stdout, &buf)
	logr.Streams[1] = &buf
	if out := logr.Streams[1]; out != &buf {
		t.Errorf("Stream = %p, want %p", out, &buf)
	}
}

func TestMultiStreams(t *testing.T) {
	rand.Seed(time.Now().UnixNano())
	fPath := filepath.Join(os.TempDir(), fmt.Sprint("go_test_",
		rand.Int()))
	file, err := os.Create(fPath)
	if err != nil {
		t.Error("Create(%q) = %v; want: nil", fPath, err)
	}
	defer file.Close()
	var buf bytes.Buffer
	eLen := 55
	logr := New(LEVEL_DEBUG, file, &buf)
	logr.Debugln("Testing debug output!")
	b := make([]byte, eLen)
	n, err := file.ReadAt(b, 0)
	if n != eLen || err != nil {
		t.Errorf("Read(%d) = %d, %v; want: %d, nil", eLen, n, err,
			eLen)
	}
	if buf.Len() != eLen {
		t.Errorf("buf.Len() = %d; want: %d", buf.Len(), eLen)
	}
}

func TestLongFileFlag(t *testing.T) {
	var buf bytes.Buffer
	SetStreams(&buf)
	SetLevel(LEVEL_DEBUG)
	SetFlags(LnoPrefix | LlongFileName)
	Debugln("Test long file flag")
	_, file, _, _ := runtime.Caller(0)
	expect := fmt.Sprintf("[DEBUG] %s: Test long file flag\n", file)
	if buf.String() != expect {
		t.Errorf("\nExpect:\n\t%q\nGot:\n\t%q\n", expect, buf.String())
	}
}

func TestShortFileFlag(t *testing.T) {
	var buf bytes.Buffer
	SetStreams(&buf)
	SetLevel(LEVEL_DEBUG)
	SetFlags(LnoPrefix | LshortFileName)

	Debugln("Test short file flag")
	_, file, _, _ := runtime.Caller(0)
	short := file

	for i := len(file) - 1; i > 0; i-- {
		if file[i] == '/' {
			short = file[i+1:]
			break
		}
	}

	file = short
	expect := fmt.Sprintf("[DEBUG] %s: Test short file flag\n", file)
	if buf.String() != expect {
		t.Errorf("\nExpect:\n\t%q\nGot:\n\t%q\n", expect, buf.String())
	}
}

var (
	boldPrefix  = AnsiEscape(ANSI_BOLD, "TEST>", ANSI_OFF)
	colorPrefix = AnsiEscape(ANSI_BOLD, ANSI_RED, "TEST>", ANSI_OFF)
	date        = "Mon 20060102 15:04:05"
)

var outputTests = []struct {
	template   string
	prefix     string
	level      level
	dateFormat string
	flags      int
	text       string
	want       string
	wantErr    bool
}{

	// The %s format specifier is the placeholder for the date.
	{logFmt, boldPrefix, LEVEL_ALL, date, LstdFlags, "test number 1",
		"%s \x1b[1mTEST>\x1b[0m test number 1", false},

	{logFmt, colorPrefix, LEVEL_ALL, date, LstdFlags, "test number 2",
		"%s \x1b[1m\x1b[31mTEST>\x1b[0m test number 2", false},

	// Test output with coloring turned off
	{logFmt, AnsiEscape(ANSI_BOLD, "::", ANSI_OFF), LEVEL_ALL, date, Ldate,
		"test number 3", "%s :: test number 3", false},

	{logFmt, defaultPrefixColor, LEVEL_DEBUG, time.RubyDate, LstdFlags,
		"test number 4",
		"%s \x1b[1m\x1b[32m::\x1b[0m \x1b[1m\x1b[37m[DEBUG]\x1b[0m test number 4",
		false},

	{logFmt, defaultPrefixColor, LEVEL_INFO, time.RubyDate, LstdFlags,
		"test number 5",
		"%s \x1b[1m\x1b[32m::\x1b[0m \x1b[1m\x1b[32m[INFO]\x1b[0m test number 5",
		false},

	{logFmt, defaultPrefixColor, LEVEL_WARNING, time.RubyDate, LstdFlags,
		"test number 6",
		"%s \x1b[1m\x1b[32m::\x1b[0m \x1b[1m\x1b[33m[WARNING]\x1b[0m test number 6",
		false},

	{logFmt, defaultPrefixColor, LEVEL_ERROR, time.RubyDate, LstdFlags,
		"test number 7",
		"%s \x1b[1m\x1b[32m::\x1b[0m \x1b[1m\x1b[35m[ERROR]\x1b[0m test number 7",
		false},

	{logFmt, defaultPrefixColor, LEVEL_CRITICAL, time.RubyDate, LstdFlags,
		"test number 8",
		"%s \x1b[1m\x1b[32m::\x1b[0m \x1b[1m\x1b[31m[CRITICAL]\x1b[0m test number 8",
		false},

	// Test date format
	{logFmt, defaultPrefixColor, LEVEL_ALL, "Mon 20060102 15:04:05",
		Ldate, "test number 9",
		"%s :: test number 9", false},
}

func TestOutput(t *testing.T) {
	for i, k := range outputTests {
		var buf bytes.Buffer
		logr := New(LEVEL_DEBUG, &buf)
		logr.Prefix = k.prefix
		logr.DateFormat = k.dateFormat
		logr.Flags = k.flags
		logr.Level = k.level
		d := time.Now().Format(logr.DateFormat)
		n, err := logr.Fprint(k.level, 1, k.text, &buf)
		if n != buf.Len() {
			t.Error("Error: ", io.ErrShortWrite)
		}
		want := fmt.Sprintf(k.want, d)
		if buf.String() != want || err != nil && !k.wantErr {
			t.Errorf("Print test %d failed, \ngot:  %q\nwant: "+
				"%q", i+1, buf.String(), want)
			continue
		}
	}
}

func TestLevel(t *testing.T) {
	var buf bytes.Buffer
	logr := New(LEVEL_CRITICAL, &buf)
	logr.Debug("This level should produce no output")
	if buf.Len() != 0 {
		t.Errorf("Debug() produced output at LEVEL_CRITICAL logging level")
	}
	logr.Level = LEVEL_DEBUG
	logr.Debug("This level should produce output")
	if buf.Len() == 0 {
		t.Errorf("Debug() did not produce output at the LEVEL_DEBUG logging level")
	}
	buf.Reset()
	logr.Level = LEVEL_CRITICAL
	logr.Println("This level should produce output")
	if buf.Len() == 0 {
		t.Errorf("Debug() did not produce output at the ALL logging level")
	}
	buf.Reset()
	logr.Level = LEVEL_ALL
	logr.Debug("This level should produce output")
	if buf.Len() == 0 {
		t.Errorf("Debug() did not produce output at the ALL logging level")
	}
}

func TestPrefixNewline(t *testing.T) {
	var buf bytes.Buffer
	SetStreams(&buf)
	SetLevel(LEVEL_DEBUG)
	SetFlags(LnoPrefix)
	Debug("\n\nThis line should be padded with newlines.\n\n")
	expect := "\n\n[DEBUG] This line should be padded with newlines.\n\n"
	if buf.String() != expect {
		t.Errorf("\nExpect:\n%q\nGot:\n%q\n", expect, buf.String())
	}
}

func TestFlagsLdate(t *testing.T) {
	var buf bytes.Buffer
	SetStreams(&buf)
	SetLevel(LEVEL_DEBUG)
	SetFlags(LnoPrefix)
	Debugln("This output should not have a date.")
	expect := "[DEBUG] This output should not have a date.\n"
	if buf.String() != expect {
		t.Errorf("\nExpect:\n\t%q\nGot:\n\t%q\n", expect, buf.String())
	}
}

func TestFlagsLfunctionName(t *testing.T) {
	var buf bytes.Buffer
	SetStreams(&buf)
	SetLevel(LEVEL_DEBUG)
	SetFlags(LnoPrefix | LfunctionName)
	Debugln("This output should have a function name.")
	expect := "[DEBUG] TestFlagsLfunctionName: This output should have a function name.\n"
	if buf.String() != expect {
		t.Errorf("\nExpect:\n\t%q\nGot:\n\t%q\n", expect, buf.String())
	}
}

func TestFlagsLfunctionNameWithFileName(t *testing.T) {
	var buf bytes.Buffer
	SetStreams(&buf)
	SetLevel(LEVEL_DEBUG)
	SetFlags(LnoPrefix | LfunctionName | LshortFileName)
	Debug("This output should have a file name and a function name.")
	expect := "[DEBUG] logger_test.go: TestFlagsLfunctionNameWithFileName" +
		": This output should have a file name and a function name."
	if buf.String() != expect {
		t.Errorf("\nExpect:\n\t%q\nGot:\n\t%q\n", expect, buf.String())
	}
}

func TestFlagsLansiWithNewlinePaddingDebug(t *testing.T) {
	var buf bytes.Buffer
	SetStreams(&buf)
	SetLevel(LEVEL_DEBUG)
	SetFlags(LnoPrefix | Lansi)
	Debug("\n\nThis output should be padded with newlines and colored.\n\n")
	expect := "\n\n\x1b[1m\x1b[37m[DEBUG]\x1b[0m This output should be " +
		"padded with newlines and colored.\n\n"
	if buf.String() != expect {
		t.Errorf("\nExpect:\n\t%q\nGot:\n\t%q\n", expect, buf.String())
	}
}

func TestFlagsLansiWithNewlinePaddingDebugln(t *testing.T) {
	var buf bytes.Buffer
	SetStreams(&buf)
	SetLevel(LEVEL_DEBUG)
	SetFlags(LnoPrefix | Lansi)
	Debugln("\n\nThis output should be padded with newlines and colored.\n\n")
	expect := "\n\n\x1b[1m\x1b[37m[DEBUG]\x1b[0m This output should be " +
		"padded with newlines and colored.\n\n\n"
	if buf.String() != expect {
		t.Errorf("\nExpect:\n\t%q\nGot:\n\t%q\n", expect, buf.String())
	}
	buf.Reset()
	Debugln("\n\n", "### HELLO", "NEWMAN", "###", "\n\n")
	expect = "\n\n\x1b[1m\x1b[37m[DEBUG]\x1b[0m  ### HELLO NEWMAN ### \n\n\n"
	if buf.String() != expect {
		t.Errorf("\nExpect:\n\t%q\nGot:\n\t%q\n", expect, buf.String())
	}
	buf.Reset()
	Debugln("\n\n### HELLO", "NEWMAN", "###\n\n")
	expect = "\n\n\x1b[1m\x1b[37m[DEBUG]\x1b[0m ### HELLO NEWMAN ###\n\n\n"
	if buf.String() != expect {
		t.Errorf("\nExpect:\n\t%q\nGot:\n\t%q\n", expect, buf.String())
	}
}

func TestFlagsNoLansiWithNewlinePadding(t *testing.T) {
	var buf bytes.Buffer
	SetStreams(&buf)
	SetLevel(LEVEL_DEBUG)
	SetFlags(LnoPrefix)
	Debug("\n\nThis output should be padded with newlines and not colored.\n\n")
	expect := "\n\n[DEBUG] This output should be padded with newlines and not colored.\n\n"
	if buf.String() != expect {
		t.Errorf("\nExpect:\n\t%q\nGot:\n\t%q\n", expect, buf.String())
	}
}
<commit_msg>Add TestFlagsLansiWithNewlinePaddingDebugf<commit_after>// Copyright 2013,2014 The go-logger Authors. All rights reserved.
// This code is MIT licensed. See the LICENSE file for more info.

package log

import (
	"bytes"
	"fmt"
	"io"
	"math/rand"
	"os"
	"path/filepath"
	"runtime"
	"testing"
	"time"
)

func TestStream(t *testing.T) {
	var buf bytes.Buffer
	logr := New(LEVEL_CRITICAL, os.Stdout, &buf)
	logr.Streams[1] = &buf
	if out := logr.Streams[1]; out != &buf {
		t.Errorf("Stream = %p, want %p", out, &buf)
	}
}

func TestMultiStreams(t *testing.T) {
	rand.Seed(time.Now().UnixNano())
	fPath := filepath.Join(os.TempDir(), fmt.Sprint("go_test_",
		rand.Int()))
	file, err := os.Create(fPath)
	if err != nil {
		t.Error("Create(%q) = %v; want: nil", fPath, err)
	}
	defer file.Close()
	var buf bytes.Buffer
	eLen := 55
	logr := New(LEVEL_DEBUG, file, &buf)
	logr.Debugln("Testing debug output!")
	b := make([]byte, eLen)
	n, err := file.ReadAt(b, 0)
	if n != eLen || err != nil {
		t.Errorf("Read(%d) = %d, %v; want: %d, nil", eLen, n, err,
			eLen)
	}
	if buf.Len() != eLen {
		t.Errorf("buf.Len() = %d; want: %d", buf.Len(), eLen)
	}
}

func TestLongFileFlag(t *testing.T) {
	var buf bytes.Buffer
	SetStreams(&buf)
	SetLevel(LEVEL_DEBUG)
	SetFlags(LnoPrefix | LlongFileName)
	Debugln("Test long file flag")
	_, file, _, _ := runtime.Caller(0)
	expect := fmt.Sprintf("[DEBUG] %s: Test long file flag\n", file)
	if buf.String() != expect {
		t.Errorf("\nExpect:\n\t%q\nGot:\n\t%q\n", expect, buf.String())
	}
}

func TestShortFileFlag(t *testing.T) {
	var buf bytes.Buffer
	SetStreams(&buf)
	SetLevel(LEVEL_DEBUG)
	SetFlags(LnoPrefix | LshortFileName)

	Debugln("Test short file flag")
	_, file, _, _ := runtime.Caller(0)
	short := file

	for i := len(file) - 1; i > 0; i-- {
		if file[i] == '/' {
			short = file[i+1:]
			break
		}
	}

	file = short
	expect := fmt.Sprintf("[DEBUG] %s: Test short file flag\n", file)
	if buf.String() != expect {
		t.Errorf("\nExpect:\n\t%q\nGot:\n\t%q\n", expect, buf.String())
	}
}

var (
	boldPrefix  = AnsiEscape(ANSI_BOLD, "TEST>", ANSI_OFF)
	colorPrefix = AnsiEscape(ANSI_BOLD, ANSI_RED, "TEST>", ANSI_OFF)
	date        = "Mon 20060102 15:04:05"
)

var outputTests = []struct {
	template   string
	prefix     string
	level      level
	dateFormat string
	flags      int
	text       string
	want       string
	wantErr    bool
}{

	// The %s format specifier is the placeholder for the date.
	{logFmt, boldPrefix, LEVEL_ALL, date, LstdFlags, "test number 1",
		"%s \x1b[1mTEST>\x1b[0m test number 1", false},

	{logFmt, colorPrefix, LEVEL_ALL, date, LstdFlags, "test number 2",
		"%s \x1b[1m\x1b[31mTEST>\x1b[0m test number 2", false},

	// Test output with coloring turned off
	{logFmt, AnsiEscape(ANSI_BOLD, "::", ANSI_OFF), LEVEL_ALL, date, Ldate,
		"test number 3", "%s :: test number 3", false},

	{logFmt, defaultPrefixColor, LEVEL_DEBUG, time.RubyDate, LstdFlags,
		"test number 4",
		"%s \x1b[1m\x1b[32m::\x1b[0m \x1b[1m\x1b[37m[DEBUG]\x1b[0m test number 4",
		false},

	{logFmt, defaultPrefixColor, LEVEL_INFO, time.RubyDate, LstdFlags,
		"test number 5",
		"%s \x1b[1m\x1b[32m::\x1b[0m \x1b[1m\x1b[32m[INFO]\x1b[0m test number 5",
		false},

	{logFmt, defaultPrefixColor, LEVEL_WARNING, time.RubyDate, LstdFlags,
		"test number 6",
		"%s \x1b[1m\x1b[32m::\x1b[0m \x1b[1m\x1b[33m[WARNING]\x1b[0m test number 6",
		false},

	{logFmt, defaultPrefixColor, LEVEL_ERROR, time.RubyDate, LstdFlags,
		"test number 7",
		"%s \x1b[1m\x1b[32m::\x1b[0m \x1b[1m\x1b[35m[ERROR]\x1b[0m test number 7",
		false},

	{logFmt, defaultPrefixColor, LEVEL_CRITICAL, time.RubyDate, LstdFlags,
		"test number 8",
		"%s \x1b[1m\x1b[32m::\x1b[0m \x1b[1m\x1b[31m[CRITICAL]\x1b[0m test number 8",
		false},

	// Test date format
	{logFmt, defaultPrefixColor, LEVEL_ALL, "Mon 20060102 15:04:05",
		Ldate, "test number 9",
		"%s :: test number 9", false},
}

func TestOutput(t *testing.T) {
	for i, k := range outputTests {
		var buf bytes.Buffer
		logr := New(LEVEL_DEBUG, &buf)
		logr.Prefix = k.prefix
		logr.DateFormat = k.dateFormat
		logr.Flags = k.flags
		logr.Level = k.level
		d := time.Now().Format(logr.DateFormat)
		n, err := logr.Fprint(k.level, 1, k.text, &buf)
		if n != buf.Len() {
			t.Error("Error: ", io.ErrShortWrite)
		}
		want := fmt.Sprintf(k.want, d)
		if buf.String() != want || err != nil && !k.wantErr {
			t.Errorf("Print test %d failed, \ngot:  %q\nwant: "+
				"%q", i+1, buf.String(), want)
			continue
		}
	}
}

func TestLevel(t *testing.T) {
	var buf bytes.Buffer
	logr := New(LEVEL_CRITICAL, &buf)
	logr.Debug("This level should produce no output")
	if buf.Len() != 0 {
		t.Errorf("Debug() produced output at LEVEL_CRITICAL logging level")
	}
	logr.Level = LEVEL_DEBUG
	logr.Debug("This level should produce output")
	if buf.Len() == 0 {
		t.Errorf("Debug() did not produce output at the LEVEL_DEBUG logging level")
	}
	buf.Reset()
	logr.Level = LEVEL_CRITICAL
	logr.Println("This level should produce output")
	if buf.Len() == 0 {
		t.Errorf("Debug() did not produce output at the ALL logging level")
	}
	buf.Reset()
	logr.Level = LEVEL_ALL
	logr.Debug("This level should produce output")
	if buf.Len() == 0 {
		t.Errorf("Debug() did not produce output at the ALL logging level")
	}
}

func TestPrefixNewline(t *testing.T) {
	var buf bytes.Buffer
	SetStreams(&buf)
	SetLevel(LEVEL_DEBUG)
	SetFlags(LnoPrefix)
	Debug("\n\nThis line should be padded with newlines.\n\n")
	expect := "\n\n[DEBUG] This line should be padded with newlines.\n\n"
	if buf.String() != expect {
		t.Errorf("\nExpect:\n%q\nGot:\n%q\n", expect, buf.String())
	}
}

func TestFlagsLdate(t *testing.T) {
	var buf bytes.Buffer
	SetStreams(&buf)
	SetLevel(LEVEL_DEBUG)
	SetFlags(LnoPrefix)
	Debugln("This output should not have a date.")
	expect := "[DEBUG] This output should not have a date.\n"
	if buf.String() != expect {
		t.Errorf("\nExpect:\n\t%q\nGot:\n\t%q\n", expect, buf.String())
	}
}

func TestFlagsLfunctionName(t *testing.T) {
	var buf bytes.Buffer
	SetStreams(&buf)
	SetLevel(LEVEL_DEBUG)
	SetFlags(LnoPrefix | LfunctionName)
	Debugln("This output should have a function name.")
	expect := "[DEBUG] TestFlagsLfunctionName: This output should have a function name.\n"
	if buf.String() != expect {
		t.Errorf("\nExpect:\n\t%q\nGot:\n\t%q\n", expect, buf.String())
	}
}

func TestFlagsLfunctionNameWithFileName(t *testing.T) {
	var buf bytes.Buffer
	SetStreams(&buf)
	SetLevel(LEVEL_DEBUG)
	SetFlags(LnoPrefix | LfunctionName | LshortFileName)
	Debug("This output should have a file name and a function name.")
	expect := "[DEBUG] logger_test.go: TestFlagsLfunctionNameWithFileName" +
		": This output should have a file name and a function name."
	if buf.String() != expect {
		t.Errorf("\nExpect:\n\t%q\nGot:\n\t%q\n", expect, buf.String())
	}
}

func TestFlagsLansiWithNewlinePaddingDebug(t *testing.T) {
	var buf bytes.Buffer
	SetStreams(&buf)
	SetLevel(LEVEL_DEBUG)
	SetFlags(LnoPrefix | Lansi)
	Debug("\n\nThis output should be padded with newlines and colored.\n\n")
	expect := "\n\n\x1b[1m\x1b[37m[DEBUG]\x1b[0m This output should be " +
		"padded with newlines and colored.\n\n"
	if buf.String() != expect {
		t.Errorf("\nExpect:\n\t%q\nGot:\n\t%q\n", expect, buf.String())
	}
}

func TestFlagsLansiWithNewlinePaddingDebugf(t *testing.T) {
	var buf bytes.Buffer
	SetStreams(&buf)
	SetLevel(LEVEL_DEBUG)
	SetFlags(LnoPrefix | Lansi)
	Debugf("\n\nThis output should be padded with newlines and %s.\n\n",
		"colored")
	expect := "\n\n\x1b[1m\x1b[37m[DEBUG]\x1b[0m This output should be " +
		"padded with newlines and colored.\n\n"
	if buf.String() != expect {
		t.Errorf("\nExpect:\n\t%q\nGot:\n\t%q\n", expect, buf.String())
	}
	buf.Reset()
	Debugf("\n\n##### HELLO %s #####\n\n", "NEWMAN")
	expect = "\n\n\x1b[1m\x1b[37m[DEBUG]\x1b[0m ##### HELLO NEWMAN #####\n\n"
	if buf.String() != expect {
		t.Errorf("\nExpect:\n\t%q\nGot:\n\t%q\n", expect, buf.String())
	}
}

func TestFlagsLansiWithNewlinePaddingDebugln(t *testing.T) {
	var buf bytes.Buffer
	SetStreams(&buf)
	SetLevel(LEVEL_DEBUG)
	SetFlags(LnoPrefix | Lansi)
	Debugln("\n\nThis output should be padded with newlines and colored.\n\n")
	expect := "\n\n\x1b[1m\x1b[37m[DEBUG]\x1b[0m This output should be " +
		"padded with newlines and colored.\n\n\n"
	if buf.String() != expect {
		t.Errorf("\nExpect:\n\t%q\nGot:\n\t%q\n", expect, buf.String())
	}
	buf.Reset()
	Debugln("\n\n", "### HELLO", "NEWMAN", "###", "\n\n")
	expect = "\n\n\x1b[1m\x1b[37m[DEBUG]\x1b[0m  ### HELLO NEWMAN ### \n\n\n"
	if buf.String() != expect {
		t.Errorf("\nExpect:\n\t%q\nGot:\n\t%q\n", expect, buf.String())
	}
	buf.Reset()
	Debugln("\n\n### HELLO", "NEWMAN", "###\n\n")
	expect = "\n\n\x1b[1m\x1b[37m[DEBUG]\x1b[0m ### HELLO NEWMAN ###\n\n\n"
	if buf.String() != expect {
		t.Errorf("\nExpect:\n\t%q\nGot:\n\t%q\n", expect, buf.String())
	}
}

func TestFlagsNoLansiWithNewlinePadding(t *testing.T) {
	var buf bytes.Buffer
	SetStreams(&buf)
	SetLevel(LEVEL_DEBUG)
	SetFlags(LnoPrefix)
	Debug("\n\nThis output should be padded with newlines and not colored.\n\n")
	expect := "\n\n[DEBUG] This output should be padded with newlines and not colored.\n\n"
	if buf.String() != expect {
		t.Errorf("\nExpect:\n\t%q\nGot:\n\t%q\n", expect, buf.String())
	}
}
<|endoftext|>


<commit_before>package reader

import (
	"encoding/binary"
	"io"
	"reflect"
	"sync"
	"strings"

	"github.com/apache/thrift/lib/go/thrift"
	"github.com/xitongsys/parquet-go/common"
	"github.com/xitongsys/parquet-go/layout"
	"github.com/xitongsys/parquet-go/marshal"
	"github.com/xitongsys/parquet-go/source"
	"github.com/xitongsys/parquet-go/schema"
	"github.com/xitongsys/parquet-go/parquet"
)

type ParquetReader struct {
	SchemaHandler *schema.SchemaHandler
	NP            int64 //parallel number
	Footer        *parquet.FileMetaData
	PFile         source.ParquetFile

	ColumnBuffers map[string]*ColumnBufferType

	//One reader can only read one type objects
	ObjType			reflect.Type
	ObjPartialType	reflect.Type
}

//Create a parquet reader
func NewParquetReader(pFile source.ParquetFile, obj interface{}, np int64) (*ParquetReader, error) {
	var err error
	res := new(ParquetReader)
	res.NP = np
	res.PFile = pFile
	if err = res.ReadFooter(); err != nil {
		return nil, err
	}
	res.ColumnBuffers = make(map[string]*ColumnBufferType)

	if obj != nil {
		if res.SchemaHandler, err = schema.NewSchemaHandlerFromStruct(obj); err != nil {
			return res, err
		}

	}else{
		res.SchemaHandler = schema.NewSchemaHandlerFromSchemaList(res.Footer.Schema)
	}

	res.RenameSchema()
	for i := 0; i < len(res.SchemaHandler.SchemaElements); i++ {
		schema := res.SchemaHandler.SchemaElements[i]
		if schema.GetNumChildren() == 0 {
			pathStr := res.SchemaHandler.IndexMap[int32(i)]
			if res.ColumnBuffers[pathStr], err = NewColumnBuffer(pFile, res.Footer, res.SchemaHandler, pathStr); err != nil {
				return res, err
			}
		}
	}

	return res, nil
}

func (self *ParquetReader) SetSchemaHandlerFromJSON(jsonSchema string) error {
	var err error
	if self.SchemaHandler, err = schema.NewSchemaHandlerFromJSON(jsonSchema); err != nil {
		return err
	}

	self.RenameSchema()
	for i := 0; i < len(self.SchemaHandler.SchemaElements); i++ {
		schemaElement := self.SchemaHandler.SchemaElements[i]
		if schemaElement.GetNumChildren() == 0 {
			pathStr := self.SchemaHandler.IndexMap[int32(i)]
			if self.ColumnBuffers[pathStr], err = NewColumnBuffer(self.PFile, self.Footer, self.SchemaHandler, pathStr); err != nil {
				return err
			}
		}
	}
	return nil
}

//Rename schema name to inname
func (self *ParquetReader) RenameSchema() {
	for i := 0; i < len(self.SchemaHandler.Infos); i++ {
		self.Footer.Schema[i].Name = self.SchemaHandler.Infos[i].InName
	}
	for _, rowGroup := range self.Footer.RowGroups {
		for _, chunk := range rowGroup.Columns {
			exPath := make([]string, 0)
			exPath = append(exPath, self.SchemaHandler.GetRootExName())
			exPath = append(exPath, chunk.MetaData.GetPathInSchema()...)
			exPathStr := common.PathToStr(exPath)

			inPathStr := self.SchemaHandler.ExPathToInPath[exPathStr]
			inPath := common.StrToPath(inPathStr)[1:]
			chunk.MetaData.PathInSchema = inPath
		}
	}
}

func (self *ParquetReader) GetNumRows() int64 {
	return self.Footer.GetNumRows()
}

//Get the footer size
func (self *ParquetReader) GetFooterSize() (uint32, error) {
	var err error
	buf := make([]byte, 4)
	if _, err = self.PFile.Seek(-8, io.SeekEnd); err != nil {
		return 0, err
	}
	if _, err = self.PFile.Read(buf); err != nil {
		return 0, err
	}
	size := binary.LittleEndian.Uint32(buf)
	return size, err
}

//Read footer from parquet file
func (self *ParquetReader) ReadFooter() error {
	size, err := self.GetFooterSize()
	if err != nil {
		return err
	}
	if _, err = self.PFile.Seek(-(int64)(8+size), io.SeekEnd); err != nil {
		return err
	}
	self.Footer = parquet.NewFileMetaData()
	pf := thrift.NewTCompactProtocolFactory()
	protocol := pf.GetProtocol(thrift.NewStreamTransportR(self.PFile))
	return self.Footer.Read(protocol)
}

//Skip rows of parquet file
func (self *ParquetReader) SkipRows(num int64) error {
	var err error
	if num <= 0 {
		return nil
	}
	doneChan := make(chan int, self.NP)
	taskChan := make(chan string, len(self.SchemaHandler.ValueColumns))
	stopChan := make(chan int)

	for _, pathStr := range self.SchemaHandler.ValueColumns {
		if _, ok := self.ColumnBuffers[pathStr]; !ok {
			if self.ColumnBuffers[pathStr], err = NewColumnBuffer(self.PFile, self.Footer, self.SchemaHandler, pathStr); err != nil {
				return err
			}
		}
	}

	for i := int64(0); i < self.NP; i++ {
		go func() {
			for {
				select {
				case <-stopChan:
					return
				case pathStr := <-taskChan:
					cb := self.ColumnBuffers[pathStr]
					cb.SkipRows(int64(num))
					doneChan <- 0
				}
			}
		}()
	}

	for key, _ := range self.ColumnBuffers {
		taskChan <- key
	}

	for i := 0; i < len(self.ColumnBuffers); i++ {
		<-doneChan
	}
	for i := int64(0); i < self.NP; i++ {
		stopChan <- 0
	}
	return err
}

//Read rows of parquet file and unmarshal all to dst
func (self *ParquetReader) Read(dstInterface interface{}) error {
	return self.read(dstInterface, "")
}

// Read maxReadNumber objects
func (self *ParquetReader) ReadByNumber(maxReadNumber int) ([]interface{}, error) {
	var err error
	if self.ObjType == nil {
		if self.ObjType, err = self.SchemaHandler.GetType(self.SchemaHandler.GetRootInName()); err != nil {
			return nil, err
		}
	}

	vs := reflect.MakeSlice(reflect.SliceOf(self.ObjType), maxReadNumber, maxReadNumber)
	res := reflect.New(vs.Type())
	res.Elem().Set(vs)

	if err = self.Read(res.Interface()); err != nil {
		return nil, err
	}

	ln := res.Elem().Len()
	ret := make([]interface{}, ln)
	for i := 0; i < ln; i++ {
		ret[i] = res.Elem().Index(i).Interface()
	}

	return ret, nil
}

//Read rows of parquet file and unmarshal all to dst
func (self *ParquetReader) ReadPartial(dstInterface interface{}, prefixPath string) error {
	prefixPath, err := self.SchemaHandler.ConvertToInPathStr(prefixPath)
	if err != nil {
		return err
	}

	return self.read(dstInterface, prefixPath)
}

// Read maxReadNumber partial objects
func (self *ParquetReader) ReadPartialByNumber(maxReadNumber int, prefixPath string) ([]interface{}, error) {
	var err error
	if self.ObjPartialType == nil {
		if self.ObjPartialType, err = self.SchemaHandler.GetType(prefixPath); err != nil {
			return nil, err
		}
	}

	vs := reflect.MakeSlice(reflect.SliceOf(self.ObjPartialType), maxReadNumber, maxReadNumber)
	res := reflect.New(vs.Type())
	res.Elem().Set(vs)

	if err = self.ReadPartial(res.Interface(), prefixPath); err != nil {
		return nil, err
	}

	ln := res.Elem().Len()
	ret := make([]interface{}, ln)
	for i := 0; i < ln; i++ {
		ret[i] = res.Elem().Index(i).Interface()
	}

	return ret, nil
}

//Read rows of parquet file with a prefixPath
func (self *ParquetReader) read(dstInterface interface{}, prefixPath string) error {
	var err error
	tmap := make(map[string]*layout.Table)
	locker := new(sync.Mutex)
	ot := reflect.TypeOf(dstInterface).Elem().Elem()
	num := reflect.ValueOf(dstInterface).Elem().Len()
	if num <= 0 {
		return nil
	}

	doneChan := make(chan int, self.NP)
	taskChan := make(chan string, len(self.ColumnBuffers))
	stopChan := make(chan int)

	for i := int64(0); i < self.NP; i++ {
		go func() {
			for {
				select {
				case <-stopChan:
					return
				case pathStr := <-taskChan:
					cb := self.ColumnBuffers[pathStr]
					table, _ := cb.ReadRows(int64(num))
					locker.Lock()
					if _, ok := tmap[pathStr]; ok {
						tmap[pathStr].Merge(table)
					} else {
						tmap[pathStr] = layout.NewTableFromTable(table)
						tmap[pathStr].Merge(table)
					}
					locker.Unlock()
					doneChan <- 0
				}
			}
		}()
	}

	readNum := 0
	for key, _ := range self.ColumnBuffers {
		if strings.HasPrefix(key, prefixPath) {
			taskChan <- key
			readNum++
		}
	}
	for i := 0; i < readNum; i++ {
		<-doneChan
	}

	for i := int64(0); i < self.NP; i++ {
		stopChan <- 0
	}

	dstList := make([]interface{}, self.NP)
	delta := (int64(num) + self.NP - 1) / self.NP

	doneChan = make(chan int)
	for c := int64(0); c < self.NP; c++ {
		bgn := c * delta
		end := bgn + delta
		if end > int64(num) {
			end = int64(num)
		}
		if bgn >= int64(num) {
			bgn, end = int64(num), int64(num)
		}
		go func(b, e, index int) {
			dstList[index] = reflect.New(reflect.SliceOf(ot)).Interface()
			if err2 := marshal.Unmarshal(&tmap, b, e, dstList[index], self.SchemaHandler, prefixPath); err2 != nil {
				err = err2
			}
			doneChan <- 0
		}(int(bgn), int(end), int(c))
	}
	for c := int64(0); c < self.NP; c++ {
		<-doneChan
	}

	resTmp := reflect.MakeSlice(reflect.SliceOf(ot), 0, num)
	for _, dst := range dstList {
		resTmp = reflect.AppendSlice(resTmp, reflect.ValueOf(dst).Elem())
	}
	reflect.ValueOf(dstInterface).Elem().Set(resTmp)
	return err
}

//Stop Read
func (self *ParquetReader) ReadStop() {
	for _, cb := range self.ColumnBuffers {
		if cb != nil {
			cb.PFile.Close()
		}
	}
}
<commit_msg>fixing<commit_after>package reader

import (
	"encoding/binary"
	"io"
	"reflect"
	"sync"
	"strings"

	"github.com/apache/thrift/lib/go/thrift"
	"github.com/xitongsys/parquet-go/common"
	"github.com/xitongsys/parquet-go/layout"
	"github.com/xitongsys/parquet-go/marshal"
	"github.com/xitongsys/parquet-go/source"
	"github.com/xitongsys/parquet-go/schema"
	"github.com/xitongsys/parquet-go/parquet"
)

type ParquetReader struct {
	SchemaHandler *schema.SchemaHandler
	NP            int64 //parallel number
	Footer        *parquet.FileMetaData
	PFile         source.ParquetFile

	ColumnBuffers map[string]*ColumnBufferType

	//One reader can only read one type objects
	ObjType			reflect.Type
	ObjPartialType	reflect.Type
}

//Create a parquet reader
func NewParquetReader(pFile source.ParquetFile, obj interface{}, np int64) (*ParquetReader, error) {
	var err error
	res := new(ParquetReader)
	res.NP = np
	res.PFile = pFile
	if err = res.ReadFooter(); err != nil {
		return nil, err
	}
	res.ColumnBuffers = make(map[string]*ColumnBufferType)

	if obj != nil {
		if res.SchemaHandler, err = schema.NewSchemaHandlerFromStruct(obj); err != nil {
			return res, err
		}

	}else{
		res.SchemaHandler = schema.NewSchemaHandlerFromSchemaList(res.Footer.Schema)
	}

	res.RenameSchema()
	for i := 0; i < len(res.SchemaHandler.SchemaElements); i++ {
		schema := res.SchemaHandler.SchemaElements[i]
		if schema.GetNumChildren() == 0 {
			pathStr := res.SchemaHandler.IndexMap[int32(i)]
			if res.ColumnBuffers[pathStr], err = NewColumnBuffer(pFile, res.Footer, res.SchemaHandler, pathStr); err != nil {
				return res, err
			}
		}
	}

	return res, nil
}

func (self *ParquetReader) SetSchemaHandlerFromJSON(jsonSchema string) error {
	var err error
	if self.SchemaHandler, err = schema.NewSchemaHandlerFromJSON(jsonSchema); err != nil {
		return err
	}

	for i := 0; i < len(self.SchemaHandler.SchemaElements); i++ {
		schemaElement := self.SchemaHandler.SchemaElements[i]
		if schemaElement.GetNumChildren() == 0 {
			pathStr := self.SchemaHandler.IndexMap[int32(i)]
			if self.ColumnBuffers[pathStr], err = NewColumnBuffer(self.PFile, self.Footer, self.SchemaHandler, pathStr); err != nil {
				return err
			}
		}
	}
	return nil
}

//Rename schema name to inname
func (self *ParquetReader) RenameSchema() {
	for i := 0; i < len(self.SchemaHandler.Infos); i++ {
		self.Footer.Schema[i].Name = self.SchemaHandler.Infos[i].InName
	}
	for _, rowGroup := range self.Footer.RowGroups {
		for _, chunk := range rowGroup.Columns {
			exPath := make([]string, 0)
			exPath = append(exPath, self.SchemaHandler.GetRootExName())
			exPath = append(exPath, chunk.MetaData.GetPathInSchema()...)
			exPathStr := common.PathToStr(exPath)

			inPathStr := self.SchemaHandler.ExPathToInPath[exPathStr]
			inPath := common.StrToPath(inPathStr)[1:]
			chunk.MetaData.PathInSchema = inPath
		}
	}
}

func (self *ParquetReader) GetNumRows() int64 {
	return self.Footer.GetNumRows()
}

//Get the footer size
func (self *ParquetReader) GetFooterSize() (uint32, error) {
	var err error
	buf := make([]byte, 4)
	if _, err = self.PFile.Seek(-8, io.SeekEnd); err != nil {
		return 0, err
	}
	if _, err = self.PFile.Read(buf); err != nil {
		return 0, err
	}
	size := binary.LittleEndian.Uint32(buf)
	return size, err
}

//Read footer from parquet file
func (self *ParquetReader) ReadFooter() error {
	size, err := self.GetFooterSize()
	if err != nil {
		return err
	}
	if _, err = self.PFile.Seek(-(int64)(8+size), io.SeekEnd); err != nil {
		return err
	}
	self.Footer = parquet.NewFileMetaData()
	pf := thrift.NewTCompactProtocolFactory()
	protocol := pf.GetProtocol(thrift.NewStreamTransportR(self.PFile))
	return self.Footer.Read(protocol)
}

//Skip rows of parquet file
func (self *ParquetReader) SkipRows(num int64) error {
	var err error
	if num <= 0 {
		return nil
	}
	doneChan := make(chan int, self.NP)
	taskChan := make(chan string, len(self.SchemaHandler.ValueColumns))
	stopChan := make(chan int)

	for _, pathStr := range self.SchemaHandler.ValueColumns {
		if _, ok := self.ColumnBuffers[pathStr]; !ok {
			if self.ColumnBuffers[pathStr], err = NewColumnBuffer(self.PFile, self.Footer, self.SchemaHandler, pathStr); err != nil {
				return err
			}
		}
	}

	for i := int64(0); i < self.NP; i++ {
		go func() {
			for {
				select {
				case <-stopChan:
					return
				case pathStr := <-taskChan:
					cb := self.ColumnBuffers[pathStr]
					cb.SkipRows(int64(num))
					doneChan <- 0
				}
			}
		}()
	}

	for key, _ := range self.ColumnBuffers {
		taskChan <- key
	}

	for i := 0; i < len(self.ColumnBuffers); i++ {
		<-doneChan
	}
	for i := int64(0); i < self.NP; i++ {
		stopChan <- 0
	}
	return err
}

//Read rows of parquet file and unmarshal all to dst
func (self *ParquetReader) Read(dstInterface interface{}) error {
	return self.read(dstInterface, "")
}

// Read maxReadNumber objects
func (self *ParquetReader) ReadByNumber(maxReadNumber int) ([]interface{}, error) {
	var err error
	if self.ObjType == nil {
		if self.ObjType, err = self.SchemaHandler.GetType(self.SchemaHandler.GetRootInName()); err != nil {
			return nil, err
		}
	}

	vs := reflect.MakeSlice(reflect.SliceOf(self.ObjType), maxReadNumber, maxReadNumber)
	res := reflect.New(vs.Type())
	res.Elem().Set(vs)

	if err = self.Read(res.Interface()); err != nil {
		return nil, err
	}

	ln := res.Elem().Len()
	ret := make([]interface{}, ln)
	for i := 0; i < ln; i++ {
		ret[i] = res.Elem().Index(i).Interface()
	}

	return ret, nil
}

//Read rows of parquet file and unmarshal all to dst
func (self *ParquetReader) ReadPartial(dstInterface interface{}, prefixPath string) error {
	prefixPath, err := self.SchemaHandler.ConvertToInPathStr(prefixPath)
	if err != nil {
		return err
	}

	return self.read(dstInterface, prefixPath)
}

// Read maxReadNumber partial objects
func (self *ParquetReader) ReadPartialByNumber(maxReadNumber int, prefixPath string) ([]interface{}, error) {
	var err error
	if self.ObjPartialType == nil {
		if self.ObjPartialType, err = self.SchemaHandler.GetType(prefixPath); err != nil {
			return nil, err
		}
	}

	vs := reflect.MakeSlice(reflect.SliceOf(self.ObjPartialType), maxReadNumber, maxReadNumber)
	res := reflect.New(vs.Type())
	res.Elem().Set(vs)

	if err = self.ReadPartial(res.Interface(), prefixPath); err != nil {
		return nil, err
	}

	ln := res.Elem().Len()
	ret := make([]interface{}, ln)
	for i := 0; i < ln; i++ {
		ret[i] = res.Elem().Index(i).Interface()
	}

	return ret, nil
}

//Read rows of parquet file with a prefixPath
func (self *ParquetReader) read(dstInterface interface{}, prefixPath string) error {
	var err error
	tmap := make(map[string]*layout.Table)
	locker := new(sync.Mutex)
	ot := reflect.TypeOf(dstInterface).Elem().Elem()
	num := reflect.ValueOf(dstInterface).Elem().Len()
	if num <= 0 {
		return nil
	}

	doneChan := make(chan int, self.NP)
	taskChan := make(chan string, len(self.ColumnBuffers))
	stopChan := make(chan int)

	for i := int64(0); i < self.NP; i++ {
		go func() {
			for {
				select {
				case <-stopChan:
					return
				case pathStr := <-taskChan:
					cb := self.ColumnBuffers[pathStr]
					table, _ := cb.ReadRows(int64(num))
					locker.Lock()
					if _, ok := tmap[pathStr]; ok {
						tmap[pathStr].Merge(table)
					} else {
						tmap[pathStr] = layout.NewTableFromTable(table)
						tmap[pathStr].Merge(table)
					}
					locker.Unlock()
					doneChan <- 0
				}
			}
		}()
	}

	readNum := 0
	for key, _ := range self.ColumnBuffers {
		if strings.HasPrefix(key, prefixPath) {
			taskChan <- key
			readNum++
		}
	}
	for i := 0; i < readNum; i++ {
		<-doneChan
	}

	for i := int64(0); i < self.NP; i++ {
		stopChan <- 0
	}

	dstList := make([]interface{}, self.NP)
	delta := (int64(num) + self.NP - 1) / self.NP

	doneChan = make(chan int)
	for c := int64(0); c < self.NP; c++ {
		bgn := c * delta
		end := bgn + delta
		if end > int64(num) {
			end = int64(num)
		}
		if bgn >= int64(num) {
			bgn, end = int64(num), int64(num)
		}
		go func(b, e, index int) {
			dstList[index] = reflect.New(reflect.SliceOf(ot)).Interface()
			if err2 := marshal.Unmarshal(&tmap, b, e, dstList[index], self.SchemaHandler, prefixPath); err2 != nil {
				err = err2
			}
			doneChan <- 0
		}(int(bgn), int(end), int(c))
	}
	for c := int64(0); c < self.NP; c++ {
		<-doneChan
	}

	resTmp := reflect.MakeSlice(reflect.SliceOf(ot), 0, num)
	for _, dst := range dstList {
		resTmp = reflect.AppendSlice(resTmp, reflect.ValueOf(dst).Elem())
	}
	reflect.ValueOf(dstInterface).Elem().Set(resTmp)
	return err
}

//Stop Read
func (self *ParquetReader) ReadStop() {
	for _, cb := range self.ColumnBuffers {
		if cb != nil {
			cb.PFile.Close()
		}
	}
}
<|endoftext|>


<commit_before>package reaper

import (
	"log"
	"time"

	"github.com/boltdb/bolt"
	"github.com/yosssi/boltstore/shared"
)

// Run invokes a reap function as a goroutine.
func Run(db *bolt.DB, options Options) (chan<- struct{}, <-chan struct{}) {
	options.setDefault()
	quitC, doneC := make(chan struct{}), make(chan struct{})
	go reap(db, options, quitC, doneC)
	return quitC, doneC
}

// Quit terminats the reap goroutine.
func Quit(quitC chan<- struct{}, doneC <-chan struct{}) {
	quitC <- struct{}{}
	<-doneC
}

func reap(db *bolt.DB, options Options, quitC <-chan struct{}, doneC chan<- struct{}) {
	var prevKey []byte
	for {
		err := db.View(func(tx *bolt.Tx) error {
			bucket := tx.Bucket(options.BucketName)
			if bucket == nil {
				return nil
			}

			c := bucket.Cursor()

			var i int

			for k, v := c.Seek(prevKey); ; k, v = c.Next() {
				// If we hit the end of our sessions then
				// exit and start over next time.
				if k == nil {
					prevKey = nil
					return nil
				}

				i++

				session, err := shared.Session(v)
				if err != nil {
					return err
				}

				if shared.Expired(session) {
					err := db.Update(func(txu *bolt.Tx) error {
						return txu.Bucket(options.BucketName).Delete(k)
					})
					if err != nil {
						return err
					}
				}

				if options.BatchSize == i {
					copy(prevKey, k)
					return nil
				}
			}
		})

		if err != nil {
			log.Println(err.Error())
		}

		// Check if a quit signal is sent.
		select {
		case <-quitC:
			doneC <- struct{}{}
			return
		default:
		}

		time.Sleep(options.CheckInterval)
	}
}
<commit_msg>Update reaper.go<commit_after>package reaper

import (
	"log"
	"time"

	"github.com/boltdb/bolt"
	"github.com/yosssi/boltstore/shared"
)

// Run invokes a reap function as a goroutine.
func Run(db *bolt.DB, options Options) (chan<- struct{}, <-chan struct{}) {
	options.setDefault()
	quitC, doneC := make(chan struct{}), make(chan struct{})
	go reap(db, options, quitC, doneC)
	return quitC, doneC
}

// Quit terminates the reap goroutine.
func Quit(quitC chan<- struct{}, doneC <-chan struct{}) {
	quitC <- struct{}{}
	<-doneC
}

func reap(db *bolt.DB, options Options, quitC <-chan struct{}, doneC chan<- struct{}) {
	var prevKey []byte
	for {
		err := db.View(func(tx *bolt.Tx) error {
			bucket := tx.Bucket(options.BucketName)
			if bucket == nil {
				return nil
			}

			c := bucket.Cursor()

			var i int

			for k, v := c.Seek(prevKey); ; k, v = c.Next() {
				// If we hit the end of our sessions then
				// exit and start over next time.
				if k == nil {
					prevKey = nil
					return nil
				}

				i++

				session, err := shared.Session(v)
				if err != nil {
					return err
				}

				if shared.Expired(session) {
					err := db.Update(func(txu *bolt.Tx) error {
						return txu.Bucket(options.BucketName).Delete(k)
					})
					if err != nil {
						return err
					}
				}

				if options.BatchSize == i {
					copy(prevKey, k)
					return nil
				}
			}
		})

		if err != nil {
			log.Println(err.Error())
		}

		// Check if a quit signal is sent.
		select {
		case <-quitC:
			doneC <- struct{}{}
			return
		default:
		}

		time.Sleep(options.CheckInterval)
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"testing"
)

func TestMetricMapperYAML(t *testing.T) {
	scenarios := []struct {
		config    string
		configBad bool
		mappings  map[string]map[string]string
	}{
		// Empty config.
		{},
		// Config with several mapping definitions.
		{
			config: `---
mappings:
- match: test.dispatcher.*.*.*
  labels:
    name: "dispatch_events"
    processor: "$1"
    action: "$2"
    result: "$3"
    job: "test_dispatcher"
- match: test.my-dispatch-host01.name.dispatcher.*.*.*
  labels:
    name: "host_dispatch_events"
    processor: "$1"
    action: "$2"
    result: "$3"
    job: "test_dispatcher"
- match: request_time.*.*.*.*.*.*.*.*.*.*.*.*
  labels:
    name: "tyk_http_request"
    method_and_path: "${1}"
    response_code: "${2}"
    apikey: "${3}"
    apiversion: "${4}"
    apiname: "${5}"
    apiid: "${6}"
    ipv4_t1: "${7}"
    ipv4_t2: "${8}"
    ipv4_t3: "${9}"
    ipv4_t4: "${10}"
    orgid: "${11}"
    oauthid: "${12}"
- match: "*.*"
  labels:
    name: "catchall"
    first: "$1"
    second: "$2"
    third: "$3"
    job: "$1-$2-$3"
- match: (.*)\.(.*)-(.*)\.(.*)
  match_type: regex
  labels:
    name: "proxy_requests_total"
    job: "$1"
    protocol: "$2"
    endpoint: "$3"
    result: "$4"

  `,
			mappings: map[string]map[string]string{
				"test.dispatcher.FooProcessor.send.succeeded": map[string]string{
					"name":      "dispatch_events",
					"processor": "FooProcessor",
					"action":    "send",
					"result":    "succeeded",
					"job":       "test_dispatcher",
				},
				"test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": map[string]string{
					"name":      "host_dispatch_events",
					"processor": "FooProcessor",
					"action":    "send",
					"result":    "succeeded",
					"job":       "test_dispatcher",
				},
				"request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": map[string]string{
					"name":            "tyk_http_request",
					"method_and_path": "get/threads/1/posts",
					"response_code":   "200",
					"apikey":          "00000000",
					"apiversion":      "nonversioned",
					"apiname":         "discussions",
					"apiid":           "a11bbcdf0ac64ec243658dc64b7100fb",
					"ipv4_t1":         "172",
					"ipv4_t2":         "20",
					"ipv4_t3":         "0",
					"ipv4_t4":         "1",
					"orgid":           "12ba97b7eaa1a50001000001",
					"oauthid":         "",
				},
				"foo.bar": map[string]string{
					"name":   "catchall",
					"first":  "foo",
					"second": "bar",
					"third":  "",
					"job":    "foo-bar-",
				},
				"foo.bar.baz": map[string]string{},
				"proxy-1.http-goober.success": map[string]string{
					"name":     "proxy_requests_total",
					"job":      "proxy-1",
					"protocol": "http",
					"endpoint": "goober",
					"result":   "success",
				},
			},
		},
		// Config with bad regex reference.
		{
			config: `---
mappings:
- match: test.*
  labels:
    name: "name"
    label: "$1_foo"
  `,
			mappings: map[string]map[string]string{
				"test.a": map[string]string{
					"name":  "name",
					"label": "",
				},
			},
		},
		// Config with good regex reference.
		{
			config: `
mappings:
- match: test.*
  labels:
    name: "name"
    label: "${1}_foo"
  `,
			mappings: map[string]map[string]string{
				"test.a": map[string]string{
					"name":  "name",
					"label": "a_foo",
				},
			},
		},
		// Config with bad metric line.
		{
			config: `---
mappings:
- match: bad--metric-line.*.*
  labels:
    name: "foo"
  `,
			configBad: true,
		},
		// Config with bad metric name.
		{
			config: `---
mappings:
- match: test.*.*
  labels:
    name: "0foo"
  `,
			configBad: true,
		},
		// Config with no metric name.
		{
			config: `---
mappings:
- match: test.*.*
  labels:
    this: "$1"
  `,
			configBad: true,
		},
		// Config with no mappings.
		{
			config:   ``,
			mappings: map[string]map[string]string{},
		},
		// Config without a trailing newline.
		{
			config: `mappings:
- match: test.*
  labels:
    name: "name"
    label: "${1}_foo"`,
			mappings: map[string]map[string]string{
				"test.a": map[string]string{
					"name":  "name",
					"label": "a_foo",
				},
			},
		},
		// Config with good timer type.
		{
			config: `---
mappings:
- match: test.*.*
  timer_type: summary
  labels:
    name: "foo"
  `,
			mappings: map[string]map[string]string{
				"test.*.*": map[string]string{
					"name": "foo",
				},
			},
		},
		// Config with bad timer type.
		{
			config: `---
mappings:
- match: test.*.*
  timer_type: wrong
  labels:
    name: "foo"
    `,
			configBad: true,
		},
		//Config with uncompilable regex.
		{
			config: `---
mappings:
- match: *\.foo
  match_type: regex
  labels:
    name: "foo"
    `,
			configBad: true,
		},
	}

	mapper := metricMapper{}
	for i, scenario := range scenarios {
		err := mapper.initFromYAMLString(scenario.config)
		if err != nil && !scenario.configBad {
			t.Fatalf("%d. Config load error: %s %s", i, scenario.config, err)
		}
		if err == nil && scenario.configBad {
			t.Fatalf("%d. Expected bad config, but loaded ok: %s", i, scenario.config)
		}

		for metric, mapping := range scenario.mappings {
			_, labels, present := mapper.getMapping(metric)
			if len(labels) == 0 && present {
				t.Fatalf("%d.%q: Expected metric to not be present", i, metric)
			}
			if len(labels) != len(mapping) {
				t.Fatalf("%d.%q: Expected %d labels, got %d", i, metric, len(mapping), len(labels))
			}
			for label, value := range labels {
				if mapping[label] != value {
					t.Fatalf("%d.%q: Expected labels %v, got %v", i, metric, mapping, labels)
				}
			}
		}
	}
}
<commit_msg>Add examples for quoting matches that start with `*`.<commit_after>// Copyright 2013 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"testing"
)

func TestMetricMapperYAML(t *testing.T) {
	scenarios := []struct {
		config    string
		configBad bool
		mappings  map[string]map[string]string
	}{
		// Empty config.
		{},
		// Config with several mapping definitions.
		{
			config: `---
mappings:
- match: test.dispatcher.*.*.*
  labels:
    name: "dispatch_events"
    processor: "$1"
    action: "$2"
    result: "$3"
    job: "test_dispatcher"
- match: test.my-dispatch-host01.name.dispatcher.*.*.*
  labels:
    name: "host_dispatch_events"
    processor: "$1"
    action: "$2"
    result: "$3"
    job: "test_dispatcher"
- match: request_time.*.*.*.*.*.*.*.*.*.*.*.*
  labels:
    name: "tyk_http_request"
    method_and_path: "${1}"
    response_code: "${2}"
    apikey: "${3}"
    apiversion: "${4}"
    apiname: "${5}"
    apiid: "${6}"
    ipv4_t1: "${7}"
    ipv4_t2: "${8}"
    ipv4_t3: "${9}"
    ipv4_t4: "${10}"
    orgid: "${11}"
    oauthid: "${12}"
- match: "*.*"
  labels:
    name: "catchall"
    first: "$1"
    second: "$2"
    third: "$3"
    job: "$1-$2-$3"
- match: (.*)\.(.*)-(.*)\.(.*)
  match_type: regex
  labels:
    name: "proxy_requests_total"
    job: "$1"
    protocol: "$2"
    endpoint: "$3"
    result: "$4"

  `,
			mappings: map[string]map[string]string{
				"test.dispatcher.FooProcessor.send.succeeded": map[string]string{
					"name":      "dispatch_events",
					"processor": "FooProcessor",
					"action":    "send",
					"result":    "succeeded",
					"job":       "test_dispatcher",
				},
				"test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": map[string]string{
					"name":      "host_dispatch_events",
					"processor": "FooProcessor",
					"action":    "send",
					"result":    "succeeded",
					"job":       "test_dispatcher",
				},
				"request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": map[string]string{
					"name":            "tyk_http_request",
					"method_and_path": "get/threads/1/posts",
					"response_code":   "200",
					"apikey":          "00000000",
					"apiversion":      "nonversioned",
					"apiname":         "discussions",
					"apiid":           "a11bbcdf0ac64ec243658dc64b7100fb",
					"ipv4_t1":         "172",
					"ipv4_t2":         "20",
					"ipv4_t3":         "0",
					"ipv4_t4":         "1",
					"orgid":           "12ba97b7eaa1a50001000001",
					"oauthid":         "",
				},
				"foo.bar": map[string]string{
					"name":   "catchall",
					"first":  "foo",
					"second": "bar",
					"third":  "",
					"job":    "foo-bar-",
				},
				"foo.bar.baz": map[string]string{},
				"proxy-1.http-goober.success": map[string]string{
					"name":     "proxy_requests_total",
					"job":      "proxy-1",
					"protocol": "http",
					"endpoint": "goober",
					"result":   "success",
				},
			},
		},
		// Config with bad regex reference.
		{
			config: `---
mappings:
- match: test.*
  labels:
    name: "name"
    label: "$1_foo"
  `,
			mappings: map[string]map[string]string{
				"test.a": map[string]string{
					"name":  "name",
					"label": "",
				},
			},
		},
		// Config with good regex reference.
		{
			config: `
mappings:
- match: test.*
  labels:
    name: "name"
    label: "${1}_foo"
  `,
			mappings: map[string]map[string]string{
				"test.a": map[string]string{
					"name":  "name",
					"label": "a_foo",
				},
			},
		},
		// Config with bad metric line.
		{
			config: `---
mappings:
- match: bad--metric-line.*.*
  labels:
    name: "foo"
  `,
			configBad: true,
		},
		// Config with bad metric name.
		{
			config: `---
mappings:
- match: test.*.*
  labels:
    name: "0foo"
  `,
			configBad: true,
		},
		// Config with no metric name.
		{
			config: `---
mappings:
- match: test.*.*
  labels:
    this: "$1"
  `,
			configBad: true,
		},
		// Config with no mappings.
		{
			config:   ``,
			mappings: map[string]map[string]string{},
		},
		// Config without a trailing newline.
		{
			config: `mappings:
- match: test.*
  labels:
    name: "name"
    label: "${1}_foo"`,
			mappings: map[string]map[string]string{
				"test.a": map[string]string{
					"name":  "name",
					"label": "a_foo",
				},
			},
		},
		// Config with an improperly escaped *.
		{
			config: `
mappings:
- match: *.test.*
  labels:
    name: "name"
    label: "${1}_foo"`,
			configBad: true,
		},
		// Config with a properly escaped *.
		{
			config: `
mappings:
- match: "*.test.*"
  labels:
    name: "name"
    label: "${2}_foo"`,
			mappings: map[string]map[string]string{
				"foo.test.a": map[string]string{
					"name":  "name",
					"label": "a_foo",
				},
			},
		},
		// Config with good timer type.
		{
			config: `---
mappings:
- match: test.*.*
  timer_type: summary
  labels:
    name: "foo"
  `,
			mappings: map[string]map[string]string{
				"test.*.*": map[string]string{
					"name": "foo",
				},
			},
		},
		// Config with bad timer type.
		{
			config: `---
mappings:
- match: test.*.*
  timer_type: wrong
  labels:
    name: "foo"
    `,
			configBad: true,
		},
		//Config with uncompilable regex.
		{
			config: `---
mappings:
- match: "*\.foo"
  match_type: regex
  labels:
    name: "foo"
    `,
			configBad: true,
		},
	}

	mapper := metricMapper{}
	for i, scenario := range scenarios {
		err := mapper.initFromYAMLString(scenario.config)
		if err != nil && !scenario.configBad {
			t.Fatalf("%d. Config load error: %s %s", i, scenario.config, err)
		}
		if err == nil && scenario.configBad {
			t.Fatalf("%d. Expected bad config, but loaded ok: %s", i, scenario.config)
		}

		for metric, mapping := range scenario.mappings {
			_, labels, present := mapper.getMapping(metric)
			if len(labels) == 0 && present {
				t.Fatalf("%d.%q: Expected metric to not be present", i, metric)
			}
			if len(labels) != len(mapping) {
				t.Fatalf("%d.%q: Expected %d labels, got %d", i, metric, len(mapping), len(labels))
			}
			for label, value := range labels {
				if mapping[label] != value {
					t.Fatalf("%d.%q: Expected labels %v, got %v", i, metric, mapping, labels)
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import "fmt"

func main() {
	var salutationMap map[string]string

	salutationMap = make(map[string]string)

	salutationMap['oldname'] = "azharkhan"

	myMap := map[string]string {
		"name" : "azhar",
	}

	myMap["age"] = "12"
	myMap["location"] = "Hyderbad"

	fmt.Println(myMap["location"])
	fmt.Println(salutationMap["oldname"])
}<commit_msg>Fixing map<commit_after>package main

import "fmt"

func main() {

	var salutationMap map[string]string
	salutationMap = make(map[string]string)

	salutationMap["oldname"] = "azharuddin khan"
	myMap := map[string]string {
		"name" : "azhar",
	}

	myMap["age"] = "12"

	myMap["location"] = "Hyderbad"

	fmt.Println(myMap["location"])
	fmt.Println(salutationMap["oldname"])
}<|endoftext|>