<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package s2

import (
	"fmt"
	"math"

	"github.com/golang/geo/r1"
	"github.com/golang/geo/s1"
)

// Rect represents a closed latitude-longitude rectangle.
type Rect struct {
	Lat r1.Interval
	Lng s1.Interval
}

var (
	validRectLatRange = r1.Interval{-math.Pi / 2, math.Pi / 2}
	validRectLngRange = s1.FullInterval()
)

// EmptyRect returns the empty rectangle.
func EmptyRect() Rect { return Rect{r1.EmptyInterval(), s1.EmptyInterval()} }

// FullRect returns the full rectangle.
func FullRect() Rect { return Rect{validRectLatRange, validRectLngRange} }

// RectFromLatLng constructs a rectangle containing a single point p.
func RectFromLatLng(p LatLng) Rect {
	return Rect{
		Lat: r1.Interval{p.Lat.Radians(), p.Lat.Radians()},
		Lng: s1.Interval{p.Lng.Radians(), p.Lng.Radians()},
	}
}

// RectFromCenterSize constructs a rectangle with the given size and center.
// center needs to be normalized, but size does not. The latitude
// interval of the result is clamped to [-90,90] degrees, and the longitude
// interval of the result is FullRect() if and only if the longitude size is
// 360 degrees or more.
//
// Examples of clamping (in degrees):
//   center=(80,170),  size=(40,60)   -> lat=[60,90],   lng=[140,-160]
//   center=(10,40),   size=(210,400) -> lat=[-90,90],  lng=[-180,180]
//   center=(-90,180), size=(20,50)   -> lat=[-90,-80], lng=[155,-155]
func RectFromCenterSize(center, size LatLng) Rect {
	half := LatLng{size.Lat / 2, size.Lng / 2}
	return RectFromLatLng(center).expanded(half)
}

// IsValid returns true iff the rectangle is valid.
// This requires Lat ⊆ [-π/2,π/2] and Lng ⊆ [-π,π], and Lat = ∅ iff Lng = ∅
func (r Rect) IsValid() bool {
	return math.Abs(r.Lat.Lo) <= math.Pi/2 &&
		math.Abs(r.Lat.Hi) <= math.Pi/2 &&
		r.Lng.IsValid() &&
		r.Lat.IsEmpty() == r.Lng.IsEmpty()
}

// IsEmpty reports whether the rectangle is empty.
func (r Rect) IsEmpty() bool { return r.Lat.IsEmpty() }

// IsFull reports whether the rectangle is full.
func (r Rect) IsFull() bool { return r.Lat.Equal(validRectLatRange) && r.Lng.IsFull() }

// IsPoint reports whether the rectangle is a single point.
func (r Rect) IsPoint() bool { return r.Lat.Lo == r.Lat.Hi && r.Lng.Lo == r.Lng.Hi }

// Lo returns one corner of the rectangle.
func (r Rect) Lo() LatLng {
	return LatLng{s1.Angle(r.Lat.Lo) * s1.Radian, s1.Angle(r.Lng.Lo) * s1.Radian}
}

// Hi returns the other corner of the rectangle.
func (r Rect) Hi() LatLng {
	return LatLng{s1.Angle(r.Lat.Hi) * s1.Radian, s1.Angle(r.Lng.Hi) * s1.Radian}
}

// Center returns the center of the rectangle.
func (r Rect) Center() LatLng {
	return LatLng{s1.Angle(r.Lat.Center()) * s1.Radian, s1.Angle(r.Lng.Center()) * s1.Radian}
}

// Size returns the size of the Rect.
func (r Rect) Size() LatLng {
	return LatLng{s1.Angle(r.Lat.Length()) * s1.Radian, s1.Angle(r.Lng.Length()) * s1.Radian}
}

// Area returns the surface area of the Rect.
func (r Rect) Area() float64 {
	if r.IsEmpty() {
		return 0
	}
	capDiff := math.Abs(math.Sin(r.Lat.Hi) - math.Sin(r.Lat.Lo))
	return r.Lng.Length() * capDiff
}

// ContainsLatLng reports whether the given LatLng is within the Rect.
func (r Rect) ContainsLatLng(ll LatLng) bool {
	if !ll.IsValid() {
		return false
	}
	return r.Lat.Contains(ll.Lat.Radians()) && r.Lng.Contains(ll.Lng.Radians())
}

// AddPoint increases the size of the rectangle to include the given point.
func (r Rect) AddPoint(ll LatLng) Rect {
	if !ll.IsValid() {
		return r
	}
	return Rect{
		Lat: r.Lat.AddPoint(ll.Lat.Radians()),
		Lng: r.Lng.AddPoint(ll.Lng.Radians()),
	}
}

// expanded returns a rectangle that contains all points whose latitude distance from
// this rectangle is at most margin.Lat, and whose longitude distance from
// this rectangle is at most margin.Lng. In particular, latitudes are
// clamped while longitudes are wrapped. Any expansion of an empty rectangle
// remains empty. Both components of margin must be non-negative.
//
// Note that if an expanded rectangle contains a pole, it may not contain
// all possible lat/lng representations of that pole, e.g., both points [π/2,0]
// and [π/2,1] represent the same pole, but they might not be contained by the
// same Rect.
//
// If you are trying to grow a rectangle by a certain distance on the
// sphere (e.g. 5km), refer to the ConvolveWithCap() C++ method implementation
// instead.
func (r Rect) expanded(margin LatLng) Rect {
	return Rect{
		Lat: r.Lat.Expanded(margin.Lat.Radians()).Intersection(validRectLatRange),
		Lng: r.Lng.Expanded(margin.Lng.Radians()),
	}
}

func (r Rect) String() string { return fmt.Sprintf("[Lo%v, Hi%v]", r.Lo(), r.Hi()) }

// PolarClosure returns the rectangle unmodified if it does not include either pole.
// If it includes either pole, PolarClosure returns an expansion of the rectangle along
// the longitudinal range to include all possible representations of the contained poles.
func (r Rect) PolarClosure() Rect {
	if r.Lat.Lo == -math.Pi/2 || r.Lat.Hi == math.Pi/2 {
		return Rect{r.Lat, s1.FullInterval()}
	}
	return r
}

// BUG(dsymonds): The major differences from the C++ version are:
//   - almost everything
<commit_msg>s2: Add Rect.RectBound method.<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package s2

import (
	"fmt"
	"math"

	"github.com/golang/geo/r1"
	"github.com/golang/geo/s1"
)

// Rect represents a closed latitude-longitude rectangle.
type Rect struct {
	Lat r1.Interval
	Lng s1.Interval
}

var (
	validRectLatRange = r1.Interval{-math.Pi / 2, math.Pi / 2}
	validRectLngRange = s1.FullInterval()
)

// EmptyRect returns the empty rectangle.
func EmptyRect() Rect { return Rect{r1.EmptyInterval(), s1.EmptyInterval()} }

// FullRect returns the full rectangle.
func FullRect() Rect { return Rect{validRectLatRange, validRectLngRange} }

// RectFromLatLng constructs a rectangle containing a single point p.
func RectFromLatLng(p LatLng) Rect {
	return Rect{
		Lat: r1.Interval{p.Lat.Radians(), p.Lat.Radians()},
		Lng: s1.Interval{p.Lng.Radians(), p.Lng.Radians()},
	}
}

// RectFromCenterSize constructs a rectangle with the given size and center.
// center needs to be normalized, but size does not. The latitude
// interval of the result is clamped to [-90,90] degrees, and the longitude
// interval of the result is FullRect() if and only if the longitude size is
// 360 degrees or more.
//
// Examples of clamping (in degrees):
//   center=(80,170),  size=(40,60)   -> lat=[60,90],   lng=[140,-160]
//   center=(10,40),   size=(210,400) -> lat=[-90,90],  lng=[-180,180]
//   center=(-90,180), size=(20,50)   -> lat=[-90,-80], lng=[155,-155]
func RectFromCenterSize(center, size LatLng) Rect {
	half := LatLng{size.Lat / 2, size.Lng / 2}
	return RectFromLatLng(center).expanded(half)
}

// IsValid returns true iff the rectangle is valid.
// This requires Lat ⊆ [-π/2,π/2] and Lng ⊆ [-π,π], and Lat = ∅ iff Lng = ∅
func (r Rect) IsValid() bool {
	return math.Abs(r.Lat.Lo) <= math.Pi/2 &&
		math.Abs(r.Lat.Hi) <= math.Pi/2 &&
		r.Lng.IsValid() &&
		r.Lat.IsEmpty() == r.Lng.IsEmpty()
}

// IsEmpty reports whether the rectangle is empty.
func (r Rect) IsEmpty() bool { return r.Lat.IsEmpty() }

// IsFull reports whether the rectangle is full.
func (r Rect) IsFull() bool { return r.Lat.Equal(validRectLatRange) && r.Lng.IsFull() }

// IsPoint reports whether the rectangle is a single point.
func (r Rect) IsPoint() bool { return r.Lat.Lo == r.Lat.Hi && r.Lng.Lo == r.Lng.Hi }

// Lo returns one corner of the rectangle.
func (r Rect) Lo() LatLng {
	return LatLng{s1.Angle(r.Lat.Lo) * s1.Radian, s1.Angle(r.Lng.Lo) * s1.Radian}
}

// Hi returns the other corner of the rectangle.
func (r Rect) Hi() LatLng {
	return LatLng{s1.Angle(r.Lat.Hi) * s1.Radian, s1.Angle(r.Lng.Hi) * s1.Radian}
}

// Center returns the center of the rectangle.
func (r Rect) Center() LatLng {
	return LatLng{s1.Angle(r.Lat.Center()) * s1.Radian, s1.Angle(r.Lng.Center()) * s1.Radian}
}

// Size returns the size of the Rect.
func (r Rect) Size() LatLng {
	return LatLng{s1.Angle(r.Lat.Length()) * s1.Radian, s1.Angle(r.Lng.Length()) * s1.Radian}
}

// Area returns the surface area of the Rect.
func (r Rect) Area() float64 {
	if r.IsEmpty() {
		return 0
	}
	capDiff := math.Abs(math.Sin(r.Lat.Hi) - math.Sin(r.Lat.Lo))
	return r.Lng.Length() * capDiff
}

// ContainsLatLng reports whether the given LatLng is within the Rect.
func (r Rect) ContainsLatLng(ll LatLng) bool {
	if !ll.IsValid() {
		return false
	}
	return r.Lat.Contains(ll.Lat.Radians()) && r.Lng.Contains(ll.Lng.Radians())
}

// AddPoint increases the size of the rectangle to include the given point.
func (r Rect) AddPoint(ll LatLng) Rect {
	if !ll.IsValid() {
		return r
	}
	return Rect{
		Lat: r.Lat.AddPoint(ll.Lat.Radians()),
		Lng: r.Lng.AddPoint(ll.Lng.Radians()),
	}
}

// expanded returns a rectangle that contains all points whose latitude distance from
// this rectangle is at most margin.Lat, and whose longitude distance from
// this rectangle is at most margin.Lng. In particular, latitudes are
// clamped while longitudes are wrapped. Any expansion of an empty rectangle
// remains empty. Both components of margin must be non-negative.
//
// Note that if an expanded rectangle contains a pole, it may not contain
// all possible lat/lng representations of that pole, e.g., both points [π/2,0]
// and [π/2,1] represent the same pole, but they might not be contained by the
// same Rect.
//
// If you are trying to grow a rectangle by a certain distance on the
// sphere (e.g. 5km), refer to the ConvolveWithCap() C++ method implementation
// instead.
func (r Rect) expanded(margin LatLng) Rect {
	return Rect{
		Lat: r.Lat.Expanded(margin.Lat.Radians()).Intersection(validRectLatRange),
		Lng: r.Lng.Expanded(margin.Lng.Radians()),
	}
}

func (r Rect) String() string { return fmt.Sprintf("[Lo%v, Hi%v]", r.Lo(), r.Hi()) }

// PolarClosure returns the rectangle unmodified if it does not include either pole.
// If it includes either pole, PolarClosure returns an expansion of the rectangle along
// the longitudinal range to include all possible representations of the contained poles.
func (r Rect) PolarClosure() Rect {
	if r.Lat.Lo == -math.Pi/2 || r.Lat.Hi == math.Pi/2 {
		return Rect{r.Lat, s1.FullInterval()}
	}
	return r
}

// RectBound returns itself.
func (r Rect) RectBound() Rect {
	return r
}

// BUG(dsymonds): The major differences from the C++ version are:
//   - almost everything
<|endoftext|>


<commit_before>package main

import (
	"crypto/sha1"
	"encoding/hex"
	"errors"
	"flag"
	"fmt"
	"github.com/bmizerany/pat"
	"github.com/fzzbt/radix/redis"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"strconv"
	"strings"
	"syscall"
	"time"
	"crypto/tls"
)

// HTTP headers struct
type Headers struct {
	contentType  string
	lastModified string
	cacheControl string
}

// The URL for the default avatar
const defaultAvatarUrl = "//linuxfr.org/images/default-avatar.png"

// The maximal size for an image is 5MB
const maxSize = 5 * (1 << 20)

// The directory for caching files
var directory string

// The connection to redis
var connection *redis.Client

// Check if an URL is valid and not temporary in error
func urlStatus(uri string) error {
	ok, err := connection.Hexists("img/"+uri, "created_at").Bool()
	if err != nil {
		return err
	}
	if !ok {
		return errors.New("Invalid URL")
	}

	str, err := connection.Get("img/err/" + uri).Str()
	if err == nil {
		return errors.New(str)
	}

	return nil
}

// Generate a key for cache from a string
func generateKeyForCache(s string) string {
	h := sha1.New()
	io.WriteString(h, s)
	key := h.Sum(nil)

	// Use 3 levels of hasing to avoid having too many files in the same directory
	return fmt.Sprintf("%s/%x/%x/%x/%x", directory, key[0:1], key[1:2], key[2:3], key[3:])
}

// Fetch image from cache
func fetchImageFromCache(uri string) (headers Headers, body []byte, ok bool) {
	ok = false

	contentType, err := connection.Hget("img/"+uri, "type").Str()
	if err != nil {
		return
	}

	filename := generateKeyForCache(uri)
	stat, err := os.Stat(filename)
	if err != nil {
		return
	}

	headers.contentType = contentType
	headers.lastModified = stat.ModTime().Format(time.RFC1123)

	body, err = ioutil.ReadFile(filename)
	ok = err == nil

	present, err := connection.Exists("img/updated/" + uri).Bool()
	if err == nil && !present {
		go fetchImageFromServer(uri)
	}

	return
}

// Save the body and the content-type header in cache
func saveImageInCache(uri string, headers Headers, body []byte) {
	go func() {
		filename := generateKeyForCache(uri)
		dirname := path.Dir(filename)
		err := os.MkdirAll(dirname, 0755)
		if err != nil {
			return
		}

		// Save the body on disk
		err = ioutil.WriteFile(filename, body, 0644)
		if err != nil {
			log.Printf("Error while writing %s\n", filename)
			return
		}

		// And other infos in redis
		connection.Hset("img/"+uri, "type", headers.contentType)
		connection.Set("img/updated/"+uri, headers.lastModified)
		connection.Expire("img/updated/"+uri, 600)
	}()
}

// Save the error in redis for 10 minutes
func saveErrorInCache(uri string, err error) {
	go func() {
		connection.Set("img/err/"+uri, err.Error())
		connection.Expire("img/err/"+uri, 600)
	}()
}

// Fetch the image from the distant server
func fetchImageFromServer(uri string) (headers Headers, body []byte, err error) {
	// Accepts any certificate in HTTPS
	cfg := &tls.Config{InsecureSkipVerify: true}
	tr := &http.Transport{TLSClientConfig: cfg}
	client := &http.Client{Transport: tr}
	res, err := client.Get(uri)
	if err != nil {
		return
	}
	if res.StatusCode != 200 {
		log.Printf("Status code of %s is: %d\n", uri, res.StatusCode)
		err = errors.New("Unexpected status code")
		saveErrorInCache(uri, err)
		return
	}

	defer res.Body.Close()
	body, err = ioutil.ReadAll(res.Body)
	if err != nil {
		return
	}
	if res.ContentLength > maxSize {
		log.Printf("Exceeded max size for %s: %d\n", uri, res.ContentLength)
		err = errors.New("Exceeded max size")
		saveErrorInCache(uri, err)
		return
	}
	contentType := res.Header.Get("Content-Type")
	if contentType[0:5] != "image" {
		log.Printf("%s has an invalid content-type: %s\n", uri, contentType)
		err = errors.New("Invalid content-type")
		saveErrorInCache(uri, err)
		return
	}
	log.Printf("Fetch %s (%s)\n", uri, contentType)

	headers.contentType = contentType
	headers.lastModified = time.Now().Format(time.RFC1123)
	if urlStatus(uri) == nil {
		saveImageInCache(uri, headers, body)
	}
	return
}

// Fetch image from cache if available, or from the server
func fetchImage(uri string) (headers Headers, body []byte, err error) {
	err = urlStatus(uri)
	if err != nil {
		return
	}

	headers, body, ok := fetchImageFromCache(uri)
	if !ok {
		headers, body, err = fetchImageFromServer(uri)
	}
	headers.cacheControl = "public, max-age=600"

	return
}

// Receive an HTTP request, fetch the image and respond with it
func Image(w http.ResponseWriter, r *http.Request, fn func()) {
	encoded_url := r.URL.Query().Get(":encoded_url")
	chars, err := hex.DecodeString(encoded_url)
	if err != nil {
		log.Printf("Invalid URL %s\n", encoded_url)
		http.Error(w, "Invalid parameters", 400)
		return
	}
	uri := string(chars)

	headers, body, err := fetchImage(uri)
	if err != nil {
		fn()
		return
	}
	if headers.lastModified == r.Header.Get("If-Modified-Since") {
		w.WriteHeader(http.StatusNotModified)
		return
	}
	w.Header().Add("Content-Type", headers.contentType)
	w.Header().Add("Last-Modified", headers.lastModified)
	w.Header().Add("Cache-Control", headers.cacheControl)
	w.Write(body)
}

// Receive an HTTP request for an image and respond with it
func Img(w http.ResponseWriter, r *http.Request) {
	fn := func() {
		http.NotFound(w, r)
	}
	Image(w, r, fn)
}

// Receive an HTTP request for an avatar and respond with it
func Avatar(w http.ResponseWriter, r *http.Request) {
	fn := func() {
		w.Header().Set("Location", defaultAvatarUrl)
		w.WriteHeader(http.StatusFound)
	}
	Image(w, r, fn)
}

// Returns 200 OK if the server is running (for monitoring)
func Status(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "OK")
}

func main() {
	// Parse the command-line
	var addr string
	var logs string
	var conn string
	flag.StringVar(&addr, "a", "127.0.0.1:8000", "Bind to this address:port")
	flag.StringVar(&logs, "l", "-", "Use this file for logs")
	flag.StringVar(&conn, "r", "localhost:6379/0", "The redis database to use for caching meta")
	flag.StringVar(&directory, "d", "cache", "The directory for the caching files")
	flag.Parse()

	// Logging
	if logs != "-" {
		f, err := os.OpenFile(logs, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644)
		if err != nil {
			log.Fatal("OpenFile: ", err)
		}
		syscall.Dup2(int(f.Fd()), int(os.Stdout.Fd()))
		syscall.Dup2(int(f.Fd()), int(os.Stderr.Fd()))
	}

	// Redis
	parts := strings.Split(conn, "/")
	host := parts[0]
	db := 0
	if len(parts) >= 2 {
		db, _ = strconv.Atoi(parts[1])
	}
	cfg := redis.DefaultConfig()
	cfg.Database = db
	cfg.Address = host
	connection = redis.NewClient(cfg)
	defer connection.Close()

	// Routing
	m := pat.New()
	m.Get("/status", http.HandlerFunc(Status))
	m.Get("/img/:encoded_url/:filename", http.HandlerFunc(Img))
	m.Get("/img/:encoded_url", http.HandlerFunc(Img))
	m.Get("/avatars/:encoded_url/:filename", http.HandlerFunc(Avatar))
	m.Get("/avatars/:encoded_url", http.HandlerFunc(Avatar))
	http.Handle("/", m)

	// Start the HTTP server
	log.Printf("Listening on http://%s/\n", addr)
	err := http.ListenAndServe(addr, nil)
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
}
<commit_msg>Block invalid images<commit_after>package main

import (
	"crypto/sha1"
	"crypto/tls"
	"encoding/hex"
	"errors"
	"flag"
	"fmt"
	"github.com/bmizerany/pat"
	"github.com/fzzbt/radix/redis"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"strconv"
	"strings"
	"syscall"
	"time"
)

// HTTP headers struct
type Headers struct {
	contentType  string
	lastModified string
	cacheControl string
}

// The URL for the default avatar
const defaultAvatarUrl = "//linuxfr.org/images/default-avatar.png"

// The maximal size for an image is 5MB
const maxSize = 5 * (1 << 20)

// The directory for caching files
var directory string

// The connection to redis
var connection *redis.Client

// Check if an URL is valid and not temporary in error
func urlStatus(uri string) error {
	ok, err := connection.Hexists("img/"+uri, "created_at").Bool()
	if err != nil {
		return err
	}
	if !ok {
		return errors.New("Invalid URL")
	}

	status, err := connection.Hget("img/"+uri, "status").Str()
	if err == nil && status == "Blocked" {
		return errors.New("Invalid URL")
	}

	str, err := connection.Get("img/err/" + uri).Str()
	if err == nil {
		return errors.New(str)
	}

	return nil
}

// Generate a key for cache from a string
func generateKeyForCache(s string) string {
	h := sha1.New()
	io.WriteString(h, s)
	key := h.Sum(nil)

	// Use 3 levels of hasing to avoid having too many files in the same directory
	return fmt.Sprintf("%s/%x/%x/%x/%x", directory, key[0:1], key[1:2], key[2:3], key[3:])
}

// Fetch image from cache
func fetchImageFromCache(uri string) (headers Headers, body []byte, ok bool) {
	ok = false

	contentType, err := connection.Hget("img/"+uri, "type").Str()
	if err != nil {
		return
	}

	filename := generateKeyForCache(uri)
	stat, err := os.Stat(filename)
	if err != nil {
		return
	}

	headers.contentType = contentType
	headers.lastModified = stat.ModTime().Format(time.RFC1123)

	body, err = ioutil.ReadFile(filename)
	ok = err == nil

	present, err := connection.Exists("img/updated/" + uri).Bool()
	if err == nil && !present {
		go fetchImageFromServer(uri)
	}

	return
}

// Save the body and the content-type header in cache
func saveImageInCache(uri string, headers Headers, body []byte) {
	go func() {
		filename := generateKeyForCache(uri)
		dirname := path.Dir(filename)
		err := os.MkdirAll(dirname, 0755)
		if err != nil {
			return
		}

		// Save the body on disk
		err = ioutil.WriteFile(filename, body, 0644)
		if err != nil {
			log.Printf("Error while writing %s\n", filename)
			return
		}

		// And other infos in redis
		connection.Hset("img/"+uri, "type", headers.contentType)
		connection.Set("img/updated/"+uri, headers.lastModified)
		connection.Expire("img/updated/"+uri, 600)
	}()
}

// Save the error in redis for 10 minutes
func saveErrorInCache(uri string, err error) {
	go func() {
		connection.Set("img/err/"+uri, err.Error())
		connection.Expire("img/err/"+uri, 600)
	}()
}

// Fetch the image from the distant server
func fetchImageFromServer(uri string) (headers Headers, body []byte, err error) {
	// Accepts any certificate in HTTPS
	cfg := &tls.Config{InsecureSkipVerify: true}
	tr := &http.Transport{TLSClientConfig: cfg}
	client := &http.Client{Transport: tr}
	res, err := client.Get(uri)
	if err != nil {
		return
	}
	if res.StatusCode != 200 {
		log.Printf("Status code of %s is: %d\n", uri, res.StatusCode)
		err = errors.New("Unexpected status code")
		saveErrorInCache(uri, err)
		return
	}

	defer res.Body.Close()
	body, err = ioutil.ReadAll(res.Body)
	if err != nil {
		return
	}
	if res.ContentLength > maxSize {
		log.Printf("Exceeded max size for %s: %d\n", uri, res.ContentLength)
		err = errors.New("Exceeded max size")
		saveErrorInCache(uri, err)
		return
	}
	contentType := res.Header.Get("Content-Type")
	if contentType[0:5] != "image" {
		log.Printf("%s has an invalid content-type: %s\n", uri, contentType)
		err = errors.New("Invalid content-type")
		saveErrorInCache(uri, err)
		return
	}
	log.Printf("Fetch %s (%s)\n", uri, contentType)

	headers.contentType = contentType
	headers.lastModified = time.Now().Format(time.RFC1123)
	if urlStatus(uri) == nil {
		saveImageInCache(uri, headers, body)
	}
	return
}

// Fetch image from cache if available, or from the server
func fetchImage(uri string) (headers Headers, body []byte, err error) {
	err = urlStatus(uri)
	if err != nil {
		return
	}

	headers, body, ok := fetchImageFromCache(uri)
	if !ok {
		headers, body, err = fetchImageFromServer(uri)
	}
	headers.cacheControl = "public, max-age=600"

	return
}

// Receive an HTTP request, fetch the image and respond with it
func Image(w http.ResponseWriter, r *http.Request, fn func()) {
	encoded_url := r.URL.Query().Get(":encoded_url")
	chars, err := hex.DecodeString(encoded_url)
	if err != nil {
		log.Printf("Invalid URL %s\n", encoded_url)
		http.Error(w, "Invalid parameters", 400)
		return
	}
	uri := string(chars)

	headers, body, err := fetchImage(uri)
	if err != nil {
		fn()
		return
	}
	if headers.lastModified == r.Header.Get("If-Modified-Since") {
		w.WriteHeader(http.StatusNotModified)
		return
	}
	w.Header().Add("Content-Type", headers.contentType)
	w.Header().Add("Last-Modified", headers.lastModified)
	w.Header().Add("Cache-Control", headers.cacheControl)
	w.Write(body)
}

// Receive an HTTP request for an image and respond with it
func Img(w http.ResponseWriter, r *http.Request) {
	fn := func() {
		http.NotFound(w, r)
	}
	Image(w, r, fn)
}

// Receive an HTTP request for an avatar and respond with it
func Avatar(w http.ResponseWriter, r *http.Request) {
	fn := func() {
		w.Header().Set("Location", defaultAvatarUrl)
		w.WriteHeader(http.StatusFound)
	}
	Image(w, r, fn)
}

// Returns 200 OK if the server is running (for monitoring)
func Status(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "OK")
}

func main() {
	// Parse the command-line
	var addr string
	var logs string
	var conn string
	flag.StringVar(&addr, "a", "127.0.0.1:8000", "Bind to this address:port")
	flag.StringVar(&logs, "l", "-", "Use this file for logs")
	flag.StringVar(&conn, "r", "localhost:6379/0", "The redis database to use for caching meta")
	flag.StringVar(&directory, "d", "cache", "The directory for the caching files")
	flag.Parse()

	// Logging
	if logs != "-" {
		f, err := os.OpenFile(logs, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644)
		if err != nil {
			log.Fatal("OpenFile: ", err)
		}
		syscall.Dup2(int(f.Fd()), int(os.Stdout.Fd()))
		syscall.Dup2(int(f.Fd()), int(os.Stderr.Fd()))
	}

	// Redis
	parts := strings.Split(conn, "/")
	host := parts[0]
	db := 0
	if len(parts) >= 2 {
		db, _ = strconv.Atoi(parts[1])
	}
	cfg := redis.DefaultConfig()
	cfg.Database = db
	cfg.Address = host
	connection = redis.NewClient(cfg)
	defer connection.Close()

	// Routing
	m := pat.New()
	m.Get("/status", http.HandlerFunc(Status))
	m.Get("/img/:encoded_url/:filename", http.HandlerFunc(Img))
	m.Get("/img/:encoded_url", http.HandlerFunc(Img))
	m.Get("/avatars/:encoded_url/:filename", http.HandlerFunc(Avatar))
	m.Get("/avatars/:encoded_url", http.HandlerFunc(Avatar))
	http.Handle("/", m)

	// Start the HTTP server
	log.Printf("Listening on http://%s/\n", addr)
	err := http.ListenAndServe(addr, nil)
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"

	"crypto/rand"
	"encoding/hex"

	"github.com/go-martini/martini"
	_ "github.com/lib/pq"
	"github.com/mark-adams/inc/backends"
)

var app *martini.ClassicMartini

func getRandomID() (string, error) {
	newID := make([]byte, 16)
	_, err := rand.Read(newID)
	if err != nil {
		return "", err
	}

	return hex.EncodeToString(newID), nil
}

func init() {

	app = martini.Classic()

	app.Get("/_healthcheck", func() string {
		return "OK"
	})

	app.Post("/new", func() (int, string) {
		id, err := getRandomID()
		if err != nil {
			log.Printf("Error: %s", err)
			return 500, "😞 Something bad happened... try again?"
		}
		db, err := backends.GetBackend()
		if err != nil {
			log.Printf("Database error: %s", err)
			return 500, err.Error()
		}

		err = db.CreateToken(id)
		if err != nil {
			log.Printf("Insert error: %s", err)
			return 500, err.Error()
		}
		return 201, id
	})

	app.Put("/(?P<token>[a-zA-Z0-9]{32})", func(params martini.Params) (int, string) {
		db, err := backends.GetBackend()
		if err != nil {
			return 500, err.Error()
		}
		count, err := db.IncrementAndGetToken(params["token"])
		if err != nil {
			return 500, err.Error()
		}

		return 200, fmt.Sprintf("%d", count)
	})
}

func main() {
	db, err := backends.GetBackend()
	if err != nil {
		log.Fatal(err)
	}

	// Create the database schema if needed
	err = db.CreateSchema()
	if err != nil {
		log.Fatal(err)
	}

	app.Run()
}
<commit_msg>Add statsd metrics for incrementing tokens<commit_after>package main

import (
	"fmt"
	"log"

	"crypto/rand"
	"encoding/hex"

	"github.com/go-martini/martini"
	_ "github.com/lib/pq"
	"github.com/mark-adams/inc/backends"
	"gopkg.in/alexcesaro/statsd.v2"
)

var app *martini.ClassicMartini

func getRandomID() (string, error) {
	newID := make([]byte, 16)
	_, err := rand.Read(newID)
	if err != nil {
		return "", err
	}

	return hex.EncodeToString(newID), nil
}

func init() {

	app = martini.Classic()
	statsd, err := statsd.New()
	if err != nil {
		log.Printf("error initializing metrics: %s", err)
	}

	app.Get("/_healthcheck", func() string {
		return "OK"
	})

	app.Post("/new", func() (int, string) {
		statsd.Increment("inc.api.create_token")

		id, err := getRandomID()
		if err != nil {
			log.Printf("Error: %s", err)
			return 500, "😞 Something bad happened... try again?"
		}
		db, err := backends.GetBackend()
		if err != nil {
			log.Printf("Database error: %s", err)
			return 500, err.Error()
		}

		err = db.CreateToken(id)
		if err != nil {
			log.Printf("Insert error: %s", err)
			return 500, err.Error()
		}
		return 201, id
	})

	app.Put("/(?P<token>[a-zA-Z0-9]{32})", func(params martini.Params) (int, string) {
		statsd.Increment("inc.api.increment_token")

		db, err := backends.GetBackend()
		if err != nil {
			return 500, err.Error()
		}
		count, err := db.IncrementAndGetToken(params["token"])
		if err != nil {
			return 500, err.Error()
		}

		return 200, fmt.Sprintf("%d", count)
	})
}

func main() {
	db, err := backends.GetBackend()
	if err != nil {
		log.Fatal(err)
	}

	// Create the database schema if needed
	err = db.CreateSchema()
	if err != nil {
		log.Fatal(err)
	}

	app.Run()
}
<|endoftext|>


<commit_before>package auth

import (
	"bufio"
	"net/http"
	"net/url"
	"strings"
	"testing"
	"time"
)

func TestAuthDigest(t *testing.T) {
	secrets := HtdigestFileProvider("test.htdigest")
	da := &DigestAuth{Opaque: "U7H+ier3Ae8Skd/g",
		Realm:   "example.com",
		Secrets: secrets,
		clients: map[string]*digest_client{}}
	r := &http.Request{}
	r.Method = "GET"
	if u, _ := da.CheckAuth(r); u != "" {
		t.Fatal("non-empty auth for empty request header")
	}
	r.Header = http.Header(make(map[string][]string))
	r.Header.Set("Authorization", "Digest blabla")
	if u, _ := da.CheckAuth(r); u != "" {
		t.Fatal("non-empty auth for bad request header")
	}
	r.Header.Set("Authorization", `Digest username="test", realm="example.com", nonce="Vb9BP/h81n3GpTTB", uri="/t2", cnonce="NjE4MTM2", nc=00000001, qop="auth", response="ffc357c4eba74773c8687e0bc724c9a3", opaque="U7H+ier3Ae8Skd/g", algorithm="MD5"`)
	if u, _ := da.CheckAuth(r); u != "" {
		t.Fatal("non-empty auth for unknown client")
	}

	r.URL, _ = url.Parse("/t2")
	da.clients["Vb9BP/h81n3GpTTB"] = &digest_client{nc: 0, last_seen: time.Now().UnixNano()}
	if u, _ := da.CheckAuth(r); u != "test" {
		t.Fatal("empty auth for legitimate client")
	}
	if u, _ := da.CheckAuth(r); u != "" {
		t.Fatal("non-empty auth for outdated nc")
	}

	r.URL, _ = url.Parse("/")
	da.clients["Vb9BP/h81n3GpTTB"] = &digest_client{nc: 0, last_seen: time.Now().UnixNano()}
	if u, _ := da.CheckAuth(r); u != "" {
		t.Fatal("non-empty auth for bad request path")
	}

	r.URL, _ = url.Parse("/t3")
	da.clients["Vb9BP/h81n3GpTTB"] = &digest_client{nc: 0, last_seen: time.Now().UnixNano()}
	if u, _ := da.CheckAuth(r); u != "" {
		t.Fatal("non-empty auth for bad request path")
	}

	da.clients["+RbVXSbIoa1SaJk1"] = &digest_client{nc: 0, last_seen: time.Now().UnixNano()}
	r.Header.Set("Authorization", `Digest username="test", realm="example.com", nonce="+RbVXSbIoa1SaJk1", uri="/", cnonce="NjE4NDkw", nc=00000001, qop="auth", response="c08918024d7faaabd5424654c4e3ad1c", opaque="U7H+ier3Ae8Skd/g", algorithm="MD5"`)
	if u, _ := da.CheckAuth(r); u != "test" {
		t.Fatal("empty auth for valid request in subpath")
	}
}

func TestDigestAuthParams(t *testing.T) {
	body := `GET http://fonts.googleapis.com/css?family=Source+Sans+Pro:400,700,400italic,700italic|Source+Code+Pro HTTP/1.1
Host: fonts.googleapis.com
User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:43.0) Gecko/20100101 Firefox/43.0
Accept: text/css,/;q=0.1
Accept-Language: en-US,en;q=0.5
Accept-Encoding: gzip, deflate
Referer: http://elm-lang.org/assets/style.css
Authorization: Digest username="test", realm="", nonce="FRPnGdb8lvM1UHhi", uri="/css?family=Source+Sans+Pro:400,700,400italic,700italic|Source+Code+Pro", algorithm=MD5, response="fdcdd78e5b306ffed343d0ec3967f2e5", opaque="lEgVjogmIar2fg/t", qop=auth, nc=00000001, cnonce="e76b05db27a3b323"
Connection: keep-alive

`
	req, _ := http.ReadRequest(bufio.NewReader(strings.NewReader(body)))
	params := DigestAuthParams(req)
	if params["uri"] != "/css?family=Source+Sans+Pro:400,700,400italic,700italic|Source+Code+Pro" {
		t.Fatal("failed to parse uri with embedded commas")
	}

}
<commit_msg>Minor style fixes for digest_test.go.<commit_after>package auth

import (
	"bufio"
	"net/http"
	"net/url"
	"strings"
	"testing"
	"time"
)

func TestAuthDigest(t *testing.T) {
	secrets := HtdigestFileProvider("test.htdigest")
	da := &DigestAuth{Opaque: "U7H+ier3Ae8Skd/g",
		Realm:   "example.com",
		Secrets: secrets,
		clients: map[string]*digest_client{}}
	r := &http.Request{}
	r.Method = "GET"
	if u, _ := da.CheckAuth(r); u != "" {
		t.Fatal("non-empty auth for empty request header")
	}
	r.Header = http.Header(make(map[string][]string))
	r.Header.Set("Authorization", "Digest blabla")
	if u, _ := da.CheckAuth(r); u != "" {
		t.Fatal("non-empty auth for bad request header")
	}
	r.Header.Set("Authorization", `Digest username="test", realm="example.com", nonce="Vb9BP/h81n3GpTTB", uri="/t2", cnonce="NjE4MTM2", nc=00000001, qop="auth", response="ffc357c4eba74773c8687e0bc724c9a3", opaque="U7H+ier3Ae8Skd/g", algorithm="MD5"`)
	if u, _ := da.CheckAuth(r); u != "" {
		t.Fatal("non-empty auth for unknown client")
	}

	r.URL, _ = url.Parse("/t2")
	da.clients["Vb9BP/h81n3GpTTB"] = &digest_client{nc: 0, last_seen: time.Now().UnixNano()}
	if u, _ := da.CheckAuth(r); u != "test" {
		t.Fatal("empty auth for legitimate client")
	}
	if u, _ := da.CheckAuth(r); u != "" {
		t.Fatal("non-empty auth for outdated nc")
	}

	r.URL, _ = url.Parse("/")
	da.clients["Vb9BP/h81n3GpTTB"] = &digest_client{nc: 0, last_seen: time.Now().UnixNano()}
	if u, _ := da.CheckAuth(r); u != "" {
		t.Fatal("non-empty auth for bad request path")
	}

	r.URL, _ = url.Parse("/t3")
	da.clients["Vb9BP/h81n3GpTTB"] = &digest_client{nc: 0, last_seen: time.Now().UnixNano()}
	if u, _ := da.CheckAuth(r); u != "" {
		t.Fatal("non-empty auth for bad request path")
	}

	da.clients["+RbVXSbIoa1SaJk1"] = &digest_client{nc: 0, last_seen: time.Now().UnixNano()}
	r.Header.Set("Authorization", `Digest username="test", realm="example.com", nonce="+RbVXSbIoa1SaJk1", uri="/", cnonce="NjE4NDkw", nc=00000001, qop="auth", response="c08918024d7faaabd5424654c4e3ad1c", opaque="U7H+ier3Ae8Skd/g", algorithm="MD5"`)
	if u, _ := da.CheckAuth(r); u != "test" {
		t.Fatal("empty auth for valid request in subpath")
	}
}

func TestDigestAuthParams(t *testing.T) {
	body := `GET http://fonts.googleapis.com/css?family=Source+Sans+Pro:400,700,400italic,700italic|Source+Code+Pro HTTP/1.1
Host: fonts.googleapis.com
User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:43.0) Gecko/20100101 Firefox/43.0
Accept: text/css,/;q=0.1
Accept-Language: en-US,en;q=0.5
Accept-Encoding: gzip, deflate
Referer: http://elm-lang.org/assets/style.css
Authorization: Digest username="test", realm="", nonce="FRPnGdb8lvM1UHhi", uri="/css?family=Source+Sans+Pro:400,700,400italic,700italic|Source+Code+Pro", algorithm=MD5, response="fdcdd78e5b306ffed343d0ec3967f2e5", opaque="lEgVjogmIar2fg/t", qop=auth, nc=00000001, cnonce="e76b05db27a3b323"
Connection: keep-alive

`
	req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(body)))
	if err != nil {
		t.Fatalf("failed to read request: %v", err)
	}
	params := DigestAuthParams(req)
	want := "/css?family=Source+Sans+Pro:400,700,400italic,700italic|Source+Code+Pro"
	if params["uri"] != want {
		t.Fatalf("failed to parse uri with embedded commas, got %q want %q", params["uri"], want)
	}

}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"

	// static assets
	_ "github.com/SpectoLabs/hoverfly/statik"
	"github.com/rakyll/statik/fs"

	log "github.com/Sirupsen/logrus"
	"github.com/codegangsta/negroni"
	"github.com/go-zoo/bone"
	"github.com/meatballhat/negroni-logrus"
)

// recordedRequests struct encapsulates payload data
type recordedRequests struct {
	Data []Payload `json:"data"`
}

type recordsCount struct {
	Count int `json:"count"`
}

type stateRequest struct {
	Mode        string `json:"mode"`
	Destination string `json:"destination"`
}

type messageResponse struct {
	Message string `json:"message"`
}

func (d *DBClient) startAdminInterface() {
	// starting admin interface
	mux := getBoneRouter(*d)
	n := negroni.Classic()

	loglevel := log.InfoLevel

	if d.cfg.verbose {
		loglevel = log.DebugLevel
	}

	n.Use(negronilogrus.NewCustomMiddleware(loglevel, &log.JSONFormatter{}, "admin"))
	n.UseHandler(mux)

	// admin interface starting message
	log.WithFields(log.Fields{
		"AdminPort": d.cfg.adminPort,
	}).Info("Admin interface is starting...")

	n.Run(fmt.Sprintf(":%s", d.cfg.adminPort))
}

// getBoneRouter returns mux for admin interface
func getBoneRouter(d DBClient) *bone.Mux {
	mux := bone.New()

	// preparing static assets for embedded admin
	statikFS, err := fs.New()
	if err != nil {
		log.WithFields(log.Fields{
			"Error": err.Error(),
		}).Error("Failed to load statikFS, admin UI might not work :(")
	}

	mux.Get("/records", http.HandlerFunc(d.AllRecordsHandler))
	mux.Delete("/records", http.HandlerFunc(d.DeleteAllRecordsHandler))
	mux.Post("/records", http.HandlerFunc(d.ImportRecordsHandler))

	mux.Get("/count", http.HandlerFunc(d.RecordsCount))

	mux.Get("/state", http.HandlerFunc(d.CurrentStateHandler))
	mux.Post("/state", http.HandlerFunc(d.StateHandler))

	mux.Handle("/*", http.FileServer(statikFS))

	return mux
}

// AllRecordsHandler returns JSON content type http response
func (d *DBClient) AllRecordsHandler(w http.ResponseWriter, req *http.Request) {
	records, err := d.cache.GetAllRequests()

	if err == nil {

		w.Header().Set("Content-Type", "application/json")

		var response recordedRequests
		response.Data = records
		b, err := json.Marshal(response)

		if err != nil {
			log.Error(err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
		} else {
			w.Write(b)
			return
		}
	} else {
		log.WithFields(log.Fields{
			"Error": err.Error(),
		}).Error("Failed to get data from cache!")

		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
		w.WriteHeader(500) // can't process this entity
		return
	}
}

// RecordsCount returns number of captured requests as a JSON payload
func (d *DBClient) RecordsCount(w http.ResponseWriter, req *http.Request) {
	records, err := d.cache.GetAllRequests()

	if err == nil {

		w.Header().Set("Content-Type", "application/json")

		var response recordsCount
		response.Count = len(records)
		b, err := json.Marshal(response)

		if err != nil {
			log.Error(err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
		} else {
			w.Write(b)
			return
		}
	} else {
		log.WithFields(log.Fields{
			"Error": err.Error(),
		}).Error("Failed to get data from cache!")

		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
		w.WriteHeader(500) // can't process this entity
		return
	}
}

// ImportRecordsHandler - accepts JSON payload and saves it to cache
func (d *DBClient) ImportRecordsHandler(w http.ResponseWriter, req *http.Request) {

	var requests recordedRequests

	defer req.Body.Close()
	body, err := ioutil.ReadAll(req.Body)

	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
	var response messageResponse

	if err != nil {
		// failed to read response body
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Error("Could not read response body!")
		response.Message = "Bad request. Nothing to import!"
		http.Error(w, "Failed to read request body.", 400)
		return
	}

	err = json.Unmarshal(body, &requests)

	if err != nil {
		w.WriteHeader(422) // can't process this entity
		return
	}

	payloads := requests.Data
	if len(payloads) > 0 {
		for _, pl := range payloads {
			bts, err := pl.encode()
			if err != nil {
				log.WithFields(log.Fields{
					"error": err.Error(),
				}).Error("Failed to encode payload")
			} else {
				// recalculating request hash and storing it in database
				r := request{details: pl.Request}
				d.cache.Set([]byte(r.hash()), bts)
			}
		}
		response.Message = fmt.Sprintf("%d requests imported successfully", len(payloads))
	} else {
		response.Message = "Bad request. Nothing to import!"
		w.WriteHeader(400)
	}

	b, err := json.Marshal(response)
	w.Write(b)

}

// DeleteAllRecordsHandler - deletes all captured requests
func (d *DBClient) DeleteAllRecordsHandler(w http.ResponseWriter, req *http.Request) {
	err := d.cache.DeleteBucket(d.cache.requestsBucket)

	w.Header().Set("Content-Type", "application/json")

	var response messageResponse
	if err != nil {
		if err.Error() == "bucket not found" {
			response.Message = fmt.Sprintf("No records found")
			w.WriteHeader(200)
		} else {
			response.Message = fmt.Sprintf("Something went wrong: %s", err.Error())
			w.WriteHeader(500)
		}
	} else {
		response.Message = "Proxy cache deleted successfuly"
		w.WriteHeader(200)
	}
	b, err := json.Marshal(response)

	w.Write(b)
	return
}

// CurrentStateHandler returns current state
func (d *DBClient) CurrentStateHandler(w http.ResponseWriter, req *http.Request) {
	var resp stateRequest
	resp.Mode = d.cfg.GetMode()
	resp.Destination = d.cfg.destination

	b, _ := json.Marshal(resp)
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
	w.Write(b)
}

// StateHandler handles current proxy state
func (d *DBClient) StateHandler(w http.ResponseWriter, r *http.Request) {
	var sr stateRequest

	defer r.Body.Close()
	body, err := ioutil.ReadAll(r.Body)

	if err != nil {
		// failed to read response body
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Error("Could not read response body!")
		http.Error(w, "Failed to read request body.", 400)
		return
	}

	err = json.Unmarshal(body, &sr)

	if err != nil {
		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
		w.WriteHeader(422) // can't process this entity
		return
	}

	availableModes := map[string]bool{
		"virtualize": true,
		"capture":    true,
		"modify":     true,
		"synthesize": true,
	}

	if !availableModes[sr.Mode] {
		log.WithFields(log.Fields{
			"suppliedMode": sr.Mode,
		}).Error("Wrong mode found, can't change state")
		http.Error(w, "Bad mode supplied, available modes: virtualize, capture, modify, synthesize.", 400)
		return
	}

	log.WithFields(log.Fields{
		"newState": sr.Mode,
		"body":     string(body),
	}).Info("Handling state change request!")

	// setting new state
	d.cfg.SetMode(sr.Mode)

	var resp stateRequest
	resp.Mode = d.cfg.GetMode()
	resp.Destination = d.cfg.destination
	b, _ := json.Marshal(resp)
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
	w.Write(b)

}
<commit_msg>additional check if request has body<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"

	// static assets
	_ "github.com/SpectoLabs/hoverfly/statik"
	"github.com/rakyll/statik/fs"

	log "github.com/Sirupsen/logrus"
	"github.com/codegangsta/negroni"
	"github.com/go-zoo/bone"
	"github.com/meatballhat/negroni-logrus"
)

// recordedRequests struct encapsulates payload data
type recordedRequests struct {
	Data []Payload `json:"data"`
}

type recordsCount struct {
	Count int `json:"count"`
}

type stateRequest struct {
	Mode        string `json:"mode"`
	Destination string `json:"destination"`
}

type messageResponse struct {
	Message string `json:"message"`
}

func (d *DBClient) startAdminInterface() {
	// starting admin interface
	mux := getBoneRouter(*d)
	n := negroni.Classic()

	loglevel := log.InfoLevel

	if d.cfg.verbose {
		loglevel = log.DebugLevel
	}

	n.Use(negronilogrus.NewCustomMiddleware(loglevel, &log.JSONFormatter{}, "admin"))
	n.UseHandler(mux)

	// admin interface starting message
	log.WithFields(log.Fields{
		"AdminPort": d.cfg.adminPort,
	}).Info("Admin interface is starting...")

	n.Run(fmt.Sprintf(":%s", d.cfg.adminPort))
}

// getBoneRouter returns mux for admin interface
func getBoneRouter(d DBClient) *bone.Mux {
	mux := bone.New()

	// preparing static assets for embedded admin
	statikFS, err := fs.New()
	if err != nil {
		log.WithFields(log.Fields{
			"Error": err.Error(),
		}).Error("Failed to load statikFS, admin UI might not work :(")
	}

	mux.Get("/records", http.HandlerFunc(d.AllRecordsHandler))
	mux.Delete("/records", http.HandlerFunc(d.DeleteAllRecordsHandler))
	mux.Post("/records", http.HandlerFunc(d.ImportRecordsHandler))

	mux.Get("/count", http.HandlerFunc(d.RecordsCount))

	mux.Get("/state", http.HandlerFunc(d.CurrentStateHandler))
	mux.Post("/state", http.HandlerFunc(d.StateHandler))

	mux.Handle("/*", http.FileServer(statikFS))

	return mux
}

// AllRecordsHandler returns JSON content type http response
func (d *DBClient) AllRecordsHandler(w http.ResponseWriter, req *http.Request) {
	records, err := d.cache.GetAllRequests()

	if err == nil {

		w.Header().Set("Content-Type", "application/json")

		var response recordedRequests
		response.Data = records
		b, err := json.Marshal(response)

		if err != nil {
			log.Error(err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
		} else {
			w.Write(b)
			return
		}
	} else {
		log.WithFields(log.Fields{
			"Error": err.Error(),
		}).Error("Failed to get data from cache!")

		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
		w.WriteHeader(500) // can't process this entity
		return
	}
}

// RecordsCount returns number of captured requests as a JSON payload
func (d *DBClient) RecordsCount(w http.ResponseWriter, req *http.Request) {
	records, err := d.cache.GetAllRequests()

	if err == nil {

		w.Header().Set("Content-Type", "application/json")

		var response recordsCount
		response.Count = len(records)
		b, err := json.Marshal(response)

		if err != nil {
			log.Error(err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
		} else {
			w.Write(b)
			return
		}
	} else {
		log.WithFields(log.Fields{
			"Error": err.Error(),
		}).Error("Failed to get data from cache!")

		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
		w.WriteHeader(500) // can't process this entity
		return
	}
}

// ImportRecordsHandler - accepts JSON payload and saves it to cache
func (d *DBClient) ImportRecordsHandler(w http.ResponseWriter, req *http.Request) {

	var requests recordedRequests

	defer req.Body.Close()
	body, err := ioutil.ReadAll(req.Body)

	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
	var response messageResponse

	if err != nil {
		// failed to read response body
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Error("Could not read response body!")
		response.Message = "Bad request. Nothing to import!"
		http.Error(w, "Failed to read request body.", 400)
		return
	}

	err = json.Unmarshal(body, &requests)

	if err != nil {
		w.WriteHeader(422) // can't process this entity
		return
	}

	payloads := requests.Data
	if len(payloads) > 0 {
		for _, pl := range payloads {
			bts, err := pl.encode()
			if err != nil {
				log.WithFields(log.Fields{
					"error": err.Error(),
				}).Error("Failed to encode payload")
			} else {
				// recalculating request hash and storing it in database
				r := request{details: pl.Request}
				d.cache.Set([]byte(r.hash()), bts)
			}
		}
		response.Message = fmt.Sprintf("%d requests imported successfully", len(payloads))
	} else {
		response.Message = "Bad request. Nothing to import!"
		w.WriteHeader(400)
	}

	b, err := json.Marshal(response)
	w.Write(b)

}

// DeleteAllRecordsHandler - deletes all captured requests
func (d *DBClient) DeleteAllRecordsHandler(w http.ResponseWriter, req *http.Request) {
	err := d.cache.DeleteBucket(d.cache.requestsBucket)

	w.Header().Set("Content-Type", "application/json")

	var response messageResponse
	if err != nil {
		if err.Error() == "bucket not found" {
			response.Message = fmt.Sprintf("No records found")
			w.WriteHeader(200)
		} else {
			response.Message = fmt.Sprintf("Something went wrong: %s", err.Error())
			w.WriteHeader(500)
		}
	} else {
		response.Message = "Proxy cache deleted successfuly"
		w.WriteHeader(200)
	}
	b, err := json.Marshal(response)

	w.Write(b)
	return
}

// CurrentStateHandler returns current state
func (d *DBClient) CurrentStateHandler(w http.ResponseWriter, req *http.Request) {
	var resp stateRequest
	resp.Mode = d.cfg.GetMode()
	resp.Destination = d.cfg.destination

	b, _ := json.Marshal(resp)
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
	w.Write(b)
}

// StateHandler handles current proxy state
func (d *DBClient) StateHandler(w http.ResponseWriter, r *http.Request) {
	var sr stateRequest

	// this is mainly for testing, since when you create
	if r.Body == nil {
		r.Body = ioutil.NopCloser(bytes.NewBuffer([]byte("")))
	}

	defer r.Body.Close()
	body, err := ioutil.ReadAll(r.Body)

	if err != nil {
		// failed to read response body
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Error("Could not read response body!")
		http.Error(w, "Failed to read request body.", 400)
		return
	}

	err = json.Unmarshal(body, &sr)

	if err != nil {
		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
		w.WriteHeader(400) // can't process this entity
		return
	}

	availableModes := map[string]bool{
		"virtualize": true,
		"capture":    true,
		"modify":     true,
		"synthesize": true,
	}

	if !availableModes[sr.Mode] {
		log.WithFields(log.Fields{
			"suppliedMode": sr.Mode,
		}).Error("Wrong mode found, can't change state")
		http.Error(w, "Bad mode supplied, available modes: virtualize, capture, modify, synthesize.", 400)
		return
	}

	log.WithFields(log.Fields{
		"newState": sr.Mode,
		"body":     string(body),
	}).Info("Handling state change request!")

	// setting new state
	d.cfg.SetMode(sr.Mode)

	var resp stateRequest
	resp.Mode = d.cfg.GetMode()
	resp.Destination = d.cfg.destination
	b, _ := json.Marshal(resp)
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
	w.Write(b)

}
<|endoftext|>


<commit_before>package local_test

import (
	"io/ioutil"
	"restic"
	"testing"

	"restic/backend/local"
	"restic/backend/test"
	. "restic/test"
)

func TestBackend(t *testing.T) {
	suite := test.Suite{
		// NewConfig returns a config for a new temporary backend that will be used in tests.
		NewConfig: func() (interface{}, error) {
			dir, err := ioutil.TempDir(TestTempDir, "restic-test-local-")
			if err != nil {
				t.Fatal(err)
			}

			t.Logf("create new backend at %v", dir)

			cfg := local.Config{
				Path: dir,
			}
			return cfg, nil
		},

		// CreateFn is a function that creates a temporary repository for the tests.
		Create: func(config interface{}) (restic.Backend, error) {
			cfg := config.(local.Config)
			return local.Create(cfg)
		},

		// OpenFn is a function that opens a previously created temporary repository.
		Open: func(config interface{}) (restic.Backend, error) {
			cfg := config.(local.Config)
			return local.Open(cfg)
		},

		// CleanupFn removes data created during the tests.
		Cleanup: func(config interface{}) error {
			cfg := config.(local.Config)
			if !TestCleanupTempDirs {
				t.Logf("leaving test backend dir at %v", cfg.Path)
			}

			RemoveAll(t, cfg.Path)
			return nil
		},
	}

	suite.RunTests(t)
}
<commit_msg>local: Add benchmark<commit_after>package local_test

import (
	"io/ioutil"
	"restic"
	"testing"

	"restic/backend/local"
	"restic/backend/test"
	. "restic/test"
)

func newTestSuite(t testing.TB) *test.Suite {
	return &test.Suite{
		// NewConfig returns a config for a new temporary backend that will be used in tests.
		NewConfig: func() (interface{}, error) {
			dir, err := ioutil.TempDir(TestTempDir, "restic-test-local-")
			if err != nil {
				t.Fatal(err)
			}

			t.Logf("create new backend at %v", dir)

			cfg := local.Config{
				Path: dir,
			}
			return cfg, nil
		},

		// CreateFn is a function that creates a temporary repository for the tests.
		Create: func(config interface{}) (restic.Backend, error) {
			cfg := config.(local.Config)
			return local.Create(cfg)
		},

		// OpenFn is a function that opens a previously created temporary repository.
		Open: func(config interface{}) (restic.Backend, error) {
			cfg := config.(local.Config)
			return local.Open(cfg)
		},

		// CleanupFn removes data created during the tests.
		Cleanup: func(config interface{}) error {
			cfg := config.(local.Config)
			if !TestCleanupTempDirs {
				t.Logf("leaving test backend dir at %v", cfg.Path)
			}

			RemoveAll(t, cfg.Path)
			return nil
		},
	}
}

func TestBackend(t *testing.T) {
	newTestSuite(t).RunTests(t)
}

func BenchmarkBackend(t *testing.B) {
	newTestSuite(t).RunBenchmarks(t)
}
<|endoftext|>


<commit_before>package gitcli

import (
	"strings"
	"fmt"
)

func (r *Repo)Push(extraArgs ...string) error {
	args := []string{`push`}
	return r.commonCmd(args, extraArgs...)
}

func (r *Repo)Pull(extraArgs ...string) error {
	args := []string{`pull`}
	return r.commonCmd(args, extraArgs...)
}

func (r *Repo)Fetch(extraArgs ...string) error {
	args := []string{`fetch`}
	return r.commonCmd(args, extraArgs...)
}

func (r *Repo)Checkout(extraArgs ...string) error {
	args := []string{`checkout`}
	return r.commonCmd(args, extraArgs...)
}

func (r *Repo)Log(extraArgs ...string) error {
	args := []string{`log`}
	return r.commonCmd(args, extraArgs...)
}

// Add or set a given remote
func (r *Repo)SetRemote(remoteName string, remoteUrl string ) error {
	stdout, stderr, err := r.cmd("remote","set-url",remoteName,remoteUrl)
	if strings.Contains(stderr,"No such remote") {
		err = r.commonCmd([]string{"remote","add"},remoteName,remoteUrl)
		return err
	} else if err != nil {
		return fmt.Errorf("set remote failed with stdout:%s , stderr: %s", stdout, stderr)
	}
	return err
}
<commit_msg>add clean<commit_after>package gitcli

import (
	"strings"
	"fmt"
)

func (r *Repo)Push(extraArgs ...string) error {
	args := []string{`push`}
	return r.commonCmd(args, extraArgs...)
}

func (r *Repo)Pull(extraArgs ...string) error {
	args := []string{`pull`}
	return r.commonCmd(args, extraArgs...)
}

func (r *Repo)Fetch(extraArgs ...string) error {
	args := []string{`fetch`}
	return r.commonCmd(args, extraArgs...)
}

func (r *Repo)Checkout(extraArgs ...string) error {
	args := []string{`checkout`}
	return r.commonCmd(args, extraArgs...)
}

func (r *Repo)Log(extraArgs ...string) error {
	args := []string{`log`}
	return r.commonCmd(args, extraArgs...)
}

func (r *Repo)Clean(extraArgs ...string) error {
	args := []string{`clean`}
	return r.commonCmd(args, extraArgs...)
}

// Add or set a given remote
func (r *Repo)SetRemote(remoteName string, remoteUrl string ) error {
	stdout, stderr, err := r.cmd("remote","set-url",remoteName,remoteUrl)
	if strings.Contains(stderr,"No such remote") {
		err = r.commonCmd([]string{"remote","add"},remoteName,remoteUrl)
		return err
	} else if err != nil {
		return fmt.Errorf("set remote failed with stdout:%s , stderr: %s", stdout, stderr)
	}
	return err
}
<|endoftext|>


<commit_before>package immortal

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"
)

// ScanDir struct
type ScanDir struct {
	scandir string
}

// NewScanDir returns ScanDir struct
func NewScanDir(path string) (*ScanDir, error) {
	if info, err := os.Stat(path); os.IsNotExist(err) {
		return nil, fmt.Errorf("%q no such file or directory.", path)
	} else if !info.IsDir() {
		return nil, fmt.Errorf("%q is not a directory.", path)
	}

	dir, err := filepath.EvalSymlinks(path)
	if err != nil {
		return nil, err
	}

	dir, err = filepath.Abs(filepath.Clean(dir))
	if err != nil {
		return nil, err
	}

	d, err := os.Open(dir)
	if err != nil {
		if os.IsPermission(err) {
			return nil, os.ErrPermission
		}
		return nil, err
	}
	defer d.Close()

	return &ScanDir{
		scandir: dir,
	}, nil
}

// Start scans directory every 5 seconds
func (s *ScanDir) Start() {
	log.Printf("immortal scandir: %s", s.scandir)
	s.Scaner()
	ticker := time.NewTicker(5 * time.Second)
	for {
		select {
		case <-ticker.C:
			s.Scaner()
		}
	}
}

// Scaner searchs for run.yml and based on the perms start/stops the process
func (s *ScanDir) Scaner() {
	time := time.Now()
	find := func(path string, f os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if strings.HasSuffix(f.Name(), ".yml") {
			name := strings.TrimSuffix(f.Name(), filepath.Ext(f.Name()))
			refresh := (time.Unix() - f.ModTime().Unix()) <= 5
			log.Printf("name: %s  refresh: %v", name, refresh)
			if refresh {
				// chmod 3444 for example
				if m := f.Mode(); m&os.ModeSticky != 0 {
					println("turn off ", name)
				}
				cmd := exec.Command("immortal", "-c", path, "-ctl", name)
				cmd.Env = os.Environ()
				fmt.Printf("cmd.Path = %+v\n", cmd.Path)
				stdoutStderr, err := cmd.CombinedOutput()
				if err != nil {
					return err
				}
				log.Printf("%s\n", stdoutStderr)
			}
		}
		return nil
	}
	err := filepath.Walk(s.scandir, find)
	if err != nil {
		log.Println(err)
	}
	return
}
<commit_msg>	modified:   scandir.go<commit_after>package immortal

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"
)

// ScanDir struct
type ScanDir struct {
	scandir string
}

// NewScanDir returns ScanDir struct
func NewScanDir(path string) (*ScanDir, error) {
	if info, err := os.Stat(path); os.IsNotExist(err) {
		return nil, fmt.Errorf("%q no such file or directory.", path)
	} else if !info.IsDir() {
		return nil, fmt.Errorf("%q is not a directory.", path)
	}

	dir, err := filepath.EvalSymlinks(path)
	if err != nil {
		return nil, err
	}

	dir, err = filepath.Abs(filepath.Clean(dir))
	if err != nil {
		return nil, err
	}

	d, err := os.Open(dir)
	if err != nil {
		if os.IsPermission(err) {
			return nil, os.ErrPermission
		}
		return nil, err
	}
	defer d.Close()

	return &ScanDir{
		scandir: dir,
	}, nil
}

// Start scans directory every 5 seconds
func (s *ScanDir) Start() {
	log.Printf("immortal scandir: %s", s.scandir)
	s.Scaner()
	ticker := time.NewTicker(5 * time.Second)
	for {
		select {
		case <-ticker.C:
			s.Scaner()
		}
	}
}

// Scaner searchs for run.yml and based on the perms start/stops the process
func (s *ScanDir) Scaner() {
	time := time.Now()
	find := func(path string, f os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if strings.HasSuffix(f.Name(), ".yml") {
			name := strings.TrimSuffix(f.Name(), filepath.Ext(f.Name()))
			refresh := (time.Unix() - f.ModTime().Unix()) <= 5
			log.Printf("name: %s  refresh: %v", name, refresh)
			if refresh {
				if m := f.Mode(); m&0111 != 0 {
					println("turn on ", name)
				} else {
					println("turn off ", name)
				}
				cmd := exec.Command("immortal", "-c", path, "-ctl", name)
				cmd.Env = os.Environ()
				fmt.Printf("cmd.Path = %+v\n", cmd.Path)
				stdoutStderr, err := cmd.CombinedOutput()
				if err != nil {
					return err
				}
				log.Printf("%s\n", stdoutStderr)
			}
		}
		return nil
	}
	err := filepath.Walk(s.scandir, find)
	if err != nil {
		log.Println(err)
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/xml"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

type (
	Protocol struct {
		XMLName    xml.Name    `xml:"protocol"`
		Name       string      `xml:"name,attr"`
		Copyright  string      `xml:"copyright"`
		Interfaces []Interface `xml:"interface"`
	}

	Description struct {
		XMLName     xml.Name `xml:"description"`
		Summary     string   `xml:"summary,attr"`
		Description string   `xml:"description"`
	}

	Interface struct {
		XMLName     xml.Name    `xml:"interface"`
		Name        string      `xml:"name,attr"`
		Version     int         `xml:"version,attr"`
		Since       int         `xml:"since,attr"` // maybe in future versions
		Description Description `xml:"description"`
		Requests    []Request   `xml:"request"`
		Events      []Event     `xml:"event"`
		Enums       []Enum      `xml:"enum"`
	}

	Request struct {
		XMLName     xml.Name    `xml:"request"`
		Name        string      `xml:"name,attr"`
		Type        string      `xml:"type,attr"`
		Since       int         `xml:"since,attr"`
		Description Description `xml:"description"`
		Args        []Arg       `xml:"arg"`
	}

	Arg struct {
		XMLName   xml.Name `xml:"arg"`
		Name      string   `xml:"name,attr"`
		Type      string   `xml:"type,attr"`
		Interface string   `xml:"interface,attr"`
		Enum      string   `xml:"enum,attr"`
		AllowNull bool     `xml:"allow-null,attr"`
		Summary   string   `xml:"summary,attr"`
	}

	Event struct {
		XMLName     xml.Name    `xml:"event"`
		Name        string      `xml:"name,attr"`
		Since       int         `xml:"since,attr"`
		Description Description `xml:"description"`
		Args        []Arg       `xml:"arg"`
	}

	Enum struct {
		XMLName     xml.Name    `xml:"enum"`
		Name        string      `xml:"name,attr"`
		BitField    bool        `xml:"bitfield,attr"`
		Description Description `xml:"description"`
		Entries     []Entry     `xml:"entry"`
	}

	Entry struct {
		XMLName xml.Name `xml:"entry"`
		Name    string   `xml:"name,attr"`
		Value   string   `xml:"value,attr"`
		Summary string   `xml:"summary,attr"`
	}
)

var (
	wlTypes map[string]string = map[string]string{
		"int":    "int32",
		"uint":   "uint32",
		"string": "string",
		"fd":     "uintptr",
		"fixed":  "float32",
		"array":  "[]int32",
	}

	wlNames        map[string]string
	constBuffer    bytes.Buffer
	ifaceBuffer    bytes.Buffer
	reqCodesBuffer bytes.Buffer

	overwrite = flag.Bool("o", false, "Overwrite existing client.go file")
	develXml  = flag.Bool("dev", false, "Get development version of wayland.xml from repository")
)

func init() {
	flag.Parse()
	log.SetFlags(0)
}

func main() {
	var xmlFile *os.File

	if *develXml {
		file, err := getDevelXml()
		if err != nil {
			file.Close()
			log.Fatalf("Error while reading xml file : %s", err)
		}
		xmlFile = file
		xmlFile.Seek(0, 0)
	} else {
		xmlFilePath, err := filepath.Abs("wayland.xml")
		if err != nil {
			log.Fatalf("Cannot find wayland.xml: %s", err)
		}

		file, err := os.Open(xmlFilePath)
		if err != nil {
			log.Fatal("Cannot open wayland.xml")
		}
		xmlFile = file
	}

	defer xmlFile.Close()

	var protocol Protocol
	if err := xml.NewDecoder(xmlFile).Decode(&protocol); err != nil {
		log.Fatalf("Cannot decode wayland.xml : %s", err)
	}

	wlNames = make(map[string]string)

	fmt.Fprint(&constBuffer, "package wl")

	for _, iface := range protocol.Interfaces {
		//required for arg type's determine
		caseAndRegister(iface.Name)
	}

	fmt.Fprint(&reqCodesBuffer, "\n//Interface Request Codes\n") // request codes
	fmt.Fprint(&reqCodesBuffer, "\nconst (\n")                   // request codes

	for _, iface := range protocol.Interfaces {
		eventBuffer, eventNames := interfaceEvents(iface)
		eventBuffer.WriteTo(&ifaceBuffer)

		interfaceTypes(iface, eventNames)
		interfaceConstructor(iface, eventNames)
		interfaceRequests(iface)
		interfaceEnums(iface)
	}

	fmt.Fprint(&reqCodesBuffer, ")") // request codes end

	// if file exists
	if _, err := os.Stat("client.go"); err == nil {
		if !*overwrite {
			log.Print("client.go exists if you want to overwrite try -o flag")
			return
		}
	}

	file, err := os.Create("client.go")
	if err != nil {
		log.Fatalf("Cannot create file: %s", err)
	}

	constBuffer.WriteTo(file)
	reqCodesBuffer.WriteTo(file)
	ifaceBuffer.WriteTo(file)

	file.Close()

	// go fmt file
	fmtFile()
}

// register names to map
func caseAndRegister(wlName string) string {
	var orj string = wlName
	wlName = CamelCase(wlName)
	wlNames[orj] = wlName
	return wlName
}

func enumArgName(ifaceName, enumName string) string {
	if strings.Index(enumName, ".") == -1 {
		return ifaceName + CamelCase(enumName)
	} else {
		parts := strings.Split(enumName, ".")
		if len(parts) != 2 {
			log.Fatal("enum args must be \"interface.enum\" format")
		}
		return CamelCase(parts[0]) + CamelCase(parts[1])
	}
}

func CamelCase(wlName string) string {
	if strings.HasPrefix(wlName, "wl_") {
		wlName = strings.TrimPrefix(wlName, "wl_")
	}

	// replace all "_" chars to " " chars
	wlName = strings.Replace(wlName, "_", " ", -1)

	// Capitalize first chars
	wlName = strings.Title(wlName)

	// remove all spaces
	wlName = strings.Replace(wlName, " ", "", -1)

	return wlName
}

func interfaceConstructor(iface Interface, eventNames []string) {
	ifaceName := wlNames[iface.Name]

	// interface constructor
	fmt.Fprintf(&ifaceBuffer, "\nfunc New%s(conn *Connection) *%s {\n", ifaceName, ifaceName)
	fmt.Fprintf(&ifaceBuffer, "ret := new(%s)\n", ifaceName)
	for _, evName := range eventNames {
		fmt.Fprintf(&ifaceBuffer, "ret.%sChan = make(chan %s%sEvent)\n", evName, ifaceName, evName)
	}

	fmt.Fprint(&ifaceBuffer, "conn.Register(ret)\n")
	fmt.Fprint(&ifaceBuffer, "return ret\n")
	fmt.Fprint(&ifaceBuffer, "}\n")
}

func interfaceTypes(iface Interface, eventNames []string) {
	ifaceName := wlNames[iface.Name]
	// interface type definition
	fmt.Fprintf(&ifaceBuffer, "\ntype %s struct {\n", ifaceName)
	fmt.Fprint(&ifaceBuffer, "BaseProxy\n")
	for _, evName := range eventNames {
		fmt.Fprintf(&ifaceBuffer, "%sChan chan %s%sEvent\n", evName, ifaceName, evName)
	}
	fmt.Fprint(&ifaceBuffer, "}\n")
}

func interfaceRequests(iface Interface) {
	ifaceName := wlNames[iface.Name]

	// interface method definitions (requests)
	// order used for request identification
	for order, req := range iface.Requests {
		reqName := CamelCase(req.Name)
		reqCodeName := strings.ToTitle(fmt.Sprintf("_%s_%s", ifaceName, reqName)) // first _ for not export constant
		fmt.Fprintf(&reqCodesBuffer, "%s = %d\n", reqCodeName, order)

		fmt.Fprintf(&ifaceBuffer, "\nfunc (p *%s) %s(", ifaceName, reqName)
		// get args buffer
		requestArgs(ifaceName, req).WriteTo(&ifaceBuffer)

		fmt.Fprint(&ifaceBuffer, ")") // close the args

		// get returns buffer
		requestRets(req).WriteTo(&ifaceBuffer)
		fmt.Fprint(&ifaceBuffer, "{\n")

		// get method body
		requestBody(req, reqCodeName).WriteTo(&ifaceBuffer)

		fmt.Fprint(&ifaceBuffer, "\n}\n")
	}
}

func interfaceEnums(iface Interface) {
	ifaceName := wlNames[iface.Name]

	// Enums - Constants
	for _, enum := range iface.Enums {
		enumName := caseAndRegister(enum.Name)
		constTypeName := ifaceName + enumName
		fmt.Fprintf(&constBuffer, "\ntype %s uint32\n", constTypeName) // enums are uint
		fmt.Fprint(&constBuffer, "const (\n")
		for _, entry := range enum.Entries {
			entryName := caseAndRegister(entry.Name)
			constName := ifaceName + enumName + entryName
			fmt.Fprintf(&constBuffer, "%s %s = %s\n", constName, constTypeName, entry.Value)
		}
		fmt.Fprint(&constBuffer, ")\n")
	}
}

func interfaceEvents(iface Interface) (bytes.Buffer, []string) {
	var (
		eventBuffer bytes.Buffer
		eventNames  []string
		ifaceName   = wlNames[iface.Name]
	)

	// Event struct types
	for _, event := range iface.Events {
		eventName := caseAndRegister(event.Name)
		fmt.Fprintf(&eventBuffer, "\ntype %s%sEvent struct {\n", ifaceName, eventName)
		for _, arg := range event.Args {
			if t, ok := wlTypes[arg.Type]; ok { // if basic type
				if arg.Type == "uint" && arg.Enum != "" { // enum type
					enumTypeName := ifaceName + CamelCase(arg.Enum)
					fmt.Fprintf(&eventBuffer, "%s %s\n", CamelCase(arg.Name), enumTypeName)
				} else {
					fmt.Fprintf(&eventBuffer, "%s %s\n", CamelCase(arg.Name), t)
				}
			} else { // interface type
				if (arg.Type == "object" || arg.Type == "new_id") && arg.Interface != "" {
					t = "*" + wlNames[arg.Interface]
				} else {
					t = "Proxy"
				}
				fmt.Fprintf(&eventBuffer, "%s %s\n", CamelCase(arg.Name), t)
			}
		}

		eventNames = append(eventNames, eventName)
		fmt.Fprint(&eventBuffer, "}\n")
	}

	return eventBuffer, eventNames
}

func requestArgs(ifaceName string, req Request) *bytes.Buffer {
	var (
		args       []string
		argsBuffer bytes.Buffer
	)

	for _, arg := range req.Args {
		// special type, for example registry.bind
		if arg.Type == "new_id" {
			if arg.Interface == "" {
				args = append(args, "iface string")
				args = append(args, "version uint32")
				args = append(args, fmt.Sprintf("%s Proxy", arg.Name))
			} else {
				continue
			}
		} else if arg.Type == "object" && arg.Interface != "" {
			argTypeName := wlNames[arg.Interface]
			args = append(args, fmt.Sprintf("%s *%s", arg.Name, argTypeName))
		} else if arg.Type == "uint" && arg.Enum != "" {
			args = append(args, fmt.Sprintf("%s %s", arg.Name, enumArgName(ifaceName, arg.Enum)))
		} else {
			args = append(args, fmt.Sprintf("%s %s", arg.Name, wlTypes[arg.Type]))
		}
	}

	fmt.Fprint(&argsBuffer, strings.Join(args, ","))

	return &argsBuffer
}

func requestRets(req Request) *bytes.Buffer {
	var (
		rets       []string
		retsBuffer bytes.Buffer
	)

	for _, arg := range req.Args {
		if arg.Type == "new_id" && arg.Interface != "" {
			retTypeName := wlNames[arg.Interface]
			rets = append(rets, fmt.Sprintf("*%s", retTypeName))
		}
	}

	// all request have an error return
	rets = append(rets, "error")

	retstr := strings.Join(rets, ",")

	if len(rets) > 1 {
		fmt.Fprintf(&retsBuffer, "( %s )", retstr)
	} else {
		fmt.Fprint(&retsBuffer, retstr)
	}

	return &retsBuffer
}

func requestBody(req Request, reqCodeName string) *bytes.Buffer {
	var (
		params       []string
		bodyBuffer   bytes.Buffer
		paramsBuffer bytes.Buffer
		hasRet       string
	)

	for _, arg := range req.Args {
		if arg.Type == "new_id" {
			if arg.Interface != "" {
				retTypeName := wlNames[arg.Interface]
				fmt.Fprintf(&bodyBuffer, "ret := New%s(p.Connection())\n", retTypeName)
				params = append(params, "Proxy(ret)")
				hasRet = "ret,"
			} else {
				params = append(params, "iface")
				params = append(params, "version")
				params = append(params, arg.Name)
			}
		} else {
			params = append(params, arg.Name)
		}
	}

	for _, param := range params {
		fmt.Fprintf(&paramsBuffer, ",%s", param)
	}

	fmt.Fprintf(&bodyBuffer, "return %s p.Connection().SendRequest(p,%s%s)", hasRet, reqCodeName, paramsBuffer.String())

	return &bodyBuffer
}

func getDevelXml() (*os.File, error) {
	url := "https://cgit.freedesktop.org/wayland/wayland/plain/protocol/wayland.xml"
	resp, err := http.Get(url)
	if err != nil {
		return nil, fmt.Errorf("http get error")
	}

	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("Cannot get wayland.xml StatusCode error")
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("Cannot read response body")
	} else {
		file, err := ioutil.TempFile("", "devel_wayland_xml")
		if err != nil {
			return nil, fmt.Errorf("Cannot create temp file")
		} else {
			file.Write(body)
			return file, nil
		}
	}
}

func fmtFile() {
	goex, err := exec.LookPath("go")
	if err != nil {
		log.Println("go executable cannot found run \"go fmt client.go\" yourself")
	} else {
		cmd := exec.Command(goex, "fmt", "client.go")
		err := cmd.Run()
		if err != nil {
			log.Fatalf("Cannot run cmd : %s", err)
		}
	}
}
<commit_msg>add err to error strings<commit_after>package main

import (
	"bytes"
	"encoding/xml"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

type (
	Protocol struct {
		XMLName    xml.Name    `xml:"protocol"`
		Name       string      `xml:"name,attr"`
		Copyright  string      `xml:"copyright"`
		Interfaces []Interface `xml:"interface"`
	}

	Description struct {
		XMLName     xml.Name `xml:"description"`
		Summary     string   `xml:"summary,attr"`
		Description string   `xml:"description"`
	}

	Interface struct {
		XMLName     xml.Name    `xml:"interface"`
		Name        string      `xml:"name,attr"`
		Version     int         `xml:"version,attr"`
		Since       int         `xml:"since,attr"` // maybe in future versions
		Description Description `xml:"description"`
		Requests    []Request   `xml:"request"`
		Events      []Event     `xml:"event"`
		Enums       []Enum      `xml:"enum"`
	}

	Request struct {
		XMLName     xml.Name    `xml:"request"`
		Name        string      `xml:"name,attr"`
		Type        string      `xml:"type,attr"`
		Since       int         `xml:"since,attr"`
		Description Description `xml:"description"`
		Args        []Arg       `xml:"arg"`
	}

	Arg struct {
		XMLName   xml.Name `xml:"arg"`
		Name      string   `xml:"name,attr"`
		Type      string   `xml:"type,attr"`
		Interface string   `xml:"interface,attr"`
		Enum      string   `xml:"enum,attr"`
		AllowNull bool     `xml:"allow-null,attr"`
		Summary   string   `xml:"summary,attr"`
	}

	Event struct {
		XMLName     xml.Name    `xml:"event"`
		Name        string      `xml:"name,attr"`
		Since       int         `xml:"since,attr"`
		Description Description `xml:"description"`
		Args        []Arg       `xml:"arg"`
	}

	Enum struct {
		XMLName     xml.Name    `xml:"enum"`
		Name        string      `xml:"name,attr"`
		BitField    bool        `xml:"bitfield,attr"`
		Description Description `xml:"description"`
		Entries     []Entry     `xml:"entry"`
	}

	Entry struct {
		XMLName xml.Name `xml:"entry"`
		Name    string   `xml:"name,attr"`
		Value   string   `xml:"value,attr"`
		Summary string   `xml:"summary,attr"`
	}
)

var (
	wlTypes map[string]string = map[string]string{
		"int":    "int32",
		"uint":   "uint32",
		"string": "string",
		"fd":     "uintptr",
		"fixed":  "float32",
		"array":  "[]int32",
	}

	wlNames        map[string]string
	constBuffer    bytes.Buffer
	ifaceBuffer    bytes.Buffer
	reqCodesBuffer bytes.Buffer

	overwrite = flag.Bool("o", false, "Overwrite existing client.go file")
	develXml  = flag.Bool("dev", false, "Get development version of wayland.xml from repository")
)

func init() {
	flag.Parse()
	log.SetFlags(0)
}

func main() {
	var xmlFile *os.File

	if *develXml {
		file, err := getDevelXml()
		if err != nil {
			file.Close()
			log.Fatalf("Error while reading xml file : %s", err)
		}
		xmlFile = file
		xmlFile.Seek(0, 0)
	} else {
		xmlFilePath, err := filepath.Abs("wayland.xml")
		if err != nil {
			log.Fatalf("Cannot find wayland.xml: %s", err)
		}

		file, err := os.Open(xmlFilePath)
		if err != nil {
			log.Fatalf("Cannot open wayland.xml:%s", err)
		}
		xmlFile = file
	}

	defer xmlFile.Close()

	var protocol Protocol
	if err := xml.NewDecoder(xmlFile).Decode(&protocol); err != nil {
		log.Fatalf("Cannot decode wayland.xml : %s", err)
	}

	wlNames = make(map[string]string)

	fmt.Fprint(&constBuffer, "package wl")

	for _, iface := range protocol.Interfaces {
		//required for arg type's determine
		caseAndRegister(iface.Name)
	}

	fmt.Fprint(&reqCodesBuffer, "\n//Interface Request Codes\n") // request codes
	fmt.Fprint(&reqCodesBuffer, "\nconst (\n")                   // request codes

	for _, iface := range protocol.Interfaces {
		eventBuffer, eventNames := interfaceEvents(iface)
		eventBuffer.WriteTo(&ifaceBuffer)

		interfaceTypes(iface, eventNames)
		interfaceConstructor(iface, eventNames)
		interfaceRequests(iface)
		interfaceEnums(iface)
	}

	fmt.Fprint(&reqCodesBuffer, ")") // request codes end

	// if file exists
	if _, err := os.Stat("client.go"); err == nil {
		if !*overwrite {
			log.Print("client.go exists if you want to overwrite try -o flag")
			return
		}
	}

	file, err := os.Create("client.go")
	if err != nil {
		log.Fatalf("Cannot create file: %s", err)
	}

	constBuffer.WriteTo(file)
	reqCodesBuffer.WriteTo(file)
	ifaceBuffer.WriteTo(file)

	file.Close()

	// go fmt file
	fmtFile()
}

// register names to map
func caseAndRegister(wlName string) string {
	var orj string = wlName
	wlName = CamelCase(wlName)
	wlNames[orj] = wlName
	return wlName
}

func enumArgName(ifaceName, enumName string) string {
	if strings.Index(enumName, ".") == -1 {
		return ifaceName + CamelCase(enumName)
	} else {
		parts := strings.Split(enumName, ".")
		if len(parts) != 2 {
			log.Fatal("enum args must be \"interface.enum\" format")
		}
		return CamelCase(parts[0]) + CamelCase(parts[1])
	}
}

func CamelCase(wlName string) string {
	if strings.HasPrefix(wlName, "wl_") {
		wlName = strings.TrimPrefix(wlName, "wl_")
	}

	// replace all "_" chars to " " chars
	wlName = strings.Replace(wlName, "_", " ", -1)

	// Capitalize first chars
	wlName = strings.Title(wlName)

	// remove all spaces
	wlName = strings.Replace(wlName, " ", "", -1)

	return wlName
}

func interfaceConstructor(iface Interface, eventNames []string) {
	ifaceName := wlNames[iface.Name]

	// interface constructor
	fmt.Fprintf(&ifaceBuffer, "\nfunc New%s(conn *Connection) *%s {\n", ifaceName, ifaceName)
	fmt.Fprintf(&ifaceBuffer, "ret := new(%s)\n", ifaceName)
	for _, evName := range eventNames {
		fmt.Fprintf(&ifaceBuffer, "ret.%sChan = make(chan %s%sEvent)\n", evName, ifaceName, evName)
	}

	fmt.Fprint(&ifaceBuffer, "conn.Register(ret)\n")
	fmt.Fprint(&ifaceBuffer, "return ret\n")
	fmt.Fprint(&ifaceBuffer, "}\n")
}

func interfaceTypes(iface Interface, eventNames []string) {
	ifaceName := wlNames[iface.Name]
	// interface type definition
	fmt.Fprintf(&ifaceBuffer, "\ntype %s struct {\n", ifaceName)
	fmt.Fprint(&ifaceBuffer, "BaseProxy\n")
	for _, evName := range eventNames {
		fmt.Fprintf(&ifaceBuffer, "%sChan chan %s%sEvent\n", evName, ifaceName, evName)
	}
	fmt.Fprint(&ifaceBuffer, "}\n")
}

func interfaceRequests(iface Interface) {
	ifaceName := wlNames[iface.Name]

	// interface method definitions (requests)
	// order used for request identification
	for order, req := range iface.Requests {
		reqName := CamelCase(req.Name)
		reqCodeName := strings.ToTitle(fmt.Sprintf("_%s_%s", ifaceName, reqName)) // first _ for not export constant
		fmt.Fprintf(&reqCodesBuffer, "%s = %d\n", reqCodeName, order)

		fmt.Fprintf(&ifaceBuffer, "\nfunc (p *%s) %s(", ifaceName, reqName)
		// get args buffer
		requestArgs(ifaceName, req).WriteTo(&ifaceBuffer)

		fmt.Fprint(&ifaceBuffer, ")") // close the args

		// get returns buffer
		requestRets(req).WriteTo(&ifaceBuffer)
		fmt.Fprint(&ifaceBuffer, "{\n")

		// get method body
		requestBody(req, reqCodeName).WriteTo(&ifaceBuffer)

		fmt.Fprint(&ifaceBuffer, "\n}\n")
	}
}

func interfaceEnums(iface Interface) {
	ifaceName := wlNames[iface.Name]

	// Enums - Constants
	for _, enum := range iface.Enums {
		enumName := caseAndRegister(enum.Name)
		constTypeName := ifaceName + enumName
		fmt.Fprintf(&constBuffer, "\ntype %s uint32\n", constTypeName) // enums are uint
		fmt.Fprint(&constBuffer, "const (\n")
		for _, entry := range enum.Entries {
			entryName := caseAndRegister(entry.Name)
			constName := ifaceName + enumName + entryName
			fmt.Fprintf(&constBuffer, "%s %s = %s\n", constName, constTypeName, entry.Value)
		}
		fmt.Fprint(&constBuffer, ")\n")
	}
}

func interfaceEvents(iface Interface) (bytes.Buffer, []string) {
	var (
		eventBuffer bytes.Buffer
		eventNames  []string
		ifaceName   = wlNames[iface.Name]
	)

	// Event struct types
	for _, event := range iface.Events {
		eventName := caseAndRegister(event.Name)
		fmt.Fprintf(&eventBuffer, "\ntype %s%sEvent struct {\n", ifaceName, eventName)
		for _, arg := range event.Args {
			if t, ok := wlTypes[arg.Type]; ok { // if basic type
				if arg.Type == "uint" && arg.Enum != "" { // enum type
					enumTypeName := ifaceName + CamelCase(arg.Enum)
					fmt.Fprintf(&eventBuffer, "%s %s\n", CamelCase(arg.Name), enumTypeName)
				} else {
					fmt.Fprintf(&eventBuffer, "%s %s\n", CamelCase(arg.Name), t)
				}
			} else { // interface type
				if (arg.Type == "object" || arg.Type == "new_id") && arg.Interface != "" {
					t = "*" + wlNames[arg.Interface]
				} else {
					t = "Proxy"
				}
				fmt.Fprintf(&eventBuffer, "%s %s\n", CamelCase(arg.Name), t)
			}
		}

		eventNames = append(eventNames, eventName)
		fmt.Fprint(&eventBuffer, "}\n")
	}

	return eventBuffer, eventNames
}

func requestArgs(ifaceName string, req Request) *bytes.Buffer {
	var (
		args       []string
		argsBuffer bytes.Buffer
	)

	for _, arg := range req.Args {
		// special type, for example registry.bind
		if arg.Type == "new_id" {
			if arg.Interface == "" {
				args = append(args, "iface string")
				args = append(args, "version uint32")
				args = append(args, fmt.Sprintf("%s Proxy", arg.Name))
			} else {
				continue
			}
		} else if arg.Type == "object" && arg.Interface != "" {
			argTypeName := wlNames[arg.Interface]
			args = append(args, fmt.Sprintf("%s *%s", arg.Name, argTypeName))
		} else if arg.Type == "uint" && arg.Enum != "" {
			args = append(args, fmt.Sprintf("%s %s", arg.Name, enumArgName(ifaceName, arg.Enum)))
		} else {
			args = append(args, fmt.Sprintf("%s %s", arg.Name, wlTypes[arg.Type]))
		}
	}

	fmt.Fprint(&argsBuffer, strings.Join(args, ","))

	return &argsBuffer
}

func requestRets(req Request) *bytes.Buffer {
	var (
		rets       []string
		retsBuffer bytes.Buffer
	)

	for _, arg := range req.Args {
		if arg.Type == "new_id" && arg.Interface != "" {
			retTypeName := wlNames[arg.Interface]
			rets = append(rets, fmt.Sprintf("*%s", retTypeName))
		}
	}

	// all request have an error return
	rets = append(rets, "error")

	retstr := strings.Join(rets, ",")

	if len(rets) > 1 {
		fmt.Fprintf(&retsBuffer, "( %s )", retstr)
	} else {
		fmt.Fprint(&retsBuffer, retstr)
	}

	return &retsBuffer
}

func requestBody(req Request, reqCodeName string) *bytes.Buffer {
	var (
		params       []string
		bodyBuffer   bytes.Buffer
		paramsBuffer bytes.Buffer
		hasRet       string
	)

	for _, arg := range req.Args {
		if arg.Type == "new_id" {
			if arg.Interface != "" {
				retTypeName := wlNames[arg.Interface]
				fmt.Fprintf(&bodyBuffer, "ret := New%s(p.Connection())\n", retTypeName)
				params = append(params, "Proxy(ret)")
				hasRet = "ret,"
			} else {
				params = append(params, "iface")
				params = append(params, "version")
				params = append(params, arg.Name)
			}
		} else {
			params = append(params, arg.Name)
		}
	}

	for _, param := range params {
		fmt.Fprintf(&paramsBuffer, ",%s", param)
	}

	fmt.Fprintf(&bodyBuffer, "return %s p.Connection().SendRequest(p,%s%s)", hasRet, reqCodeName, paramsBuffer.String())

	return &bodyBuffer
}

func getDevelXml() (*os.File, error) {
	url := "https://cgit.freedesktop.org/wayland/wayland/plain/protocol/wayland.xml"
	resp, err := http.Get(url)
	if err != nil {
		return nil, fmt.Errorf("http get error")
	}

	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("Cannot get wayland.xml StatusCode != StatusOK")
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("Cannot read response body: %s", err)
	} else {
		file, err := ioutil.TempFile("", "devel_wayland_xml")
		if err != nil {
			return nil, fmt.Errorf("Cannot create temp file: %s", err)
		} else {
			file.Write(body)
			return file, nil
		}
	}
}

func fmtFile() {
	goex, err := exec.LookPath("go")
	if err != nil {
		log.Printf("go executable cannot found run \"go fmt client.go\" yourself: %s", err)
	} else {
		cmd := exec.Command(goex, "fmt", "client.go")
		err := cmd.Run()
		if err != nil {
			log.Fatalf("Cannot run cmd : %s", err)
		}
	}
}
<|endoftext|>


<commit_before>package config

import (
	"bufio"
	"bytes"
	"errors"
	"io"
	"unicode"
)

// TokenType is the type of tokem that will be returned by the Scanner.
type TokenType int

var eof = rune(-1)

// The following are the token types that are recognized by the scanner
const (
	EOF TokenType = iota
	Comment
	Section
	WhiteSpace
	NewLine
	Ident
	Operand
	OpenBrace
	ClosingBrace
)

// Token is the identifier for a check of text.
type Token struct {
	Type   TokenType
	Text   string
	Line   int
	Column int
}

// Scanner is a lexical scanner for scanning configuration files.
// This works only on UTF-& text.
type Scanner struct {
	r      *bufio.Reader
	txt    *bytes.Buffer
	line   int
	err    error
	column int
}

// NewScanner takes src and returns a new Scanner.
func NewScanner(src io.Reader) *Scanner {
	return &Scanner{
		r:   bufio.NewReader(src),
		txt: &bytes.Buffer{},
	}
}

//Scan returns a new token for every call by advancing on the consumed UTF-8
//encoded input text.
//
// Anything after ; is considered a comment. White space is preserved together
// with  new lines. New lines and spaces are interpreted differently.
func (s *Scanner) Scan() (*Token, error) {
	ch := s.peek()
	if isIdent(ch) {
		return s.scanIdent()
	}
	switch ch {
	case ';':
		return s.scanComment()
	case ' ', '\t':
		return s.scanWhitespace()
	case '\n', '\r':
		return s.scanNewline()
	case '=':
		return s.scanRune(Operand)
	case '[':
		return s.scanRune(OpenBrace)
	case ']':
		return s.scanRune(ClosingBrace)
	case eof:
		return nil, io.EOF
	}
	return nil, errors.New("unrecognized token " + string(ch))
}

//scanComment scans the input for Comments, only single line comments are
//supported.
//
// A comment is all the text that is after a comment identifier, This does not
// enforce the identifier, so it is up to the caller to decide where the comment
// starts, this will read all the text up to the end of the line and return it
// as a single comment token.
//
// TODO(gernest) accept the comment identifier, or check whether the first
// rune is the supported token identifier.
func (s *Scanner) scanComment() (*Token, error) {
	tok := &Token{}
	buf := &bytes.Buffer{}
END:
	for {
		ch, _, err := s.r.ReadRune()
		if err != nil {
			if err.Error() == io.EOF.Error() {

				break END
			}
			return nil, err
		}
		switch ch {
		case '\n', '\r':
			s.r.UnreadRune()
			break END
		default:
			buf.WriteRune(ch)
		}
	}
	s.column++
	tok.Column = s.column
	tok.Type = Comment
	tok.Text = buf.String()
	tok.Line = s.line
	return tok, nil
}

//scanWhitespace scans all utf-8 white space characters until it hits a non
//whitespace character.
//
// Tabs ('\t') and space(' ') all represent white space.
func (s *Scanner) scanWhitespace() (*Token, error) {
	tok := &Token{}

	// There can be arbitrary spaces so we need to bugger them up.
	buf := &bytes.Buffer{}
END:
	for {
		ch, _, err := s.r.ReadRune()
		if err != nil {
			if err.Error() == io.EOF.Error() {
				break END
			}
			return nil, err
		}
		switch ch {
		case ' ', '\t':
			buf.WriteRune(ch)
		default:
			// Stop after hitting non whitespace character
			// Reseting the buffer is necessary so that the scanned character can be
			// accessed for the next call to Scan method.
			s.r.UnreadRune()
			break END
		}
	}
	tok.Column = s.column
	tok.Type = Comment
	tok.Text = buf.String()
	tok.Line = s.line
	return tok, nil
}

//scanNewline returns a token of type NewLine. It is necessary to separate
//newlines from normal spaces because many configuration files formats make use
//of new lines.
//
// A new line can either be a carriage return( '\r') or a new line
// character('\n')
//
// TODO(gernest) accept a new line character as input.
func (s *Scanner) scanNewline() (*Token, error) {
	ch, _, err := s.r.ReadRune()
	if err != nil {
		return nil, err
	}
	tok := &Token{}
	tok.Type = NewLine
	tok.Text = string(ch)
	s.column = 0
	s.line++
	tok.Column = s.column
	tok.Line = s.line
	return tok, nil
}

//isIdent returns true if ch is a valid identifier
// valid identifiers are
//	underscore _
//	dash -
//	plus +
//	a unicode letter a-zA-Z
//	a unicode digit 0-9
func isIdent(ch rune) bool {
	return ch == '_' || ch == '-' || ch == '+' || unicode.IsLetter(ch) || unicode.IsDigit(ch)
}

//scanIdent returns the current character in the input source as an Ident Token
//
// TODO(gernest) Accept the character as input argument.
func (s *Scanner) scanIdent() (*Token, error) {
	return s.scanRune(Ident)
}

// scanRune scans the current rune and returns a token of type typ, whose Text
// is the scanned character
//
// Use this for single character tokens
func (s *Scanner) scanRune(typ TokenType) (*Token, error) {
	ch, _, err := s.r.ReadRune()
	if err != nil {
		return nil, err
	}
	tok := &Token{}
	tok.Type = typ
	tok.Text = string(ch)
	s.column++
	tok.Column = s.column
	tok.Line = s.line
	return tok, nil
}

// peek returns the next rune in the input buffer but does not advance the
// position of the current buffer.
//
// This is a safe way to peek at the next  rune character without actually
// reading it.
func (s *Scanner) peek() rune {
	ch, _, err := s.r.ReadRune()
	defer s.r.UnreadRune()
	if err != nil {
		if err.Error() == io.EOF.Error() {
			return eof
		}
		panic(err)
	}
	return ch
}
<commit_msg>Fix linting errors<commit_after>package config

import (
	"bufio"
	"bytes"
	"errors"
	"io"
	"unicode"
)

// TokenType is the type of tokem that will be returned by the Scanner.
type TokenType int

var eof = rune(-1)

// The following are the token types that are recognized by the scanner
const (
	EOF TokenType = iota
	Comment
	Section
	WhiteSpace
	NewLine
	Ident
	Operand
	OpenBrace
	ClosingBrace
)

// Token is the identifier for a check of text.
type Token struct {
	Type   TokenType
	Text   string
	Line   int
	Column int
}

// Scanner is a lexical scanner for scanning configuration files.
// This works only on UTF-& text.
type Scanner struct {
	r      *bufio.Reader
	txt    *bytes.Buffer
	line   int
	err    error
	column int
}

// NewScanner takes src and returns a new Scanner.
func NewScanner(src io.Reader) *Scanner {
	return &Scanner{
		r:   bufio.NewReader(src),
		txt: &bytes.Buffer{},
	}
}

//Scan returns a new token for every call by advancing on the consumed UTF-8
//encoded input text.
//
// Anything after ; is considered a comment. White space is preserved together
// with  new lines. New lines and spaces are interpreted differently.
func (s *Scanner) Scan() (*Token, error) {
	ch := s.peek()
	if isIdent(ch) {
		return s.scanIdent()
	}
	switch ch {
	case ';':
		return s.scanComment()
	case ' ', '\t':
		return s.scanWhitespace()
	case '\n', '\r':
		return s.scanNewline()
	case '=':
		return s.scanRune(Operand)
	case '[':
		return s.scanRune(OpenBrace)
	case ']':
		return s.scanRune(ClosingBrace)
	case eof:
		return nil, io.EOF
	}
	return nil, errors.New("unrecognized token " + string(ch))
}

//scanComment scans the input for Comments, only single line comments are
//supported.
//
// A comment is all the text that is after a comment identifier, This does not
// enforce the identifier, so it is up to the caller to decide where the comment
// starts, this will read all the text up to the end of the line and return it
// as a single comment token.
//
// TODO(gernest) accept the comment identifier, or check whether the first
// rune is the supported token identifier.
func (s *Scanner) scanComment() (*Token, error) {
	tok := &Token{}
	buf := &bytes.Buffer{}
END:
	for {
		ch, _, err := s.r.ReadRune()
		if err != nil {
			if err.Error() == io.EOF.Error() {

				break END
			}
			return nil, err
		}
		switch ch {
		case '\n', '\r':
			_ = s.r.UnreadRune()
			break END
		default:
			_, _ = buf.WriteRune(ch)
		}
	}
	s.column++
	tok.Column = s.column
	tok.Type = Comment
	tok.Text = buf.String()
	tok.Line = s.line
	return tok, nil
}

//scanWhitespace scans all utf-8 white space characters until it hits a non
//whitespace character.
//
// Tabs ('\t') and space(' ') all represent white space.
func (s *Scanner) scanWhitespace() (*Token, error) {
	tok := &Token{}

	// There can be arbitrary spaces so we need to bugger them up.
	buf := &bytes.Buffer{}
END:
	for {
		ch, _, err := s.r.ReadRune()
		if err != nil {
			if err.Error() == io.EOF.Error() {
				break END
			}
			return nil, err
		}
		switch ch {
		case ' ', '\t':
			_, _ = buf.WriteRune(ch)
		default:
			// Stop after hitting non whitespace character
			// Reseting the buffer is necessary so that the scanned character can be
			// accessed for the next call to Scan method.
			_ = s.r.UnreadRune()
			break END
		}
	}
	tok.Column = s.column
	tok.Type = Comment
	tok.Text = buf.String()
	tok.Line = s.line
	return tok, nil
}

//scanNewline returns a token of type NewLine. It is necessary to separate
//newlines from normal spaces because many configuration files formats make use
//of new lines.
//
// A new line can either be a carriage return( '\r') or a new line
// character('\n')
//
// TODO(gernest) accept a new line character as input.
func (s *Scanner) scanNewline() (*Token, error) {
	ch, _, err := s.r.ReadRune()
	if err != nil {
		return nil, err
	}
	tok := &Token{}
	tok.Type = NewLine
	tok.Text = string(ch)
	s.column = 0
	s.line++
	tok.Column = s.column
	tok.Line = s.line
	return tok, nil
}

//isIdent returns true if ch is a valid identifier
// valid identifiers are
//	underscore _
//	dash -
//	plus +
//	a unicode letter a-zA-Z
//	a unicode digit 0-9
func isIdent(ch rune) bool {
	return ch == '_' || ch == '-' || ch == '+' || unicode.IsLetter(ch) || unicode.IsDigit(ch)
}

//scanIdent returns the current character in the input source as an Ident Token
//
// TODO(gernest) Accept the character as input argument.
func (s *Scanner) scanIdent() (*Token, error) {
	return s.scanRune(Ident)
}

// scanRune scans the current rune and returns a token of type typ, whose Text
// is the scanned character
//
// Use this for single character tokens
func (s *Scanner) scanRune(typ TokenType) (*Token, error) {
	ch, _, err := s.r.ReadRune()
	if err != nil {
		return nil, err
	}
	tok := &Token{}
	tok.Type = typ
	tok.Text = string(ch)
	s.column++
	tok.Column = s.column
	tok.Line = s.line
	return tok, nil
}

// peek returns the next rune in the input buffer but does not advance the
// position of the current buffer.
//
// This is a safe way to peek at the next  rune character without actually
// reading it.
func (s *Scanner) peek() rune {
	ch, _, err := s.r.ReadRune()
	if err != nil {
		if err.Error() == io.EOF.Error() {
			return eof
		}
		panic(err)
	}
	_ = s.r.UnreadRune()
	return ch
}
<|endoftext|>


<commit_before>package main

import (
	"archive/zip"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"
)

//unzipFile will un-compress a zip archive
func unzipFile(zipFile zipFile, progress chan map[string]float64) ([]csvFile, error) {

	dest := filepath.Dir(zipFile.path)

	r, err := zip.OpenReader(zipFile.path)
	if err != nil {
		return nil, err
	}
	defer r.Close()

	var result []csvFile

	for _, f := range r.File {
		rc, err := f.Open()
		if err != nil {
			return nil, err
		}
		defer rc.Close()

		fpath := filepath.Join(dest, f.Name)

		file := f
		if f.FileInfo().IsDir() {
			os.MkdirAll(fpath, os.ModePerm)
		} else {
			var fdir string
			if lastIndex := strings.LastIndex(fpath, string(os.PathSeparator)); lastIndex > -1 {
				fdir = fpath[:lastIndex]
			}

			if fdir != "" {
				err = os.MkdirAll(fdir, os.ModePerm)
				if err != nil {
					log.Fatal(err)
					return nil, err
				}
			}

			f, err := os.OpenFile(
				fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
			if err != nil {
				return nil, err
			}
			defer f.Close()

			src := &PassThru{
				Reader:   rc,
				total:    float64(file.FileInfo().Size()),
				filename: zipFile.filename,
				progress: progress,
			}

			_, err = io.Copy(f, src)
			if err != nil {
				return nil, err
			}

			result = append(result, csvFile{
				filename: fpath,
				path:     f.Name(),
			})
		}
	}

	return result, nil
}
<commit_msg>Fix filename and path of csv files<commit_after>package main

import (
	"archive/zip"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"
)

//unzipFile will un-compress a zip archive
func unzipFile(zipFile zipFile, progress chan map[string]float64) ([]csvFile, error) {

	dest := filepath.Dir(zipFile.path)

	r, err := zip.OpenReader(zipFile.path)
	if err != nil {
		return nil, err
	}
	defer r.Close()

	var result []csvFile

	for _, f := range r.File {
		rc, err := f.Open()
		if err != nil {
			return nil, err
		}
		defer rc.Close()

		fpath := filepath.Join(dest, f.Name)

		file := f
		if f.FileInfo().IsDir() {
			os.MkdirAll(fpath, os.ModePerm)
		} else {
			var fdir string
			if lastIndex := strings.LastIndex(fpath, string(os.PathSeparator)); lastIndex > -1 {
				fdir = fpath[:lastIndex]
			}

			if fdir != "" {
				err = os.MkdirAll(fdir, os.ModePerm)
				if err != nil {
					log.Fatal(err)
					return nil, err
				}
			}

			f, err := os.OpenFile(
				fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
			if err != nil {
				return nil, err
			}
			defer f.Close()

			src := &PassThru{
				Reader:   rc,
				total:    float64(file.FileInfo().Size()),
				filename: zipFile.filename,
				progress: progress,
			}

			_, err = io.Copy(f, src)
			if err != nil {
				return nil, err
			}

			result = append(result, csvFile{
				filename: file.Name,
				path:     fpath,
			})
		}
	}

	return result, nil
}
<|endoftext|>