<commit_before>/*-
 * Copyright 2015 Square Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"runtime"
	"time"

	"github.com/rcrowley/go-metrics"
)

// Metrics bridge posts metrics to an HTTP/JSON bridge endpoint
type metricsConfig struct {
	url      string
	registry metrics.Registry
	prefix   string
	hostname string
}

func (mb *metricsConfig) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	metrics := mb.serializeMetrics()
	raw, err := json.Marshal(metrics)
	if err != nil {
		logger.Printf("%s", err)
	}
	w.Write(raw)
}

// Publish metrics to bridge
func (mb *metricsConfig) publishMetrics() {
	for _ = range time.Tick(1 * time.Second) {
		mb.postMetrics()
	}
}

// Collect memory usage metrics
func (mb *metricsConfig) collectSystemMetrics() {
	var mem runtime.MemStats

	alloc := metrics.GetOrRegisterGauge("runtime.mem.alloc", mb.registry)
	totalAlloc := metrics.GetOrRegisterGauge("runtime.mem.total-alloc", mb.registry)
	sys := metrics.GetOrRegisterGauge("runtime.mem.sys", mb.registry)
	heapAlloc := metrics.GetOrRegisterGauge("runtime.mem.heap.alloc", mb.registry)
	heapSys := metrics.GetOrRegisterGauge("runtime.mem.heap.sys", mb.registry)
	heapInUse := metrics.GetOrRegisterGauge("runtime.mem.heap.in-use", mb.registry)
	stackSys := metrics.GetOrRegisterGauge("runtime.mem.stack.sys", mb.registry)
	stackInUse := metrics.GetOrRegisterGauge("runtime.mem.stack.in-use", mb.registry)
	gcPauseTotal := metrics.GetOrRegisterGauge("runtime.mem.gc.pause-total", mb.registry)
	gcCPUFraction := metrics.GetOrRegisterGaugeFloat64("runtime.mem.gc.cpu-fraction", mb.registry)
	numGoRoutines := metrics.GetOrRegisterGauge("runtime.goroutines", mb.registry)

	for _ = range time.Tick(1 * time.Second) {
		runtime.ReadMemStats(&mem)

		alloc.Update(int64(mem.Alloc))
		totalAlloc.Update(int64(mem.TotalAlloc))
		sys.Update(int64(mem.Sys))

		heapAlloc.Update(int64(mem.HeapAlloc))
		heapSys.Update(int64(mem.HeapSys))
		heapInUse.Update(int64(mem.HeapInuse))

		stackSys.Update(int64(mem.StackSys))
		stackInUse.Update(int64(mem.StackInuse))

		gcPauseTotal.Update(int64(mem.PauseTotalNs))
		gcCPUFraction.Update(mem.GCCPUFraction)

		numGoRoutines.Update(int64(runtime.NumGoroutine()))
	}
}

func (mb *metricsConfig) postMetrics() {
	metrics := mb.serializeMetrics()
	raw, err := json.Marshal(metrics)
	if err != nil {
		logger.Printf("%s", err)
	}
	http.Post(mb.url, "application/json", bytes.NewReader(raw))
}

func (mb *metricsConfig) serializeMetric(now int64, metric tuple) map[string]interface{} {
	return map[string]interface{}{
		"timestamp": now,
		"metric":    fmt.Sprintf("%s.%s", mb.prefix, metric.name),
		"value":     metric.value,
		"hostname":  mb.hostname,
	}
}

type tuple struct {
	name  string
	value interface{}
}

func (mb *metricsConfig) serializeMetrics() []map[string]interface{} {
	nvs := []tuple{}

	mb.registry.Each(func(name string, i interface{}) {
		switch metric := i.(type) {
		case metrics.Counter:
			nvs = append(nvs, tuple{name, metric.Count()})
		case metrics.Gauge:
			nvs = append(nvs, tuple{name, metric.Value()})
		case metrics.GaugeFloat64:
			nvs = append(nvs, tuple{name, metric.Value()})
		case metrics.Timer:
			timer := metric.Snapshot()
			nvs = append(nvs, []tuple{
				{fmt.Sprintf("%s.count", name), timer.Count()},
				{fmt.Sprintf("%s.min", name), timer.Min()},
				{fmt.Sprintf("%s.max", name), timer.Max()},
				{fmt.Sprintf("%s.mean", name), timer.Mean()},
				{fmt.Sprintf("%s.std-dev", name), timer.StdDev()},
				{fmt.Sprintf("%s.one-minute", name), timer.Rate1()},
				{fmt.Sprintf("%s.five-minute", name), timer.Rate5()},
				{fmt.Sprintf("%s.fifteen-minute", name), timer.Rate15()},
				{fmt.Sprintf("%s.mean-rate", name), timer.RateMean()},
				{fmt.Sprintf("%s.50-percentile", name), timer.Percentile(0.5)},
				{fmt.Sprintf("%s.75-percentile", name), timer.Percentile(0.75)},
				{fmt.Sprintf("%s.95-percentile", name), timer.Percentile(0.95)},
				{fmt.Sprintf("%s.99-percentile", name), timer.Percentile(0.99)},
				{fmt.Sprintf("%s.999-percentile", name), timer.Percentile(0.999)},
			}...)
		}
	})

	now := time.Now().Unix()
	out := []map[string]interface{}{}
	for _, nv := range nvs {
		out = append(out, mb.serializeMetric(now, nv))
	}

	return out
}
<commit_msg>Add metric for number of Cgo calls<commit_after>/*-
 * Copyright 2015 Square Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"runtime"
	"time"

	"github.com/rcrowley/go-metrics"
)

// Metrics bridge posts metrics to an HTTP/JSON bridge endpoint
type metricsConfig struct {
	url      string
	registry metrics.Registry
	prefix   string
	hostname string
}

func (mb *metricsConfig) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	metrics := mb.serializeMetrics()
	raw, err := json.Marshal(metrics)
	if err != nil {
		logger.Printf("%s", err)
	}
	w.Write(raw)
}

// Publish metrics to bridge
func (mb *metricsConfig) publishMetrics() {
	for _ = range time.Tick(1 * time.Second) {
		mb.postMetrics()
	}
}

// Collect memory usage metrics
func (mb *metricsConfig) collectSystemMetrics() {
	var mem runtime.MemStats

	alloc := metrics.GetOrRegisterGauge("runtime.mem.alloc", mb.registry)
	totalAlloc := metrics.GetOrRegisterGauge("runtime.mem.total-alloc", mb.registry)
	sys := metrics.GetOrRegisterGauge("runtime.mem.sys", mb.registry)
	heapAlloc := metrics.GetOrRegisterGauge("runtime.mem.heap.alloc", mb.registry)
	heapSys := metrics.GetOrRegisterGauge("runtime.mem.heap.sys", mb.registry)
	heapInUse := metrics.GetOrRegisterGauge("runtime.mem.heap.in-use", mb.registry)
	stackSys := metrics.GetOrRegisterGauge("runtime.mem.stack.sys", mb.registry)
	stackInUse := metrics.GetOrRegisterGauge("runtime.mem.stack.in-use", mb.registry)
	gcPauseTotal := metrics.GetOrRegisterGauge("runtime.mem.gc.pause-total", mb.registry)
	gcCPUFraction := metrics.GetOrRegisterGaugeFloat64("runtime.mem.gc.cpu-fraction", mb.registry)
	numGoRoutines := metrics.GetOrRegisterGauge("runtime.goroutines", mb.registry)
	numCgoCalls := metrics.GetOrRegisterGauge("runtime.cgo-calls", mb.registry)

	for _ = range time.Tick(1 * time.Second) {
		runtime.ReadMemStats(&mem)

		alloc.Update(int64(mem.Alloc))
		totalAlloc.Update(int64(mem.TotalAlloc))
		sys.Update(int64(mem.Sys))

		heapAlloc.Update(int64(mem.HeapAlloc))
		heapSys.Update(int64(mem.HeapSys))
		heapInUse.Update(int64(mem.HeapInuse))

		stackSys.Update(int64(mem.StackSys))
		stackInUse.Update(int64(mem.StackInuse))

		gcPauseTotal.Update(int64(mem.PauseTotalNs))
		gcCPUFraction.Update(mem.GCCPUFraction)

		numGoRoutines.Update(int64(runtime.NumGoroutine()))
		numCgoCalls.Update(int64(runtime.NumCgoCall()))
	}
}

func (mb *metricsConfig) postMetrics() {
	metrics := mb.serializeMetrics()
	raw, err := json.Marshal(metrics)
	if err != nil {
		logger.Printf("%s", err)
	}
	http.Post(mb.url, "application/json", bytes.NewReader(raw))
}

func (mb *metricsConfig) serializeMetric(now int64, metric tuple) map[string]interface{} {
	return map[string]interface{}{
		"timestamp": now,
		"metric":    fmt.Sprintf("%s.%s", mb.prefix, metric.name),
		"value":     metric.value,
		"hostname":  mb.hostname,
	}
}

type tuple struct {
	name  string
	value interface{}
}

func (mb *metricsConfig) serializeMetrics() []map[string]interface{} {
	nvs := []tuple{}

	mb.registry.Each(func(name string, i interface{}) {
		switch metric := i.(type) {
		case metrics.Counter:
			nvs = append(nvs, tuple{name, metric.Count()})
		case metrics.Gauge:
			nvs = append(nvs, tuple{name, metric.Value()})
		case metrics.GaugeFloat64:
			nvs = append(nvs, tuple{name, metric.Value()})
		case metrics.Timer:
			timer := metric.Snapshot()
			nvs = append(nvs, []tuple{
				{fmt.Sprintf("%s.count", name), timer.Count()},
				{fmt.Sprintf("%s.min", name), timer.Min()},
				{fmt.Sprintf("%s.max", name), timer.Max()},
				{fmt.Sprintf("%s.mean", name), timer.Mean()},
				{fmt.Sprintf("%s.std-dev", name), timer.StdDev()},
				{fmt.Sprintf("%s.one-minute", name), timer.Rate1()},
				{fmt.Sprintf("%s.five-minute", name), timer.Rate5()},
				{fmt.Sprintf("%s.fifteen-minute", name), timer.Rate15()},
				{fmt.Sprintf("%s.mean-rate", name), timer.RateMean()},
				{fmt.Sprintf("%s.50-percentile", name), timer.Percentile(0.5)},
				{fmt.Sprintf("%s.75-percentile", name), timer.Percentile(0.75)},
				{fmt.Sprintf("%s.95-percentile", name), timer.Percentile(0.95)},
				{fmt.Sprintf("%s.99-percentile", name), timer.Percentile(0.99)},
				{fmt.Sprintf("%s.999-percentile", name), timer.Percentile(0.999)},
			}...)
		}
	})

	now := time.Now().Unix()
	out := []map[string]interface{}{}
	for _, nv := range nvs {
		out = append(out, mb.serializeMetric(now, nv))
	}

	return out
}
<|endoftext|>


<commit_before>// See LICENSE.txt for licensing information.

package main

import (
	"fmt"
	"os"
)

func (c *Config) Migrate() error {
	if c.Version < 1 {
		fmt.Println("Migrating -> 1")
		for _, g := range c.Games {
			fmt.Print("g")
			for _, s := range g.Saves {
				if s.Size > 0 {
					fmt.Print("S")
					continue
				}
				i, err := os.Stat(s.Path)
				if err != nil {
					return err
				}
				fmt.Print("s")
				size := uint64(i.Size())
				s.Size = size
				g.Size += size
			}
		}
		fmt.Println("")
	}
	return nil
}
<commit_msg>Use const in migrate.go<commit_after>// See LICENSE.txt for licensing information.

package main

import (
	"fmt"
	"os"
)

const (
	CFG_VER_1 = iota + 1
)

func (c *Config) Migrate() error {
	if c.Version < CFG_VER_1 {
		fmt.Println("Migrating -> 1")
		for _, g := range c.Games {
			fmt.Print("g")
			for _, s := range g.Saves {
				if s.Size > 0 {
					fmt.Print("S")
					continue
				}
				i, err := os.Stat(s.Path)
				if err != nil {
					return err
				}
				fmt.Print("s")
				size := uint64(i.Size())
				s.Size = size
				g.Size += size
			}
		}
		c.Version = CFG_VER_1
		fmt.Println("")
	}
	return nil
}
<|endoftext|>


<commit_before>package pcstat

/*
 * Copyright 2014 Albert P. Tobey <atobey@datastax.com> @AlTobey
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import (
	"fmt"
	"os"
	"unsafe"

	"golang.org/x/sys/unix"
)

// mmap the given file, get the mincore vector, then
// return it as an []bool
func FileMincore(f *os.File, size int64) ([]bool, error) {
        //skip could not mmap error when the file size is 0
        if int(size) == 0 {
           return nil, nil
        }
	// mmap is a []byte
	mmap, err := unix.Mmap(int(f.Fd()), 0, int(size), unix.PROT_NONE, unix.MAP_SHARED)
	if err != nil {
		return nil, fmt.Errorf("could not mmap: %v", err)
	}
	// TODO: check for MAP_FAILED which is ((void *) -1)
	// but maybe unnecessary since it looks like errno is always set when MAP_FAILED

	// one byte per page, only LSB is used, remainder is reserved and clear
	vecsz := (size + int64(os.Getpagesize()) - 1) / int64(os.Getpagesize())
	vec := make([]byte, vecsz)

	// get all of the arguments to the mincore syscall converted to uintptr
	mmap_ptr := uintptr(unsafe.Pointer(&mmap[0]))
	size_ptr := uintptr(size)
	vec_ptr := uintptr(unsafe.Pointer(&vec[0]))

	// use Go's ASM to submit directly to the kernel, no C wrapper needed
	// mincore(2): int mincore(void *addr, size_t length, unsigned char *vec);
	// 0 on success, takes the pointer to the mmap, a size, which is the
	// size that came from f.Stat(), and the vector, which is a pointer
	// to the memory behind an []byte
	// this writes a snapshot of the data into vec which a list of 8-bit flags
	// with the LSB set if the page in that position is currently in VFS cache
	ret, _, err := unix.Syscall(unix.SYS_MINCORE, mmap_ptr, size_ptr, vec_ptr)
	if ret != 0 {
		return nil, fmt.Errorf("syscall SYS_MINCORE failed: %v", err)
	}
	defer unix.Munmap(mmap)

	mc := make([]bool, vecsz)

	// there is no bitshift only bool
	for i, b := range vec {
		if b%2 == 1 {
			mc[i] = true
		} else {
			mc[i] = false
		}
	}

	return mc, nil
}
<commit_msg>go fmt<commit_after>package pcstat

/*
 * Copyright 2014 Albert P. Tobey <atobey@datastax.com> @AlTobey
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import (
	"fmt"
	"os"
	"unsafe"

	"golang.org/x/sys/unix"
)

// mmap the given file, get the mincore vector, then
// return it as an []bool
func FileMincore(f *os.File, size int64) ([]bool, error) {
	//skip could not mmap error when the file size is 0
	if int(size) == 0 {
		return nil, nil
	}
	// mmap is a []byte
	mmap, err := unix.Mmap(int(f.Fd()), 0, int(size), unix.PROT_NONE, unix.MAP_SHARED)
	if err != nil {
		return nil, fmt.Errorf("could not mmap: %v", err)
	}
	// TODO: check for MAP_FAILED which is ((void *) -1)
	// but maybe unnecessary since it looks like errno is always set when MAP_FAILED

	// one byte per page, only LSB is used, remainder is reserved and clear
	vecsz := (size + int64(os.Getpagesize()) - 1) / int64(os.Getpagesize())
	vec := make([]byte, vecsz)

	// get all of the arguments to the mincore syscall converted to uintptr
	mmap_ptr := uintptr(unsafe.Pointer(&mmap[0]))
	size_ptr := uintptr(size)
	vec_ptr := uintptr(unsafe.Pointer(&vec[0]))

	// use Go's ASM to submit directly to the kernel, no C wrapper needed
	// mincore(2): int mincore(void *addr, size_t length, unsigned char *vec);
	// 0 on success, takes the pointer to the mmap, a size, which is the
	// size that came from f.Stat(), and the vector, which is a pointer
	// to the memory behind an []byte
	// this writes a snapshot of the data into vec which a list of 8-bit flags
	// with the LSB set if the page in that position is currently in VFS cache
	ret, _, err := unix.Syscall(unix.SYS_MINCORE, mmap_ptr, size_ptr, vec_ptr)
	if ret != 0 {
		return nil, fmt.Errorf("syscall SYS_MINCORE failed: %v", err)
	}
	defer unix.Munmap(mmap)

	mc := make([]bool, vecsz)

	// there is no bitshift only bool
	for i, b := range vec {
		if b%2 == 1 {
			mc[i] = true
		} else {
			mc[i] = false
		}
	}

	return mc, nil
}
<|endoftext|>


<commit_before>/*
 * Mini Object Storage, (C) 2014,2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"fmt"
	"os"
	"sort"
	"time"

	"github.com/cheggaaa/pb"
	"github.com/codegangsta/cli"
	"github.com/minio-io/mc/pkg/s3"
)

const (
	recvFormat  = "2006-01-02T15:04:05.000Z"
	printFormat = "2006-01-02 15:04:05"
)

func parseTime(t string) string {
	ti, _ := time.Parse(recvFormat, t)
	return ti.Format(printFormat)
}

func parseLastModified(t string) string {
	ti, _ := time.Parse(time.RFC1123, t)
	return ti.Format(printFormat)
}

func printBuckets(v []*s3.Bucket) {
	for _, b := range v {
		msg := fmt.Sprintf("%s   %s", parseTime(b.CreationDate), b.Name)
		info(msg)
	}
}

func printObjects(v []*s3.Item) {
	if len(v) > 0 {
		sort.Sort(s3.BySize(v))
		for _, b := range v {
			printObject(b.LastModified, b.Size, b.Key)
		}
	}
}

func printObject(date string, v int64, key string) {
	msg := fmt.Sprintf("%s  %13s %s", parseTime(date), pb.FormatBytes(v), key)
	info(msg)
}

func doFsList(c *cli.Context) {
	var items []*s3.Item

	config, err := getMcConfig()
	if err != nil {
		fatal(err.Error())
	}

	s3c, err := getNewClient(config)
	if err != nil {
		fatal(err.Error())
	}
	fsoptions, err := parseOptions(c)
	if err != nil {
		fatal(err.Error())
	}
	switch true {
	case fsoptions.bucket == "":
		buckets, err := s3c.Buckets()
		if err != nil {
			fatal(err.Error())
		}
		printBuckets(buckets)
	case fsoptions.key == "":
		items, _, err = s3c.GetBucket(fsoptions.bucket, "", "", "", s3.MaxKeys)
		if err != nil {
			fatal(err.Error())
		}
		printObjects(items)
	case fsoptions.key != "":
		var date string
		var size int64

		size, date, err = s3c.Stat(fsoptions.key, fsoptions.bucket)
		switch true {
		case err == os.ErrNotExist:
			// Try sending a GetBucket
			items, _, err = s3c.GetBucket(fsoptions.bucket, "", fsoptions.key, "", s3.MaxKeys)
			if err != nil {
				fatal(err.Error())
			}
			// If items null list, send HEAD request again
			if len(items) == 0 {
				_, _, err = s3c.Stat(fsoptions.key, fsoptions.bucket)
				if err == os.ErrNotExist {
					fatal(err.Error())
				}
			}
			printObjects(items)
		case err == nil:
			printObject(date, size, fsoptions.key)
		default:
			fatal(err.Error())
		}
	}
}
<commit_msg>added comments<commit_after>/*
 * Mini Object Storage, (C) 2014,2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"fmt"
	"os"
	"sort"
	"time"

	"github.com/cheggaaa/pb"
	"github.com/codegangsta/cli"
	"github.com/minio-io/mc/pkg/s3"
)

const (
	recvFormat  = "2006-01-02T15:04:05.000Z"
	printFormat = "2006-01-02 15:04:05"
)

func parseTime(t string) string {
	ti, _ := time.Parse(recvFormat, t)
	return ti.Format(printFormat)
}

func parseLastModified(t string) string {
	ti, _ := time.Parse(time.RFC1123, t)
	return ti.Format(printFormat)
}

func printBuckets(v []*s3.Bucket) {
	for _, b := range v {
		msg := fmt.Sprintf("%s   %s", parseTime(b.CreationDate), b.Name)
		info(msg)
	}
}

func printObjects(v []*s3.Item) {
	if len(v) > 0 {
		sort.Sort(s3.BySize(v))
		for _, b := range v {
			printObject(b.LastModified, b.Size, b.Key)
		}
	}
}

func printObject(date string, v int64, key string) {
	msg := fmt.Sprintf("%s  %13s %s", parseTime(date), pb.FormatBytes(v), key)
	info(msg)
}

func doFsList(c *cli.Context) {
	var items []*s3.Item

	config, err := getMcConfig()
	if err != nil {
		fatal(err.Error())
	}

	s3c, err := getNewClient(config)
	if err != nil {
		fatal(err.Error())
	}
	fsoptions, err := parseOptions(c)
	if err != nil {
		fatal(err.Error())
	}
	switch true {
	case fsoptions.bucket == "": // List all buckets
		buckets, err := s3c.Buckets()
		if err != nil {
			fatal(err.Error())
		}
		printBuckets(buckets)
	case fsoptions.key == "": // List the objects in a bucket
		items, _, err = s3c.GetBucket(fsoptions.bucket, "", "", "", s3.MaxKeys)
		if err != nil {
			fatal(err.Error())
		}
		printObjects(items)
	case fsoptions.key != "": // List objects matching the key prefix
		var date string
		var size int64

		size, date, err = s3c.Stat(fsoptions.key, fsoptions.bucket)
		switch err {
		case nil: // List a single object. Exact key prefix match
			printObject(date, size, fsoptions.key)
		case os.ErrNotExist:
			// List all objects matching the key prefix
			items, _, err = s3c.GetBucket(fsoptions.bucket, "", fsoptions.key, "", s3.MaxKeys)
			if err != nil {
				fatal(err.Error())
			}
			if len(items) > 0 {
				printObjects(items)
			} else {
				fatal(os.ErrNotExist.Error())
			}
		default: // Error
			fatal(err.Error())
		}
	default:
		fatal(err.Error())
	}
}
<|endoftext|>


<commit_before>/*
Package minhash provides probabilistic data structures for computing
MinHash signatures for streaming data.

The reader should also conf https://github.com/dgryski/go-minhash and
https://github.com/tylertreat/BoomFilters.  In fact, most of the
implementation details of this package are based off the former.

MinHash signatures can be used to estimate the Jaccard index
J(A, B) := |A & B| / |A || B| of two sets that are subsets
of some finite, totally ordered set U.  If s is a permutation
of U chosen uniformly at random, then x := argmin s(A || B) is
just a random element chosen uniformly from A || B.  It's
clear that P(x in A & B) = J(A, B).  Since min s(A) = min s(B)
if and only if x is in A & B, we have just shown that
P(min s(A) = min S(B)) = J(A, B).

The central idea of minhash signatures is to repeatedly perform
the above experiment with different permutations as a way to estimate
the underlying probability
J(A, B) = P(an element x in A || B is also  in A & B).

A length k minhash signature S(A) is theoretically generated by randomly
choosing k permutations si (i=1, ..., k) in the symmetric group of U
(group of bijective endofunctions on U) and computing  hi(A) := min si(A)
for each permutation.  We take S(A) := [h1(A), ..., hk(A)].
Since each permutation is a bijection, min si(A) = min si(B) if and
only if argmin si(A) = argmin si(B) and so we could just as
well use these argmins, which is sometimes how the signature S(A) is
defined.

Specifying permutations for large U is not efficient, and so we often
take a family of integer-valued hash functions that are minwise
independent, in the sense that for most sets A,
min h(A) ! = min g(A) for two distinct hash functions in the family.
Frequently this family is parametrically  generated.

For more information,
    http://research.neustar.biz/2012/07/09/sketch-of-the-day-k-minimum-values/

    MinHashing:
    http://infolab.stanford.edu/~ullman/mmds/ch3.pdf
    https://en.wikipedia.org/wiki/MinHash

    BottomK:
    http://www.math.tau.ac.il/~haimk/papers/p225-cohen.pdf
    http://cohenwang.org/edith/Papers/metrics394-cohen.pdf

    http://www.mpi-inf.mpg.de/~rgemulla/publications/beyer07distinct.pdf

This package works best when provided with a strong 64-bit hash function,
such as CityHash, Spooky, Murmur3, or SipHash.

*/

package minhash

import (
	"encoding/binary"
	"log"
	"math"
	"strconv"
)

const (
	MaxUint  uint   = ^uint(0)
	MaxInt   int    = int(MaxUint >> 1)
	infinity uint64 = math.MaxUint64
)

type HashFunc func([]byte) uint64

// Signature is an array representing the signature of a set.
type Signature []uint64

// // MinHash is an a probabilistic data structure used to
// // compute a similarity preserving signature for a set.  It ingests
// // a stream of the set's elements and continuously updates the signature.
type MinHash interface {
	// 	// Push ingests a set element, hashes it, and updates the signature.
	// 	Push(interface{})

	// 	// Merge updates the signature of the instance with the signature
	// 	// of the input.  This results in the signature of the union of the
	// 	// two sets, which is stored in the original MinHash instance.
	// 	Merge(MinHash)

	// 	// Cardinality estimates the size of the set from the signature.
	// 	Cardinality() int

	// Signature returns the signature itself.
	Signature() []uint64

	// 	// Similarity computes the similarity between two MinHash signatures.
	// 	// The method for computing similarity depends on whether a MinWise
	// 	// or Bottom-K implementation is used.
	// 	Similarity(MinHash) float64
	// }

	// // Similarity invokes the MinHash implementation's Similarity method.
	// func Similarity(m1, m2 MinHash) float64 {
	// 	return m1.Similarity(m2)
}

// defaultSignature will return an appropriately typed array
func defaultSignature(size int) Signature {
	s := make(Signature, size)
	for i := range s {
		s[i] = infinity
	}
	return s
}

// toBytes converts various types to a byte slice
// so they can be pushed into a MinHash instance.
func toBytes(x interface{}) []byte {
	b := make([]byte, 8)
	switch t := x.(type) {
	case []byte:
		b = t
	case string:
		b = []byte(t)
	case uint:
		binary.LittleEndian.PutUint64(b, uint64(t))
	case uint16:
		binary.LittleEndian.PutUint64(b, uint64(t))
	case uint32:
		binary.LittleEndian.PutUint64(b, uint64(t))
	case uint64:
		binary.LittleEndian.PutUint64(b, t)
	case int:
		binary.LittleEndian.PutUint64(b, uint64(t))
	case int16:
		binary.LittleEndian.PutUint64(b, uint64(t))
	case int32:
		binary.LittleEndian.PutUint64(b, uint64(t))
	case int64:
		binary.LittleEndian.PutUint64(b, uint64(t))
	}
	return b
}

// stringIntToByte converts a string representation of an integer to a byte slice.
func stringIntToBytes(s string) []byte {
	n, err := strconv.ParseUint(s, 0, 64)
	var b []byte
	if err != nil {
		log.Println("Could not convert string to uint64.")
		b = []byte(s)
	} else {
		b = toBytes(n)
	}
	return b
}

// Intersection computes the cardinality of the intersection
// between two sets provided their sizes and Jaccard similarity are known.
func Intersection(js float64, size1, size2 int) int {
	var est int
	if js == 0 {
		est = 0
	} else {
		// If n, m, i, u denote |A|, |B|, |A & B|, and |A || B| respectively,
		// then given that A and B are not disjoint (i != 0), w have
		// J := J(A, B) = i / u which is equivalent to
		// 1/J = u / i = (n + m - i) / i = (n + m / i) - 1.
		// Solving for i yields |A & B| = (n + m) / ((1/J)+ 1).
		est = int(math.Floor(float64(size1+size2) / ((1.0 / js) + 1)))
	}
	return est
}

func IsEmpty(m MinHash) bool {
	var empty = true

	if m != nil {
		// Check whether each minimum is infinite.
		for _, v := range m.Signature() {
			if v < infinity {
				empty = false
				break
			}
		}
	}
	return empty
}
<commit_msg>Stop logging string conversion error.<commit_after>/*
Package minhash provides probabilistic data structures for computing
MinHash signatures for streaming data.

The reader should also conf https://github.com/dgryski/go-minhash and
https://github.com/tylertreat/BoomFilters.  In fact, most of the
implementation details of this package are based off the former.

MinHash signatures can be used to estimate the Jaccard index
J(A, B) := |A & B| / |A || B| of two sets that are subsets
of some finite, totally ordered set U.  If s is a permutation
of U chosen uniformly at random, then x := argmin s(A || B) is
just a random element chosen uniformly from A || B.  It's
clear that P(x in A & B) = J(A, B).  Since min s(A) = min s(B)
if and only if x is in A & B, we have just shown that
P(min s(A) = min S(B)) = J(A, B).

The central idea of minhash signatures is to repeatedly perform
the above experiment with different permutations as a way to estimate
the underlying probability
J(A, B) = P(an element x in A || B is also  in A & B).

A length k minhash signature S(A) is theoretically generated by randomly
choosing k permutations si (i=1, ..., k) in the symmetric group of U
(group of bijective endofunctions on U) and computing  hi(A) := min si(A)
for each permutation.  We take S(A) := [h1(A), ..., hk(A)].
Since each permutation is a bijection, min si(A) = min si(B) if and
only if argmin si(A) = argmin si(B) and so we could just as
well use these argmins, which is sometimes how the signature S(A) is
defined.

Specifying permutations for large U is not efficient, and so we often
take a family of integer-valued hash functions that are minwise
independent, in the sense that for most sets A,
min h(A) ! = min g(A) for two distinct hash functions in the family.
Frequently this family is parametrically  generated.

For more information,
    http://research.neustar.biz/2012/07/09/sketch-of-the-day-k-minimum-values/

    MinHashing:
    http://infolab.stanford.edu/~ullman/mmds/ch3.pdf
    https://en.wikipedia.org/wiki/MinHash

    BottomK:
    http://www.math.tau.ac.il/~haimk/papers/p225-cohen.pdf
    http://cohenwang.org/edith/Papers/metrics394-cohen.pdf

    http://www.mpi-inf.mpg.de/~rgemulla/publications/beyer07distinct.pdf

This package works best when provided with a strong 64-bit hash function,
such as CityHash, Spooky, Murmur3, or SipHash.

*/

package minhash

import (
	"encoding/binary"
	"math"
	"strconv"
)

const (
	MaxUint  uint   = ^uint(0)
	MaxInt   int    = int(MaxUint >> 1)
	infinity uint64 = math.MaxUint64
)

type HashFunc func([]byte) uint64

// Signature is an array representing the signature of a set.
type Signature []uint64

// // MinHash is an a probabilistic data structure used to
// // compute a similarity preserving signature for a set.  It ingests
// // a stream of the set's elements and continuously updates the signature.
type MinHash interface {
	// 	// Push ingests a set element, hashes it, and updates the signature.
	// 	Push(interface{})

	// 	// Merge updates the signature of the instance with the signature
	// 	// of the input.  This results in the signature of the union of the
	// 	// two sets, which is stored in the original MinHash instance.
	// 	Merge(MinHash)

	// 	// Cardinality estimates the size of the set from the signature.
	// 	Cardinality() int

	// Signature returns the signature itself.
	Signature() []uint64

	// 	// Similarity computes the similarity between two MinHash signatures.
	// 	// The method for computing similarity depends on whether a MinWise
	// 	// or Bottom-K implementation is used.
	// 	Similarity(MinHash) float64
	// }

	// // Similarity invokes the MinHash implementation's Similarity method.
	// func Similarity(m1, m2 MinHash) float64 {
	// 	return m1.Similarity(m2)
}

// defaultSignature will return an appropriately typed array
func defaultSignature(size int) Signature {
	s := make(Signature, size)
	for i := range s {
		s[i] = infinity
	}
	return s
}

// toBytes converts various types to a byte slice
// so they can be pushed into a MinHash instance.
func toBytes(x interface{}) []byte {
	b := make([]byte, 8)
	switch t := x.(type) {
	case []byte:
		b = t
	case string:
		b = []byte(t)
	case uint:
		binary.LittleEndian.PutUint64(b, uint64(t))
	case uint16:
		binary.LittleEndian.PutUint64(b, uint64(t))
	case uint32:
		binary.LittleEndian.PutUint64(b, uint64(t))
	case uint64:
		binary.LittleEndian.PutUint64(b, t)
	case int:
		binary.LittleEndian.PutUint64(b, uint64(t))
	case int16:
		binary.LittleEndian.PutUint64(b, uint64(t))
	case int32:
		binary.LittleEndian.PutUint64(b, uint64(t))
	case int64:
		binary.LittleEndian.PutUint64(b, uint64(t))
	}
	return b
}

// stringIntToByte converts a string representation of an integer to a byte slice.
func stringIntToBytes(s string) []byte {
	n, err := strconv.ParseUint(s, 0, 64)
	var b []byte
	if err != nil {
		// Use literal conversion if string conversion failed.
		b = []byte(s)
	} else {
		b = toBytes(n)
	}
	return b
}

// Intersection computes the cardinality of the intersection
// between two sets provided their sizes and Jaccard similarity are known.
func Intersection(js float64, size1, size2 int) int {
	var est int
	if js == 0 {
		est = 0
	} else {
		// If n, m, i, u denote |A|, |B|, |A & B|, and |A || B| respectively,
		// then given that A and B are not disjoint (i != 0), w have
		// J := J(A, B) = i / u which is equivalent to
		// 1/J = u / i = (n + m - i) / i = (n + m / i) - 1.
		// Solving for i yields |A & B| = (n + m) / ((1/J)+ 1).
		est = int(math.Floor(float64(size1+size2) / ((1.0 / js) + 1)))
	}
	return est
}

func IsEmpty(m MinHash) bool {
	var empty = true

	if m != nil {
		// Check whether each minimum is infinite.
		for _, v := range m.Signature() {
			if v < infinity {
				empty = false
				break
			}
		}
	}
	return empty
}
<|endoftext|>


<commit_before>// Package sanitize provides functions for sanitizing text.
package sanitize

import (
	"bytes"
	"html"
	"html/template"
	"io"
	"path"
	"regexp"
	"strings"

	parser "golang.org/x/net/html"
)

var (
	ignoreTags = []string{"title", "script", "style", "iframe", "frame", "frameset", "noframes", "noembed", "embed", "applet", "object", "base"}

	defaultTags = []string{"h1", "h2", "h3", "h4", "h5", "h6", "div", "span", "hr", "p", "br", "b", "i", "strong", "em", "ol", "ul", "li", "a", "img", "pre", "code", "blockquote"}

	defaultAttributes = []string{"id", "class", "src", "href", "title", "alt", "name", "rel"}
)

// HTMLAllowing sanitizes html, allowing some tags.
// Arrays of allowed tags and allowed attributes may optionally be passed as the second and third arguments.
func HTMLAllowing(s string, args ...[]string) (string, error) {

	allowedTags := defaultTags
	if len(args) > 0 {
		allowedTags = args[0]
	}
	allowedAttributes := defaultAttributes
	if len(args) > 1 {
		allowedAttributes = args[1]
	}

	// Parse the html
	tokenizer := parser.NewTokenizer(strings.NewReader(s))

	buffer := bytes.NewBufferString("")
	ignore := ""

	for {
		tokenType := tokenizer.Next()
		token := tokenizer.Token()

		switch tokenType {

		case parser.ErrorToken:
			err := tokenizer.Err()
			if err == io.EOF {
				return buffer.String(), nil
			}
			return "", err

		case parser.StartTagToken:

			if len(ignore) == 0 && includes(allowedTags, token.Data) {
				token.Attr = cleanAttributes(token.Attr, allowedAttributes)
				buffer.WriteString(token.String())
			} else if includes(ignoreTags, token.Data) {
				ignore = token.Data
			}

		case parser.SelfClosingTagToken:

			if len(ignore) == 0 && includes(allowedTags, token.Data) {
				token.Attr = cleanAttributes(token.Attr, allowedAttributes)
				buffer.WriteString(token.String())
			} else if token.Data == ignore {
				ignore = ""
			}

		case parser.EndTagToken:
			if len(ignore) == 0 && includes(allowedTags, token.Data) {
				token.Attr = []parser.Attribute{}
				buffer.WriteString(token.String())
			} else if token.Data == ignore {
				ignore = ""
			}

		case parser.TextToken:
			// We allow text content through, unless ignoring this entire tag and its contents (including other tags)
			if ignore == "" {
				buffer.WriteString(token.String())
			}
		case parser.CommentToken:
			// We ignore comments by default
		case parser.DoctypeToken:
			// We ignore doctypes by default - html5 does not require them and this is intended for sanitizing snippets of text
		default:
			// We ignore unknown token types by default

		}

	}

}

// HTML strips html tags, replace common entities, and escapes <>&;'" in the result.
// Note the returned text may contain entities as it is escaped by HTMLEscapeString, and most entities are not translated.
func HTML(s string) (output string) {

	// Shortcut strings with no tags in them
	if !strings.ContainsAny(s, "<>") {
		output = s
	} else {

		// First remove line breaks etc as these have no meaning outside html tags (except pre)
		// this means pre sections will lose formatting... but will result in less unintentional paras.
		s = strings.Replace(s, "\n", "", -1)

		// Then replace line breaks with newlines, to preserve that formatting
		s = strings.Replace(s, "
", "\n", -1)
		s = strings.Replace(s, "
", "\n", -1)
		s = strings.Replace(s, "
", "\n", -1)
		s = strings.Replace(s, "
", "\n", -1)
		s = strings.Replace(s, "
", "\n", -1)

		// Walk through the string removing all tags
		b := bytes.NewBufferString("")
		inTag := false
		for _, r := range s {
			switch r {
			case '<':
				inTag = true
			case '>':
				inTag = false
			default:
				if !inTag {
					b.WriteRune(r)
				}
			}
		}
		output = b.String()
	}

	// Remove a few common harmless entities, to arrive at something more like plain text
	output = strings.Replace(output, "‘", "'", -1)
	output = strings.Replace(output, "’", "'", -1)
	output = strings.Replace(output, "“", "\"", -1)
	output = strings.Replace(output, "”", "\"", -1)
	output = strings.Replace(output, " ", " ", -1)
	output = strings.Replace(output, """, "\"", -1)
	output = strings.Replace(output, "'", "'", -1)

	// Translate some entities into their plain text equivalent (for example accents, if encoded as entities)
	output = html.UnescapeString(output)

	// In case we have missed any tags above, escape the text - removes <, >, &, ' and ".
	output = template.HTMLEscapeString(output)

	// After processing, remove some harmless entities &, ' and " which are encoded by HTMLEscapeString
	output = strings.Replace(output, """, "\"", -1)
	output = strings.Replace(output, "'", "'", -1)
	output = strings.Replace(output, "& ", "& ", -1)     // NB space after
	output = strings.Replace(output, "&amp; ", "& ", -1) // NB space after

	return output
}

// We are very restrictive as this is intended for ascii url slugs
var illegalPath = regexp.MustCompile(`[^[:alnum:]\~\-\./]`)

// Path makes a string safe to use as an url path.
func Path(s string) string {
	// Start with lowercase string
	filePath := strings.ToLower(s)
	filePath = strings.Replace(filePath, "..", "", -1)
	filePath = path.Clean(filePath)

	// Remove illegal characters for paths, flattening accents and replacing some common separators with -
	filePath = cleanString(filePath, illegalPath)

	// NB this may be of length 0, caller must check
	return filePath
}

// Remove all other unrecognised characters apart from
var illegalName = regexp.MustCompile(`[^[:alnum:]-.]`)

// Name makes a string safe to use in a file name by first finding the path basename, then replacing non-ascii characters.
func Name(s string) string {
	// Start with lowercase string
	fileName := strings.ToLower(s)
	fileName = path.Clean(path.Base(fileName))

	// Remove illegal characters for names, replacing some common separators with -
	fileName = cleanString(fileName, illegalName)

	// NB this may be of length 0, caller must check
	return fileName
}

// Replace these separators with -
var baseNameSeparators = regexp.MustCompile(`[./]`)

// BaseName makes a string safe to use in a file name, producing a sanitized basename replacing . or / with -.
// No attempt is made to normalise a path or normalise case.
func BaseName(s string) string {

	// Replace certain joining characters with a dash
	baseName := baseNameSeparators.ReplaceAllString(s, "-")

	// Remove illegal characters for names, replacing some common separators with -
	baseName = cleanString(baseName, illegalName)

	// NB this may be of length 0, caller must check
	return baseName
}

// A very limited list of transliterations to catch common european names translated to urls.
// This set could be expanded with at least caps and many more characters.
var transliterations = map[rune]string{
	'À': "A",
	'Á': "A",
	'Â': "A",
	'Ã': "A",
	'Ä': "A",
	'Å': "AA",
	'Æ': "AE",
	'Ç': "C",
	'È': "E",
	'É': "E",
	'Ê': "E",
	'Ë': "E",
	'Ì': "I",
	'Í': "I",
	'Î': "I",
	'Ï': "I",
	'Ð': "D",
	'Ł': "L",
	'Ñ': "N",
	'Ò': "O",
	'Ó': "O",
	'Ô': "O",
	'Õ': "O",
	'Ö': "O",
	'Ø': "OE",
	'Ù': "U",
	'Ú': "U",
	'Ü': "U",
	'Û': "U",
	'Ý': "Y",
	'Þ': "Th",
	'ß': "ss",
	'à': "a",
	'á': "a",
	'â': "a",
	'ã': "a",
	'ä': "a",
	'å': "aa",
	'æ': "ae",
	'ç': "c",
	'è': "e",
	'é': "e",
	'ê': "e",
	'ë': "e",
	'ì': "i",
	'í': "i",
	'î': "i",
	'ï': "i",
	'ð': "d",
	'ł': "l",
	'ñ': "n",
	'ń': "n",
	'ò': "o",
	'ó': "o",
	'ô': "o",
	'õ': "o",
	'ō': "o",
	'ö': "o",
	'ø': "oe",
	'ś': "s",
	'ù': "u",
	'ú': "u",
	'û': "u",
	'ū': "u",
	'ü': "u",
	'ý': "y",
	'þ': "th",
	'ÿ': "y",
	'ż': "z",
	'Œ': "OE",
	'œ': "oe",
}

// Accents replaces a set of accented characters with ascii equivalents.
func Accents(s string) string {
	// Replace some common accent characters
	b := bytes.NewBufferString("")
	for _, c := range s {
		// Check transliterations first
		if val, ok := transliterations[c]; ok {
			b.WriteString(val)
		} else {
			b.WriteRune(c)
		}
	}
	return b.String()
}

var (
	// If the attribute contains data: or javascript: anywhere, ignore it
	// we don't allow this in attributes as it is so frequently used for xss
	// NB we allow spaces in the value, and lowercase.
	illegalAttr = regexp.MustCompile(`(d\s*a\s*t\s*a|j\s*a\s*v\s*a\s*s\s*c\s*r\s*i\s*p\s*t\s*)\s*:`)

	// We are far more restrictive with href attributes.
	legalHrefAttr = regexp.MustCompile(`\A[/#][^/\\]?|mailto:|http://|https://`)
)

// cleanAttributes returns an array of attributes after removing malicious ones.
func cleanAttributes(a []parser.Attribute, allowed []string) []parser.Attribute {
	if len(a) == 0 {
		return a
	}

	var cleaned []parser.Attribute
	for _, attr := range a {
		if includes(allowed, attr.Key) {

			val := strings.ToLower(attr.Val)

			// Check for illegal attribute values
			if illegalAttr.FindString(val) != "" {
				attr.Val = ""
			}

			// Check for legal href values - / mailto:// http:// or https://
			if attr.Key == "href" {
				if legalHrefAttr.FindString(val) == "" {
					attr.Val = ""
				}
			}

			// If we still have an attribute, append it to the array
			if attr.Val != "" {
				cleaned = append(cleaned, attr)
			}
		}
	}
	return cleaned
}

// A list of characters we consider separators in normal strings and replace with our canonical separator - rather than removing.
var (
	separators = regexp.MustCompile(`[ &_=+:]`)

	dashes = regexp.MustCompile(`[\-]+`)
)

// cleanString replaces separators with - and removes characters listed in the regexp provided from string.
// Accents, spaces, and all characters not in A-Za-z0-9 are replaced.
func cleanString(s string, r *regexp.Regexp) string {

	// Remove any trailing space to avoid ending on -
	s = strings.Trim(s, " ")

	// Flatten accents first so that if we remove non-ascii we still get a legible name
	s = Accents(s)

	// Replace certain joining characters with a dash
	s = separators.ReplaceAllString(s, "-")

	// Remove all other unrecognised characters - NB we do allow any printable characters
	s = r.ReplaceAllString(s, "")

	// Remove any multiple dashes caused by replacements above
	s = dashes.ReplaceAllString(s, "-")

	return s
}

// includes checks for inclusion of a string in a []string.
func includes(a []string, s string) bool {
	for _, as := range a {
		if as == s {
			return true
		}
	}
	return false
}
<commit_msg>german transliterations corrected<commit_after>// Package sanitize provides functions for sanitizing text.
package sanitize

import (
	"bytes"
	"html"
	"html/template"
	"io"
	"path"
	"regexp"
	"strings"

	parser "golang.org/x/net/html"
)

var (
	ignoreTags = []string{"title", "script", "style", "iframe", "frame", "frameset", "noframes", "noembed", "embed", "applet", "object", "base"}

	defaultTags = []string{"h1", "h2", "h3", "h4", "h5", "h6", "div", "span", "hr", "p", "br", "b", "i", "strong", "em", "ol", "ul", "li", "a", "img", "pre", "code", "blockquote"}

	defaultAttributes = []string{"id", "class", "src", "href", "title", "alt", "name", "rel"}
)

// HTMLAllowing sanitizes html, allowing some tags.
// Arrays of allowed tags and allowed attributes may optionally be passed as the second and third arguments.
func HTMLAllowing(s string, args ...[]string) (string, error) {

	allowedTags := defaultTags
	if len(args) > 0 {
		allowedTags = args[0]
	}
	allowedAttributes := defaultAttributes
	if len(args) > 1 {
		allowedAttributes = args[1]
	}

	// Parse the html
	tokenizer := parser.NewTokenizer(strings.NewReader(s))

	buffer := bytes.NewBufferString("")
	ignore := ""

	for {
		tokenType := tokenizer.Next()
		token := tokenizer.Token()

		switch tokenType {

		case parser.ErrorToken:
			err := tokenizer.Err()
			if err == io.EOF {
				return buffer.String(), nil
			}
			return "", err

		case parser.StartTagToken:

			if len(ignore) == 0 && includes(allowedTags, token.Data) {
				token.Attr = cleanAttributes(token.Attr, allowedAttributes)
				buffer.WriteString(token.String())
			} else if includes(ignoreTags, token.Data) {
				ignore = token.Data
			}

		case parser.SelfClosingTagToken:

			if len(ignore) == 0 && includes(allowedTags, token.Data) {
				token.Attr = cleanAttributes(token.Attr, allowedAttributes)
				buffer.WriteString(token.String())
			} else if token.Data == ignore {
				ignore = ""
			}

		case parser.EndTagToken:
			if len(ignore) == 0 && includes(allowedTags, token.Data) {
				token.Attr = []parser.Attribute{}
				buffer.WriteString(token.String())
			} else if token.Data == ignore {
				ignore = ""
			}

		case parser.TextToken:
			// We allow text content through, unless ignoring this entire tag and its contents (including other tags)
			if ignore == "" {
				buffer.WriteString(token.String())
			}
		case parser.CommentToken:
			// We ignore comments by default
		case parser.DoctypeToken:
			// We ignore doctypes by default - html5 does not require them and this is intended for sanitizing snippets of text
		default:
			// We ignore unknown token types by default

		}

	}

}

// HTML strips html tags, replace common entities, and escapes <>&;'" in the result.
// Note the returned text may contain entities as it is escaped by HTMLEscapeString, and most entities are not translated.
func HTML(s string) (output string) {

	// Shortcut strings with no tags in them
	if !strings.ContainsAny(s, "<>") {
		output = s
	} else {

		// First remove line breaks etc as these have no meaning outside html tags (except pre)
		// this means pre sections will lose formatting... but will result in less unintentional paras.
		s = strings.Replace(s, "\n", "", -1)

		// Then replace line breaks with newlines, to preserve that formatting
		s = strings.Replace(s, "
", "\n", -1)
		s = strings.Replace(s, "
", "\n", -1)
		s = strings.Replace(s, "
", "\n", -1)
		s = strings.Replace(s, "
", "\n", -1)
		s = strings.Replace(s, "
", "\n", -1)

		// Walk through the string removing all tags
		b := bytes.NewBufferString("")
		inTag := false
		for _, r := range s {
			switch r {
			case '<':
				inTag = true
			case '>':
				inTag = false
			default:
				if !inTag {
					b.WriteRune(r)
				}
			}
		}
		output = b.String()
	}

	// Remove a few common harmless entities, to arrive at something more like plain text
	output = strings.Replace(output, "‘", "'", -1)
	output = strings.Replace(output, "’", "'", -1)
	output = strings.Replace(output, "“", "\"", -1)
	output = strings.Replace(output, "”", "\"", -1)
	output = strings.Replace(output, " ", " ", -1)
	output = strings.Replace(output, """, "\"", -1)
	output = strings.Replace(output, "'", "'", -1)

	// Translate some entities into their plain text equivalent (for example accents, if encoded as entities)
	output = html.UnescapeString(output)

	// In case we have missed any tags above, escape the text - removes <, >, &, ' and ".
	output = template.HTMLEscapeString(output)

	// After processing, remove some harmless entities &, ' and " which are encoded by HTMLEscapeString
	output = strings.Replace(output, """, "\"", -1)
	output = strings.Replace(output, "'", "'", -1)
	output = strings.Replace(output, "& ", "& ", -1)     // NB space after
	output = strings.Replace(output, "&amp; ", "& ", -1) // NB space after

	return output
}

// We are very restrictive as this is intended for ascii url slugs
var illegalPath = regexp.MustCompile(`[^[:alnum:]\~\-\./]`)

// Path makes a string safe to use as an url path.
func Path(s string) string {
	// Start with lowercase string
	filePath := strings.ToLower(s)
	filePath = strings.Replace(filePath, "..", "", -1)
	filePath = path.Clean(filePath)

	// Remove illegal characters for paths, flattening accents and replacing some common separators with -
	filePath = cleanString(filePath, illegalPath)

	// NB this may be of length 0, caller must check
	return filePath
}

// Remove all other unrecognised characters apart from
var illegalName = regexp.MustCompile(`[^[:alnum:]-.]`)

// Name makes a string safe to use in a file name by first finding the path basename, then replacing non-ascii characters.
func Name(s string) string {
	// Start with lowercase string
	fileName := strings.ToLower(s)
	fileName = path.Clean(path.Base(fileName))

	// Remove illegal characters for names, replacing some common separators with -
	fileName = cleanString(fileName, illegalName)

	// NB this may be of length 0, caller must check
	return fileName
}

// Replace these separators with -
var baseNameSeparators = regexp.MustCompile(`[./]`)

// BaseName makes a string safe to use in a file name, producing a sanitized basename replacing . or / with -.
// No attempt is made to normalise a path or normalise case.
func BaseName(s string) string {

	// Replace certain joining characters with a dash
	baseName := baseNameSeparators.ReplaceAllString(s, "-")

	// Remove illegal characters for names, replacing some common separators with -
	baseName = cleanString(baseName, illegalName)

	// NB this may be of length 0, caller must check
	return baseName
}

// A very limited list of transliterations to catch common european names translated to urls.
// This set could be expanded with at least caps and many more characters.
var transliterations = map[rune]string{
	'À': "A",
	'Á': "A",
	'Â': "A",
	'Ã': "A",
	'Ä': "A",
	'Å': "AA",
	'Æ': "AE",
	'Ç': "C",
	'È': "E",
	'É': "E",
	'Ê': "E",
	'Ë': "E",
	'Ì': "I",
	'Í': "I",
	'Î': "I",
	'Ï': "I",
	'Ð': "D",
	'Ł': "L",
	'Ñ': "N",
	'Ò': "O",
	'Ó': "O",
	'Ô': "O",
	'Õ': "O",
	'Ö': "Oe",
	'Ø': "OE",
	'Ù': "U",
	'Ú': "U",
	'Ü': "Ue",
	'Û': "U",
	'Ý': "Y",
	'Þ': "Th",
	'ß': "ss",
	'à': "a",
	'á': "a",
	'â': "a",
	'ã': "a",
	'ä': "ae",
	'å': "aa",
	'æ': "ae",
	'ç': "c",
	'è': "e",
	'é': "e",
	'ê': "e",
	'ë': "e",
	'ì': "i",
	'í': "i",
	'î': "i",
	'ï': "i",
	'ð': "d",
	'ł': "l",
	'ñ': "n",
	'ń': "n",
	'ò': "o",
	'ó': "o",
	'ô': "o",
	'õ': "o",
	'ō': "o",
	'ö': "oe",
	'ø': "oe",
	'ś': "s",
	'ù': "u",
	'ú': "u",
	'û': "u",
	'ū': "u",
	'ü': "ue",
	'ý': "y",
	'þ': "th",
	'ÿ': "y",
	'ż': "z",
	'Œ': "OE",
	'œ': "oe",
}

// Accents replaces a set of accented characters with ascii equivalents.
func Accents(s string) string {
	// Replace some common accent characters
	b := bytes.NewBufferString("")
	for _, c := range s {
		// Check transliterations first
		if val, ok := transliterations[c]; ok {
			b.WriteString(val)
		} else {
			b.WriteRune(c)
		}
	}
	return b.String()
}

var (
	// If the attribute contains data: or javascript: anywhere, ignore it
	// we don't allow this in attributes as it is so frequently used for xss
	// NB we allow spaces in the value, and lowercase.
	illegalAttr = regexp.MustCompile(`(d\s*a\s*t\s*a|j\s*a\s*v\s*a\s*s\s*c\s*r\s*i\s*p\s*t\s*)\s*:`)

	// We are far more restrictive with href attributes.
	legalHrefAttr = regexp.MustCompile(`\A[/#][^/\\]?|mailto:|http://|https://`)
)

// cleanAttributes returns an array of attributes after removing malicious ones.
func cleanAttributes(a []parser.Attribute, allowed []string) []parser.Attribute {
	if len(a) == 0 {
		return a
	}

	var cleaned []parser.Attribute
	for _, attr := range a {
		if includes(allowed, attr.Key) {

			val := strings.ToLower(attr.Val)

			// Check for illegal attribute values
			if illegalAttr.FindString(val) != "" {
				attr.Val = ""
			}

			// Check for legal href values - / mailto:// http:// or https://
			if attr.Key == "href" {
				if legalHrefAttr.FindString(val) == "" {
					attr.Val = ""
				}
			}

			// If we still have an attribute, append it to the array
			if attr.Val != "" {
				cleaned = append(cleaned, attr)
			}
		}
	}
	return cleaned
}

// A list of characters we consider separators in normal strings and replace with our canonical separator - rather than removing.
var (
	separators = regexp.MustCompile(`[ &_=+:]`)

	dashes = regexp.MustCompile(`[\-]+`)
)

// cleanString replaces separators with - and removes characters listed in the regexp provided from string.
// Accents, spaces, and all characters not in A-Za-z0-9 are replaced.
func cleanString(s string, r *regexp.Regexp) string {

	// Remove any trailing space to avoid ending on -
	s = strings.Trim(s, " ")

	// Flatten accents first so that if we remove non-ascii we still get a legible name
	s = Accents(s)

	// Replace certain joining characters with a dash
	s = separators.ReplaceAllString(s, "-")

	// Remove all other unrecognised characters - NB we do allow any printable characters
	s = r.ReplaceAllString(s, "")

	// Remove any multiple dashes caused by replacements above
	s = dashes.ReplaceAllString(s, "-")

	return s
}

// includes checks for inclusion of a string in a []string.
func includes(a []string, s string) bool {
	for _, as := range a {
		if as == s {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package mithril

import "regexp"

// M returns VirtualElement
func M(selector string, opts ...interface{}) *VirtualElement {
	// declare query string
	query := "div"
	// replace query as div if selector does not existed
	if selector != "" {
		query = selector
	}
	// create virtual element
	element := &VirtualElement{"div", NewAttributes(), nil}
	// map options
	for i, opt := range opts {
		switch obj := opt.(type) {
		case *Attributes:
			if i == 0 {
				element.Attrs = obj
			}
		case string, bool, int, *VirtualElement:
			element.Children = obj
		}
	}
	// match all results
	matches := regexp.MustCompile(`(?:(^|#|\.)([^#\.\[\]]+))|(\[.+?\])`)
	for _, res := range matches.FindAllStringSubmatch(query, -1) {
		if res[1] == "" && len(res[2]) > 0 {
			element.Tag = res[2]
		} else if res[1] == "#" {
			element.Attrs.ID = res[2]
		} else if res[1] == "." {
			element.Attrs.Class = append(element.Attrs.Class, res[2])
		} else if string(res[3][0]) == "[" {
			matches := regexp.MustCompile(`\[(.+?)(?:=("|'|)(.*?)("|'|))?\]`)
			for _, pair := range matches.FindAllStringSubmatch(res[3], -1) {
				if len(pair[1]) > 0 {
					element.Attrs.Data[pair[1]] = pair[3]
				}
			}
		}
	}
	// return virtual element
	return element
}
<commit_msg>better documentations<commit_after>package mithril

import "regexp"

// M returns VirtualElement
func M(selector string, opts ...interface{}) *VirtualElement {
	// new VirtualElement channel
	c := make(chan *VirtualElement)
	// create element in go routine
	go func() {
		// declare query string
		query := "div"
		// replace query as div if selector does not existed
		if selector != "" {
			query = selector
		}
		// create virtual element
		element := &VirtualElement{"div", NewAttributes(), nil}
		// map options
		for i, opt := range opts {
			switch obj := opt.(type) {
			case *Attributes:
				if i == 0 {
					element.Attrs = obj
				}
			case string, bool, int, *VirtualElement:
				element.Children = obj
			}
		}
		// match all results
		matches := regexp.MustCompile(`(?:(^|#|\.)([^#\.\[\]]+))|(\[.+?\])`)
		for _, res := range matches.FindAllStringSubmatch(query, -1) {
			if res[1] == "" && len(res[2]) > 0 {
				element.Tag = res[2]
			} else if res[1] == "#" {
				element.Attrs.ID = res[2]
			} else if res[1] == "." {
				element.Attrs.Class = append(element.Attrs.Class, res[2])
			} else if string(res[3][0]) == "[" {
				matches := regexp.MustCompile(`\[(.+?)(?:=("|'|)(.*?)("|'|))?\]`)
				for _, pair := range matches.FindAllStringSubmatch(res[3], -1) {
					if len(pair[1]) > 0 {
						element.Attrs.Data[pair[1]] = pair[3]
					}
				}
			}
		}
		c <- element
	}()
	// return virtual element
	return <-c
}
<|endoftext|>


<commit_before>package parser

import (
	"fmt"
	"strings"

	"github.com/Zac-Garby/pluto/token"
	"github.com/fatih/color"
)

type Error struct {
	Message    string
	Start, End token.Position
}

func (p *Parser) err(msg string, start, end token.Position) {
	err := Error{
		Message: msg,
		Start:   start,
		End:     end,
	}

	p.Errors = append(p.Errors, err)
}

func (p *Parser) defaultErr(msg string) {
	err := Error{
		Message: msg,
		Start:   p.cur.Start,
		End:     p.cur.End,
	}

	p.Errors = append(p.Errors, err)
}

func (p *Parser) peekErr(ts ...token.Type) {
	if len(ts) > 1 {
		msg := "expected either "

		for i, t := range ts {
			msg += string(t)

			if i+1 < len(ts) {
				msg += ", "
			} else if i < len(ts) {
				msg += ", or "
			}
		}

		msg += ", but got " + string(p.peek.Type)

		p.err(msg, p.peek.Start, p.peek.End)
	} else if len(ts) == 1 {
		msg := fmt.Sprintf("expected %s, but got %s", ts[0], p.peek.Type)
		p.err(msg, p.peek.Start, p.peek.End)
	}
}

func (p *Parser) curErr(ts ...token.Type) {
	if len(ts) > 1 {
		msg := "expected either "

		for i, t := range ts {
			msg += string(t)

			if i+1 < len(ts) {
				msg += ", "
			} else if i < len(ts) {
				msg += ", or "
			}
		}

		msg += ", but got " + string(p.cur.Type)

		p.err(msg, p.cur.Start, p.cur.End)
	} else if len(ts) == 1 {
		msg := fmt.Sprintf("expected %s, but got %s", ts[0], p.cur.Type)
		p.err(msg, p.cur.Start, p.cur.End)
	}
}

func (p *Parser) unexpectedTokenErr(t token.Type) {
	msg := fmt.Sprintf("unexpected token: %s", t)
	p.defaultErr(msg)
}

func (p *Parser) printError(index int) {
	err := p.Errors[index]

	fmt.Printf("%s → %s\t%s\n", err.Start.String(), err.End.String(), err.Message)
}

func (p *Parser) printErrorVerbose(index int) {
	err := p.Errors[index]
	lines := strings.Split(p.text, "\n")

	red := color.New(color.FgRed)
	grey := color.New(color.FgHiWhite)

	grey.Printf("    %d| ", err.Start.Line)
	fmt.Printf("%s\n", lines[err.Start.Line-1])
	red.Printf(
		"    %s %s%s\n",
		strings.Repeat(" ", len(fmt.Sprintf("%d", err.Start.Line))),
		strings.Repeat(" ", err.Start.Column),
		strings.Repeat("^", err.End.Column-err.Start.Column+1),
	)

	red.Printf("%s → %s", err.Start.String(), err.End.String())
	fmt.Printf("\t%s\n\n", err.Message)
}

func (p *Parser) PrintErrors() {
	for i := range p.Errors {
		if i == 0 {
			p.printErrorVerbose(i)
		} else {
			p.printError(i)
		}
	}
}
<commit_msg>Make red bold<commit_after>package parser

import (
	"fmt"
	"strings"

	"github.com/Zac-Garby/pluto/token"
	"github.com/fatih/color"
)

type Error struct {
	Message    string
	Start, End token.Position
}

func (p *Parser) err(msg string, start, end token.Position) {
	err := Error{
		Message: msg,
		Start:   start,
		End:     end,
	}

	p.Errors = append(p.Errors, err)
}

func (p *Parser) defaultErr(msg string) {
	err := Error{
		Message: msg,
		Start:   p.cur.Start,
		End:     p.cur.End,
	}

	p.Errors = append(p.Errors, err)
}

func (p *Parser) peekErr(ts ...token.Type) {
	if len(ts) > 1 {
		msg := "expected either "

		for i, t := range ts {
			msg += string(t)

			if i+1 < len(ts) {
				msg += ", "
			} else if i < len(ts) {
				msg += ", or "
			}
		}

		msg += ", but got " + string(p.peek.Type)

		p.err(msg, p.peek.Start, p.peek.End)
	} else if len(ts) == 1 {
		msg := fmt.Sprintf("expected %s, but got %s", ts[0], p.peek.Type)
		p.err(msg, p.peek.Start, p.peek.End)
	}
}

func (p *Parser) curErr(ts ...token.Type) {
	if len(ts) > 1 {
		msg := "expected either "

		for i, t := range ts {
			msg += string(t)

			if i+1 < len(ts) {
				msg += ", "
			} else if i < len(ts) {
				msg += ", or "
			}
		}

		msg += ", but got " + string(p.cur.Type)

		p.err(msg, p.cur.Start, p.cur.End)
	} else if len(ts) == 1 {
		msg := fmt.Sprintf("expected %s, but got %s", ts[0], p.cur.Type)
		p.err(msg, p.cur.Start, p.cur.End)
	}
}

func (p *Parser) unexpectedTokenErr(t token.Type) {
	msg := fmt.Sprintf("unexpected token: %s", t)
	p.defaultErr(msg)
}

func (p *Parser) printError(index int) {
	err := p.Errors[index]

	fmt.Printf("%s → %s\t%s\n", err.Start.String(), err.End.String(), err.Message)
}

func (p *Parser) printErrorVerbose(index int) {
	err := p.Errors[index]
	lines := strings.Split(p.text, "\n")

	red := color.New(color.FgRed).Add(color.Bold)
	grey := color.New(color.FgHiWhite)

	grey.Printf("    %d| ", err.Start.Line)
	fmt.Printf("%s\n", lines[err.Start.Line-1])
	red.Printf(
		"    %s %s%s\n",
		strings.Repeat(" ", len(fmt.Sprintf("%d", err.Start.Line))),
		strings.Repeat(" ", err.Start.Column),
		strings.Repeat("^", err.End.Column-err.Start.Column+1),
	)

	red.Printf("%s → %s", err.Start.String(), err.End.String())
	fmt.Printf("\t%s\n\n", err.Message)
}

func (p *Parser) PrintErrors() {
	for i := range p.Errors {
		if i == 0 {
			p.printErrorVerbose(i)
		} else {
			p.printError(i)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package importer defines the Importer, which loads, parses and
// type-checks packages of Go code plus their transitive closure, and
// retains both the ASTs and the derived facts.
//
// TODO(adonovan): document and test this package better, with examples.
// Currently it's covered by the ssa/ tests.
//
package importer

import (
	"fmt"
	"go/ast"
	"go/build"
	"go/token"
	"os"
	"strconv"
	"sync"

	"code.google.com/p/go.tools/go/exact"
	"code.google.com/p/go.tools/go/types"
)

// An Importer's methods are not thread-safe unless specified.
type Importer struct {
	config   *Config                 // the client configuration
	Fset     *token.FileSet          // position info for all files seen
	Packages map[string]*PackageInfo // successfully imported packages keyed by import path
	errors   map[string]error        // cache of errors by import path

	mu       sync.Mutex          // guards 'packages' map
	packages map[string]*pkgInfo // all attempted imported packages, keyed by import path
}

// pkgInfo holds internal per-package information.
type pkgInfo struct {
	info  *PackageInfo  // success info
	err   error         // failure info
	ready chan struct{} // channel close is notification of ready state
}

// Config specifies the configuration for the importer.
//
type Config struct {
	// TypeChecker contains options relating to the type checker.
	// The Importer will override any user-supplied values for its
	// Error and Import fields; other fields will be passed
	// through to the type checker.
	TypeChecker types.Config

	// If Build is non-nil, it is used to satisfy imports.
	//
	// If it is nil, binary object files produced by the gc
	// compiler will be loaded instead of source code for all
	// imported packages.  Such files supply only the types of
	// package-level declarations and values of constants, but no
	// code, so this mode will not yield a whole program.  It is
	// intended for analyses that perform intraprocedural analysis
	// of a single package.
	Build *build.Context
}

// New returns a new, empty Importer using configuration options
// specified by config.
//
func New(config *Config) *Importer {
	imp := &Importer{
		config:   config,
		Fset:     token.NewFileSet(),
		packages: make(map[string]*pkgInfo),
		Packages: make(map[string]*PackageInfo),
		errors:   make(map[string]error),
	}
	imp.config.TypeChecker.Error = func(e error) { fmt.Fprintln(os.Stderr, e) }
	imp.config.TypeChecker.Import = imp.doImport
	return imp
}

// doImport loads the typechecker package identified by path
// Implements the types.Importer prototype.
//
func (imp *Importer) doImport(imports map[string]*types.Package, path string) (pkg *types.Package, err error) {
	// Package unsafe is handled specially, and has no PackageInfo.
	if path == "unsafe" {
		return types.Unsafe, nil
	}

	// Load the source/binary for 'path', type-check it, construct
	// a PackageInfo and update our map (imp.Packages) and the
	// type-checker's map (imports).
	// TODO(adonovan): make it the type-checker's job to
	// consult/update the 'imports' map.  Then we won't need it.
	var info *PackageInfo
	if imp.config.Build != nil {
		info, err = imp.LoadPackage(path)
	} else {
		if info, ok := imp.Packages[path]; ok {
			imports[path] = info.Pkg
			pkg = info.Pkg
			return // positive cache hit
		}

		if err = imp.errors[path]; err != nil {
			return // negative cache hit
		}

		if pkg, err = types.GcImport(imports, path); err == nil {
			info = &PackageInfo{Pkg: pkg}
			imp.Packages[path] = info
		}
	}

	if err == nil {
		// Cache success.
		pkg = info.Pkg
		imports[path] = pkg
		return pkg, nil
	}

	// Cache failure
	imp.errors[path] = err
	return nil, err
}

// imports returns the set of paths imported by the specified files.
func imports(p string, files []*ast.File) map[string]bool {
	imports := make(map[string]bool)
outer:
	for _, file := range files {
		for _, decl := range file.Decls {
			if decl, ok := decl.(*ast.GenDecl); ok {
				if decl.Tok != token.IMPORT {
					break outer // stop at the first non-import
				}
				for _, spec := range decl.Specs {
					spec := spec.(*ast.ImportSpec)
					if path, _ := strconv.Unquote(spec.Path.Value); path != "C" {
						imports[path] = true
					}
				}
			} else {
				break outer // stop at the first non-import
			}
		}
	}
	return imports
}

// LoadPackage loads the package of the specified import path,
// performs type-checking, and returns the corresponding
// PackageInfo.
//
// Precondition: Importer.config.Build != nil.
// Idempotent and thread-safe.
//
// TODO(adonovan): fix: violated in call from CreatePackageFromArgs!
// TODO(adonovan): rethink this API.
//
func (imp *Importer) LoadPackage(importPath string) (*PackageInfo, error) {
	if imp.config.Build == nil {
		panic("Importer.LoadPackage without a go/build.Config")
	}

	imp.mu.Lock()
	pi, ok := imp.packages[importPath]
	if !ok {
		// First request for this pkgInfo:
		// create a new one in !ready state.
		pi = &pkgInfo{ready: make(chan struct{})}
		imp.packages[importPath] = pi
		imp.mu.Unlock()

		// Load/parse the package.
		if files, err := loadPackage(imp.config.Build, imp.Fset, importPath); err == nil {
			// Kick off asynchronous I/O and parsing for the imports.
			for path := range imports(importPath, files) {
				go func() { imp.LoadPackage(path) }()
			}

			// Type-check the package.
			pi.info, pi.err = imp.CreateSourcePackage(importPath, files)
		} else {
			pi.err = err
		}

		close(pi.ready) // enter ready state and wake up waiters
	} else {
		imp.mu.Unlock()

		<-pi.ready // wait for ready condition.
	}

	// Invariant: pi is ready.

	if pi.err != nil {
		return nil, pi.err
	}
	return pi.info, nil
}

// CreateSourcePackage invokes the type-checker on files and returns a
// PackageInfo containing the resulting type-checker package, the
// ASTs, and other type information.
//
// The order of files determines the package initialization order.
//
// importPath is the full name under which this package is known, such
// as appears in an import declaration. e.g. "sync/atomic".
//
// Postcondition: for result (info, err), info.Err == err.
//
func (imp *Importer) CreateSourcePackage(importPath string, files []*ast.File) (*PackageInfo, error) {
	info := &PackageInfo{
		Files: files,
		Info: types.Info{
			Types:      make(map[ast.Expr]types.Type),
			Values:     make(map[ast.Expr]exact.Value),
			Objects:    make(map[*ast.Ident]types.Object),
			Implicits:  make(map[ast.Node]types.Object),
			Scopes:     make(map[ast.Node]*types.Scope),
			Selections: make(map[*ast.SelectorExpr]*types.Selection),
		},
	}
	info.Pkg, info.Err = imp.config.TypeChecker.Check(importPath, imp.Fset, files, &info.Info)
	imp.Packages[importPath] = info
	return info, info.Err
}
<commit_msg>go.tools/importer: fix classic closing-over-induction-variable gotcha.<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package importer defines the Importer, which loads, parses and
// type-checks packages of Go code plus their transitive closure, and
// retains both the ASTs and the derived facts.
//
// TODO(adonovan): document and test this package better, with examples.
// Currently it's covered by the ssa/ tests.
//
package importer

import (
	"fmt"
	"go/ast"
	"go/build"
	"go/token"
	"os"
	"strconv"
	"sync"

	"code.google.com/p/go.tools/go/exact"
	"code.google.com/p/go.tools/go/types"
)

// An Importer's methods are not thread-safe unless specified.
type Importer struct {
	config   *Config                 // the client configuration
	Fset     *token.FileSet          // position info for all files seen
	Packages map[string]*PackageInfo // successfully imported packages keyed by import path
	errors   map[string]error        // cache of errors by import path

	mu       sync.Mutex          // guards 'packages' map
	packages map[string]*pkgInfo // all attempted imported packages, keyed by import path
}

// pkgInfo holds internal per-package information.
type pkgInfo struct {
	info  *PackageInfo  // success info
	err   error         // failure info
	ready chan struct{} // channel close is notification of ready state
}

// Config specifies the configuration for the importer.
//
type Config struct {
	// TypeChecker contains options relating to the type checker.
	// The Importer will override any user-supplied values for its
	// Error and Import fields; other fields will be passed
	// through to the type checker.
	TypeChecker types.Config

	// If Build is non-nil, it is used to satisfy imports.
	//
	// If it is nil, binary object files produced by the gc
	// compiler will be loaded instead of source code for all
	// imported packages.  Such files supply only the types of
	// package-level declarations and values of constants, but no
	// code, so this mode will not yield a whole program.  It is
	// intended for analyses that perform intraprocedural analysis
	// of a single package.
	Build *build.Context
}

// New returns a new, empty Importer using configuration options
// specified by config.
//
func New(config *Config) *Importer {
	imp := &Importer{
		config:   config,
		Fset:     token.NewFileSet(),
		packages: make(map[string]*pkgInfo),
		Packages: make(map[string]*PackageInfo),
		errors:   make(map[string]error),
	}
	imp.config.TypeChecker.Error = func(e error) { fmt.Fprintln(os.Stderr, e) }
	imp.config.TypeChecker.Import = imp.doImport
	return imp
}

// doImport loads the typechecker package identified by path
// Implements the types.Importer prototype.
//
func (imp *Importer) doImport(imports map[string]*types.Package, path string) (pkg *types.Package, err error) {
	// Package unsafe is handled specially, and has no PackageInfo.
	if path == "unsafe" {
		return types.Unsafe, nil
	}

	// Load the source/binary for 'path', type-check it, construct
	// a PackageInfo and update our map (imp.Packages) and the
	// type-checker's map (imports).
	// TODO(adonovan): make it the type-checker's job to
	// consult/update the 'imports' map.  Then we won't need it.
	var info *PackageInfo
	if imp.config.Build != nil {
		info, err = imp.LoadPackage(path)
	} else {
		if info, ok := imp.Packages[path]; ok {
			imports[path] = info.Pkg
			pkg = info.Pkg
			return // positive cache hit
		}

		if err = imp.errors[path]; err != nil {
			return // negative cache hit
		}

		if pkg, err = types.GcImport(imports, path); err == nil {
			info = &PackageInfo{Pkg: pkg}
			imp.Packages[path] = info
		}
	}

	if err == nil {
		// Cache success.
		pkg = info.Pkg
		imports[path] = pkg
		return pkg, nil
	}

	// Cache failure
	imp.errors[path] = err
	return nil, err
}

// imports returns the set of paths imported by the specified files.
func imports(p string, files []*ast.File) map[string]bool {
	imports := make(map[string]bool)
outer:
	for _, file := range files {
		for _, decl := range file.Decls {
			if decl, ok := decl.(*ast.GenDecl); ok {
				if decl.Tok != token.IMPORT {
					break outer // stop at the first non-import
				}
				for _, spec := range decl.Specs {
					spec := spec.(*ast.ImportSpec)
					if path, _ := strconv.Unquote(spec.Path.Value); path != "C" {
						imports[path] = true
					}
				}
			} else {
				break outer // stop at the first non-import
			}
		}
	}
	return imports
}

// LoadPackage loads the package of the specified import path,
// performs type-checking, and returns the corresponding
// PackageInfo.
//
// Precondition: Importer.config.Build != nil.
// Idempotent and thread-safe.
//
// TODO(adonovan): fix: violated in call from CreatePackageFromArgs!
// TODO(adonovan): rethink this API.
//
func (imp *Importer) LoadPackage(importPath string) (*PackageInfo, error) {
	if imp.config.Build == nil {
		panic("Importer.LoadPackage without a go/build.Config")
	}

	imp.mu.Lock()
	pi, ok := imp.packages[importPath]
	if !ok {
		// First request for this pkgInfo:
		// create a new one in !ready state.
		pi = &pkgInfo{ready: make(chan struct{})}
		imp.packages[importPath] = pi
		imp.mu.Unlock()

		// Load/parse the package.
		if files, err := loadPackage(imp.config.Build, imp.Fset, importPath); err == nil {
			// Kick off asynchronous I/O and parsing for the imports.
			for path := range imports(importPath, files) {
				go func(path string) { imp.LoadPackage(path) }(path)
			}

			// Type-check the package.
			pi.info, pi.err = imp.CreateSourcePackage(importPath, files)
		} else {
			pi.err = err
		}

		close(pi.ready) // enter ready state and wake up waiters
	} else {
		imp.mu.Unlock()

		<-pi.ready // wait for ready condition.
	}

	// Invariant: pi is ready.

	if pi.err != nil {
		return nil, pi.err
	}
	return pi.info, nil
}

// CreateSourcePackage invokes the type-checker on files and returns a
// PackageInfo containing the resulting type-checker package, the
// ASTs, and other type information.
//
// The order of files determines the package initialization order.
//
// importPath is the full name under which this package is known, such
// as appears in an import declaration. e.g. "sync/atomic".
//
// Postcondition: for result (info, err), info.Err == err.
//
func (imp *Importer) CreateSourcePackage(importPath string, files []*ast.File) (*PackageInfo, error) {
	info := &PackageInfo{
		Files: files,
		Info: types.Info{
			Types:      make(map[ast.Expr]types.Type),
			Values:     make(map[ast.Expr]exact.Value),
			Objects:    make(map[*ast.Ident]types.Object),
			Implicits:  make(map[ast.Node]types.Object),
			Scopes:     make(map[ast.Node]*types.Scope),
			Selections: make(map[*ast.SelectorExpr]*types.Selection),
		},
	}
	info.Pkg, info.Err = imp.config.TypeChecker.Check(importPath, imp.Fset, files, &info.Info)
	imp.Packages[importPath] = info
	return info, info.Err
}
<|endoftext|>


<commit_before>/*
Copyright 2015 Steven Labrum

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package CheckIt

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"net/http"
	"path/filepath"
	"strconv"
	"strings"
	"text/template"
)

var outputs = [10]*CompileOut{}
var boxes = []*BoxStruct{}
var interfaces = []Box{}
var page = Page{}
var aboutPage = Page{}
var about = AboutStruct{}
var configuration = &Config{}

var (
	httpListen = flag.String("http", "127.0.0.1:3999", "host:port to listen on")
	htmlOutput = flag.Bool("html", false, "render program output as HTML")
)

func baseCase(w http.ResponseWriter, r *http.Request) {

	headTemp.Execute(w, nil)
	openBodyTemp.Execute(w, nil)
	pageStartTemp.Execute(w, page)

	for key := range boxes {
		boxTemp.Execute(w, boxes[key])
	}

	pageCloseTemp.Execute(w, nil)
	htmlCloseTemp.Execute(w, nil)

}

/*  FrontPage is an HTTP handler that displays the basecase
unless a stored page is being loaded.
*/
func FrontPage(w http.ResponseWriter, r *http.Request) {
	title := r.URL.Path[len("/"):]

	if len(title) < 1 {
		baseCase(w, r)
	} else {
		title := r.URL.Path[len("/"):]
		title = configuration.Path + "/" + title
		pageNames, _ := filepath.Glob(configuration.Path + "/" + title + "/*.page")
		boxNames, _ := filepath.Glob(configuration.Path + "/" + title + "/*.box")

		fmt.Println(len(pageNames))
		fmt.Println("Loaded shared page")

		if pageNames == nil || boxNames == nil {
			http.Redirect(w, r, "/", http.StatusFound)
		} else {

			pageName := pageNames[0]
			fmt.Println(boxNames)

			headTemp.Execute(w, nil)
			openBodyTemp.Execute(w, nil)

			configuration = ReadConfig(configuration.Path + pageName)

			initConfig(configuration)

			pageStartTemp.Execute(w, page)

			boxes = []*BoxStruct{}
			for key := range boxNames {
				boxP := ReadBox(configuration.Path + boxNames[key])
				boxes = append(boxes, boxP)
				boxTemp.Execute(w, boxP)
			}

			pageCloseTemp.Execute(w, nil)
			htmlCloseTemp.Execute(w, nil)
		}
	}
}

func AboutPage(w http.ResponseWriter, r *http.Request) {

	headTemp.Execute(w, nil)
	openBodyTemp.Execute(w, nil)
	pageStartTemp.Execute(w, aboutPage)
	aboutTemp.Execute(w, about)
	pageCloseTemp.Execute(w, nil)
	htmlCloseTemp.Execute(w, nil)

}

var outputText = `<pre>{{printf "%s" . |html}}</pre>`
var output = template.Must(template.New("output").Parse(outputText))
var shareText = `{{printf "%s" . |html}}`
var shareOutput = template.Must(template.New("shareOutput").Parse(shareText))

// Compile is an HTTP handler that reads Source code from the request,
// runs the program (returning any errors),
// and sends the program's output as the HTTP response.
func PipeCompile(w http.ResponseWriter, req *http.Request) {

	title := req.URL.Path[len("/pipeile/"):]

	fmt.Println(title)
	str := strings.Split(title, "/")
	title = str[0]

	position, _ := strconv.Atoi(str[1])

	body := new(bytes.Buffer)

	if _, err := body.ReadFrom(req.Body); err != nil {
		return
	}

	fmt.Println(position)

	updateBody(boxes, title, body.String())

	out, err := InterfaceRun(interfaces[position-1], body.Bytes(), title)
	compOut := CompileOut{Out: out, Error: err}

	outputs[position-1] = &compOut

	if err != nil {
		w.WriteHeader(404)
		output.Execute(w, out)
	} else if *htmlOutput {
		w.Write(out)
	} else {
		output.Execute(w, out)
	}
}

func sharHandler(w http.ResponseWriter, r *http.Request) {
	out := Share()
	fmt.Println("PATH :/ " + configuration.Path)
	Save(configuration.Path+"/", out)
	shareOutput.Execute(w, out)
}

func initConfig(config *Config) {
	configuration = config
	page.Heading = config.Heading
	page.SubHeading = config.SubHeading
	about.Text = config.About
	about.SecondaryText = config.AboutSide
	aboutPage.Heading = "About"
	aboutPage.SubHeading = ""
}

func initBoxes(boxs ...Box) {

	for key := range boxs {

		var box = BoxStruct{}

		box.Id = strconv.Itoa(key)
		box.Position = strconv.Itoa(key + 1)
		box.Total = len(boxs)
		box.Lang = boxs[key].Syntax()
		box.Body = boxs[key].Default()
		box.Head = "Heading"
		box.SubHead = "Subhead"
		box.Text = boxs[key].Help()

		boxes = append(boxes, &box)

		interfaces = append(interfaces, boxs[key])

	}
}

func Serve(config *Config, boxs ...Box) (err error) {
	initConfig(config)
	initBoxes(boxs...)

	fmt.Println("cool beans")
	http.HandleFunc("/share/", sharHandler)
	http.HandleFunc("/about", AboutPage)
	http.HandleFunc("/", FrontPage)
	http.HandleFunc("/compile/", PipeCompile)
	http.Handle("/css/", http.StripPrefix("/css/", http.FileServer(http.Dir("css"))))
	http.Handle("/fonts/", http.StripPrefix("/fonts/", http.FileServer(http.Dir("fonts"))))
	http.Handle("/js/", http.StripPrefix("/js", http.FileServer(http.Dir("js"))))
	http.ListenAndServe(":"+configuration.Port, nil)

	err = errors.New("Server crashed")
	return err
}
<commit_msg>load path<commit_after>/*
Copyright 2015 Steven Labrum

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package CheckIt

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"net/http"
	"path/filepath"
	"strconv"
	"strings"
	"text/template"
)

var outputs = [10]*CompileOut{}
var boxes = []*BoxStruct{}
var interfaces = []Box{}
var page = Page{}
var aboutPage = Page{}
var about = AboutStruct{}
var configuration = &Config{}

var (
	httpListen = flag.String("http", "127.0.0.1:3999", "host:port to listen on")
	htmlOutput = flag.Bool("html", false, "render program output as HTML")
)

func baseCase(w http.ResponseWriter, r *http.Request) {

	headTemp.Execute(w, nil)
	openBodyTemp.Execute(w, nil)
	pageStartTemp.Execute(w, page)

	for key := range boxes {
		boxTemp.Execute(w, boxes[key])
	}

	pageCloseTemp.Execute(w, nil)
	htmlCloseTemp.Execute(w, nil)

}

/*  FrontPage is an HTTP handler that displays the basecase
unless a stored page is being loaded.
*/
func FrontPage(w http.ResponseWriter, r *http.Request) {
	title := r.URL.Path[len("/"):]

	if len(title) < 1 {
		baseCase(w, r)
	} else {
		title := r.URL.Path[len("/"):]
		title = configuration.Path + "/" + title
		fmt.Println(title)
		pageNames, _ := filepath.Glob(title + "/*.page")
		boxNames, _ := filepath.Glob(title + "/*.box")

		fmt.Println(len(pageNames))
		fmt.Println("Loaded shared page")

		if pageNames == nil || boxNames == nil {
			http.Redirect(w, r, "/", http.StatusFound)
		} else {

			pageName := pageNames[0]
			fmt.Println(boxNames)

			headTemp.Execute(w, nil)
			openBodyTemp.Execute(w, nil)

			configuration = ReadConfig(configuration.Path + pageName)

			initConfig(configuration)

			pageStartTemp.Execute(w, page)

			boxes = []*BoxStruct{}
			for key := range boxNames {
				boxP := ReadBox(configuration.Path + boxNames[key])
				boxes = append(boxes, boxP)
				boxTemp.Execute(w, boxP)
			}

			pageCloseTemp.Execute(w, nil)
			htmlCloseTemp.Execute(w, nil)
		}
	}
}

func AboutPage(w http.ResponseWriter, r *http.Request) {

	headTemp.Execute(w, nil)
	openBodyTemp.Execute(w, nil)
	pageStartTemp.Execute(w, aboutPage)
	aboutTemp.Execute(w, about)
	pageCloseTemp.Execute(w, nil)
	htmlCloseTemp.Execute(w, nil)

}

var outputText = `<pre>{{printf "%s" . |html}}</pre>`
var output = template.Must(template.New("output").Parse(outputText))
var shareText = `{{printf "%s" . |html}}`
var shareOutput = template.Must(template.New("shareOutput").Parse(shareText))

// Compile is an HTTP handler that reads Source code from the request,
// runs the program (returning any errors),
// and sends the program's output as the HTTP response.
func PipeCompile(w http.ResponseWriter, req *http.Request) {

	title := req.URL.Path[len("/pipeile/"):]

	fmt.Println(title)
	str := strings.Split(title, "/")
	title = str[0]

	position, _ := strconv.Atoi(str[1])

	body := new(bytes.Buffer)

	if _, err := body.ReadFrom(req.Body); err != nil {
		return
	}

	fmt.Println(position)

	updateBody(boxes, title, body.String())

	out, err := InterfaceRun(interfaces[position-1], body.Bytes(), title)
	compOut := CompileOut{Out: out, Error: err}

	outputs[position-1] = &compOut

	if err != nil {
		w.WriteHeader(404)
		output.Execute(w, out)
	} else if *htmlOutput {
		w.Write(out)
	} else {
		output.Execute(w, out)
	}
}

func sharHandler(w http.ResponseWriter, r *http.Request) {
	out := Share()
	fmt.Println("PATH :/ " + configuration.Path)
	Save(configuration.Path+"/", out)
	shareOutput.Execute(w, out)
}

func initConfig(config *Config) {
	configuration = config
	page.Heading = config.Heading
	page.SubHeading = config.SubHeading
	about.Text = config.About
	about.SecondaryText = config.AboutSide
	aboutPage.Heading = "About"
	aboutPage.SubHeading = ""
}

func initBoxes(boxs ...Box) {

	for key := range boxs {

		var box = BoxStruct{}

		box.Id = strconv.Itoa(key)
		box.Position = strconv.Itoa(key + 1)
		box.Total = len(boxs)
		box.Lang = boxs[key].Syntax()
		box.Body = boxs[key].Default()
		box.Head = "Heading"
		box.SubHead = "Subhead"
		box.Text = boxs[key].Help()

		boxes = append(boxes, &box)

		interfaces = append(interfaces, boxs[key])

	}
}

func Serve(config *Config, boxs ...Box) (err error) {
	initConfig(config)
	initBoxes(boxs...)

	fmt.Println("cool beans")
	http.HandleFunc("/share/", sharHandler)
	http.HandleFunc("/about", AboutPage)
	http.HandleFunc("/", FrontPage)
	http.HandleFunc("/compile/", PipeCompile)
	http.Handle("/css/", http.StripPrefix("/css/", http.FileServer(http.Dir("css"))))
	http.Handle("/fonts/", http.StripPrefix("/fonts/", http.FileServer(http.Dir("fonts"))))
	http.Handle("/js/", http.StripPrefix("/js", http.FileServer(http.Dir("js"))))
	http.ListenAndServe(":"+configuration.Port, nil)

	err = errors.New("Server crashed")
	return err
}
<|endoftext|>


<commit_before>package Stitcher

import (
	"fmt"
	"github.com/disintegration/imaging"
	"image"
)

const (
	squareSide = 640
)

func CropCenter(sourceImage image.Image) {
	x0 := (sourceImage.Bounds().Size().X - squareSide) / 2
	y0 := (sourceImage.Bounds().Size().Y - squareSide) / 2

	croppedPath := fmt.Sprintf("Cropped-%s", filePath)
	cropTo640Squared(sourceImage, croppedPath, x0, y0)
}

func CropFromTop(sourceImage image.Image) {
	x0 := 0
	y0 := 0

	croppedPath := fmt.Sprintf("Cropped-%s", filePath)
	cropTo640Squared(sourceImage, croppedPath, x0, y0)
}

func CropFromBottom(sourceImage image.Image) {
	x0 := sourceImage.Bounds().Size().X - squareSide
	y0 := sourceImage.Bounds().Size().Y - squareSide

	croppedPath := fmt.Sprintf("Cropped-%s", filePath)
	cropTo640Squared(sourceImage, croppedPath, x0, y0)
}

////////////////////////////////////////////////////////////////////////////////
//Private helpers

func cropImage(sourceImage image.Image, x0, y0, x1, y1 int) image.Image {
	currImage := imaging.Clone(sourceImage)
	croppedImage := imaging.Crop(currImage, image.Rect(x0, y0, x1, y1))
	return croppedImage
}

func cropTo640Squared(sourceImage image.Image, outputFileName string, x0, y0 int) {
	x1 := squareSide + x0
	y1 := squareSide + y0

	finalImage := cropImage(sourceImage, x0, y0, x1, y1)
	imaging.Save(finalImage, outputFileName)
}
<commit_msg>Cropper now returns images instead of saving them to the file syste<commit_after>package Stitcher

import (
	"github.com/disintegration/imaging"
	"image"
)

const (
	squareSide = 640
)

func CropCenter(sourceImage image.Image) image.Image {
	x0 := (sourceImage.Bounds().Size().X - squareSide) / 2
	y0 := (sourceImage.Bounds().Size().Y - squareSide) / 2

	croppedImage := cropTo640Squared(sourceImage, x0, y0)
	return croppedImage
}

func CropFromTop(sourceImage image.Image) image.Image {
	x0 := 0
	y0 := 0

	croppedImage := cropTo640Squared(sourceImage, x0, y0)
	return croppedImage
}

func CropFromBottom(sourceImage image.Image) image.Image {
	x0 := sourceImage.Bounds().Size().X - squareSide
	y0 := sourceImage.Bounds().Size().Y - squareSide

	croppedImage := cropTo640Squared(sourceImage, x0, y0)
	return croppedImage
}

////////////////////////////////////////////////////////////////////////////////
//Private helpers

func cropImage(sourceImage image.Image, x0, y0, x1, y1 int) image.Image {
	currImage := imaging.Clone(sourceImage)
	croppedImage := imaging.Crop(currImage, image.Rect(x0, y0, x1, y1))

	return croppedImage
}

func cropTo640Squared(sourceImage image.Image, x0, y0 int) image.Image {
	x1 := squareSide + x0
	y1 := squareSide + y0

	finalImage := cropImage(sourceImage, x0, y0, x1, y1)
	return finalImage
}
<|endoftext|>