<commit_before>package fscache

import (
	"bytes"
	"encoding/gob"
	"fmt"
	"io"
	"net/http"
	"sync"
)

// Handler is a caching middle-ware for http Handlers.
// It responds to http requests via the passed http.Handler, and caches the response
// using the passed cache. The cache key for the request is the req.URL.String().
func Handler(c Cache, h http.Handler) http.Handler {
	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
		url := req.URL.String()
		r, w, err := c.Get(url)
		if err != nil {
			h.ServeHTTP(rw, req)
			return
		}
		defer r.Close()
		if w != nil {
			go func() {
				defer w.Close()
				h.ServeHTTP(&respWrapper{
					ResponseWriter: rw,
					Writer:         w,
				}, req)
			}()
		}
		decodeHeader(r, rw.Header())
		io.Copy(rw, r)
	})
}

func encodeHeader(w io.Writer, h http.Header) error {
	buf := bytes.NewBuffer(nil)
	enc := gob.NewEncoder(buf)
	err := enc.Encode(h)
	if err != nil {
		return err
	}
	_, err = fmt.Fprintln(w, int64(buf.Len()))
	if err != nil {
		return err
	}
	_, err = io.Copy(w, buf)
	return err
}

func decodeHeader(r io.Reader, h http.Header) error {
	var size int64
	_, err := fmt.Fscanln(r, &size)
	if err != nil {
		return err
	}
	dec := gob.NewDecoder(io.LimitReader(r, size))
	return dec.Decode(&h)
}

type respWrapper struct {
	http.ResponseWriter
	io.Writer
	sync.Once
}

func (r *respWrapper) Write(p []byte) (int, error) {
	r.Do(func() { encodeHeader(r.Writer, r.Header()) })
	return r.Writer.Write(p)
}
<commit_msg>removed auto-http-header encoding to the Handler, its expensive and easy to do manually<commit_after>package fscache

import (
	"io"
	"net/http"
)

// Handler is a caching middle-ware for http Handlers.
// It responds to http requests via the passed http.Handler, and caches the response
// using the passed cache. The cache key for the request is the req.URL.String().
// Note: It does not cache http headers. It is more efficient to set them yourself.
func Handler(c Cache, h http.Handler) http.Handler {
	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
		url := req.URL.String()
		r, w, err := c.Get(url)
		if err != nil {
			h.ServeHTTP(rw, req)
			return
		}
		defer r.Close()
		if w != nil {
			go func() {
				defer w.Close()
				h.ServeHTTP(&respWrapper{
					ResponseWriter: rw,
					Writer:         w,
				}, req)
			}()
		}
		io.Copy(rw, r)
	})
}

type respWrapper struct {
	http.ResponseWriter
	io.Writer
}

func (r *respWrapper) Write(p []byte) (int, error) {
	return r.Writer.Write(p)
}
<|endoftext|>


<commit_before>package compress

import (
	"compress/gzip"
	"net/http"
	"strings"

	"github.com/volatile/core"
	"github.com/volatile/core/coreutil"
)

// Use tells the core to use this handler.
func Use() {
	core.Use(func(c *core.Context) {
		var gzw *gzip.Writer
		gzw, c = setWriter(c)
		c.Next()
		gzw.Close()
	})
}

// LocalUse allows to compress locally, inside a single handler.
// Only the ResponseWriter is transmitted to avoid errors like calling c.Next() and risk to use multiple compressors over the response.
func LocalUse(c *core.Context, wf func(http.ResponseWriter)) {
	var gzw *gzip.Writer
	gzw, c = setWriter(c)
	wf(c.ResponseWriter)
	gzw.Close()
}

// setWriter create new ResponseWriter that will write to GZIP and returns it for the next handlers.
// The GZIP writer is also returned to close if ony after the handler has been exexuted.
func setWriter(c *core.Context) (*gzip.Writer, *core.Context) {
	var gzw *gzip.Writer

	if strings.Contains(c.Request.Header.Get("Accept-Encoding"), "gzip") && len(c.Request.Header.Get("Sec-WebSocket-Key")) == 0 {
		c.ResponseWriter.Header().Set("Content-Encoding", "gzip")

		gzw = gzip.NewWriter(c.ResponseWriter)
		// defer gzw.Close()

		// Set the new ResponseWriter.
		c.ResponseWriter = core.ResponseWriterBinder{
			Writer:         gzw,
			ResponseWriter: c.ResponseWriter,
			BeforeWrite: func(b []byte) {
				coreutil.SetDetectedContentType(c.ResponseWriter, b)
			},
		}
	}

	return gzw, c
}
<commit_msg>Remove error-prone LocalUse<commit_after>package compress

import (
	"compress/gzip"
	"strings"

	"github.com/volatile/core"
	"github.com/volatile/core/coreutil"
)

// Use tells the core to compress all responses.
func Use() {
	core.Use(func(c *core.Context) {
		if strings.Contains(c.Request.Header.Get("Accept-Encoding"), "gzip") && len(c.Request.Header.Get("Sec-WebSocket-Key")) == 0 {
			c.ResponseWriter.Header().Set("Content-Encoding", "gzip")

			gzw := gzip.NewWriter(c.ResponseWriter)
			defer gzw.Close()

			// Set the new ResponseWriter.
			c.ResponseWriter = core.ResponseWriterBinder{
				Writer:         gzw,
				ResponseWriter: c.ResponseWriter,
				BeforeWrite: func(b []byte) {
					coreutil.SetDetectedContentType(c.ResponseWriter, b)
				},
			}
		}

		c.Next()
	})
}
<|endoftext|>


<commit_before>//
// Package eprinttools is a collection of structures and functions for working with the E-Prints REST API
//
// @author R. S. Doiel, <rsdoiel@caltech.edu>
//
// Copyright (c) 2017, Caltech
// All rights not granted herein are expressly reserved by Caltech.
//
// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
package eprinttools

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"sort"
	"strings"
	"time"

	// CaltechLibrary packages
	"github.com/caltechlibrary/dataset"
)

// ExportEPrintsKeyList export a list of eprints from a list of keys
func (api *EPrintsAPI) ExportEPrintsKeyList(keys []string, saveKeys string, verbose bool) error {
	var (
		exportedKeys []string
		err          error
	)

	c, err := dataset.Open(api.Dataset)
	if err != nil {
		return fmt.Errorf("ExportEPrintsKeyList() %s, %s", api.Dataset, err)
	}
	defer c.Close()

	uris := []string{}
	for _, key := range keys {
		uri := fmt.Sprintf("/rest/eprint/%s.xml", strings.TrimSpace(key))
		uris = append(uris, uri)
	}

	uriCount := len(uris)
	count := uriCount
	j := 0 // success count
	k := 0 // error count
	if verbose == true {
		log.Printf("Exporting %d of %d uris", count, uriCount)
	}
	for i := 0; i < uriCount && i < count; i++ {
		uri := uris[i]
		rec, xmlSrc, err := api.GetEPrint(uri)
		if err != nil {
			log.Printf("Failed, %s\n", err)
			k++
		} else {
			key := fmt.Sprintf("%d", rec.EPrintID)
			// NOTE: Check to see if we're doing an update or create
			if c.HasKey(key) == true {
				err = c.UpdateFrom(key, rec)
			} else {
				err = c.CreateFrom(key, rec)
			}
			if err == nil {
				if len(saveKeys) > 0 {
					exportedKeys = append(exportedKeys, key)
				}
				// We've exported a record successfully, now update select lists
				j++
			} else {
				if verbose == true {
					log.Printf("Failed to save eprint %s (%s) to %s, %s\n", key, uri, api.Dataset, err)
				}
				k++
			}
			c.AttachFile(key, key+".xml", bytes.NewReader(xmlSrc))
		}
		if verbose == true && (i%EPrintsExportBatchSize) == 0 {
			log.Printf("%d/%d uri processed, %d exported, %d unexported", i+1, count, j, k)
		}
	}
	if verbose == true {
		log.Printf("%d/%d uri processed, %d exported, %d unexported", len(uris), count, j, k)
	}
	if len(saveKeys) > 0 {
		if err := ioutil.WriteFile(saveKeys, []byte(strings.Join(exportedKeys, "\n")), 0664); err != nil {
			return fmt.Errorf("Failed to export %s, %s", saveKeys, err)
		}
	}
	return nil
}

// ExportEPrints from highest ID to lowest for cnt. Saves each record in a DB and indexes published ones
func (api *EPrintsAPI) ExportEPrints(count int, saveKeys string, verbose bool) error {
	var (
		exportedKeys []string
		err          error
	)

	c, err := dataset.InitCollection(api.Dataset)
	if err != nil {
		return fmt.Errorf("ExportEPrints() %s, %s", api.Dataset, err)
	}
	defer c.Close()

	uris, err := api.ListEPrintsURI()
	if err != nil {
		return fmt.Errorf("Export %s failed, %s", api.URL.String(), err)
	}

	// NOTE: I am sorting the URI by decscending ID number so that the
	// newest articles are exported first
	sort.Sort(byURI(uris))

	uriCount := len(uris)
	if count < 0 {
		count = uriCount
	}
	j := 0 // success count
	k := 0 // error count
	if verbose == true {
		log.Printf("Exporting %d of %d uris", count, uriCount)
	}
	for i := 0; i < uriCount && i < count; i++ {
		uri := uris[i]
		rec, xmlSrc, err := api.GetEPrint(uri)
		if err != nil {
			log.Printf("Failed, %s\n", err)
			k++
		} else {
			key := rec.ID
			// NOTE: Check to see if we're doing an update or create
			if c.HasKey(key) == true {
				err = c.UpdateFrom(key, rec)
			} else {
				err = c.CreateFrom(key, rec)
			}
			if err == nil {
				if len(saveKeys) > 0 {
					exportedKeys = append(exportedKeys, key)
				}
				// We've exported a record successfully, now update select lists
				j++
			} else {
				if verbose == true {
					log.Printf("Failed to save eprint %s (%s) to %s, %s\n", key, uri, api.Dataset, err)
				}
				k++
			}
			c.AttachFile(key, key+".xml", bytes.NewReader(xmlSrc))
		}
		if verbose == true && (i%EPrintsExportBatchSize) == 0 {
			log.Printf("%d/%d uri processed, %d exported, %d unexported", i+1, count, j, k)
		}
	}
	if verbose == true {
		log.Printf("%d/%d uri processed, %d exported, %d unexported", len(uris), count, j, k)
	}
	if len(saveKeys) > 0 {
		if err := ioutil.WriteFile(saveKeys, []byte(strings.Join(exportedKeys, "\n")), 0664); err != nil {
			return fmt.Errorf("Failed to export %s, %s", saveKeys, err)
		}
	}
	return nil
}

// ExportModifiedEPrints returns a list of ids modified in one or between the start, end times
func (api *EPrintsAPI) ExportModifiedEPrints(start, end time.Time, saveKeys string, verbose bool) error {
	var (
		exportedKeys []string
		err          error
	)

	c, err := dataset.InitCollection(api.Dataset)
	if err != nil {
		return fmt.Errorf("ExportModifiedEPrints() %s, %s", api.Dataset, err)
	}
	defer c.Close()

	uris, err := api.ListModifiedEPrintURI(start, end, verbose)
	if err != nil {
		return fmt.Errorf("Export modified %s to %s failed, %s", start, end, err)
	}

	// NOTE: I am sorting the URI by decscending ID number so that the
	// newest articles are exported first
	sort.Sort(byURI(uris))

	count := len(uris)
	j := 0 // success count
	k := 0 // error count
	if verbose == true {
		log.Printf("Exporting %d uris", count)
	}
	for i := 0; i < count; i++ {
		uri := uris[i]
		rec, xmlSrc, err := api.GetEPrint(uri)
		if err != nil {
			if verbose == true {
				log.Printf("Failed to get %s, %s\n", uri, err)
			}
			k++
		} else {
			key := rec.ID
			// NOTE: Check to see if we're doing an update or create
			if c.HasKey(key) == true {
				err = c.UpdateFrom(key, rec)
			} else {
				err = c.CreateFrom(key, rec)
			}
			if err == nil {
				if len(saveKeys) > 0 {
					exportedKeys = append(exportedKeys, key)
				}
				// We've exported a record successfully, now update select lists
				j++
			} else {
				if verbose == true {
					log.Printf("Failed to save eprint %s (%s) to %s, %s\n", key, uri, api.Dataset, err)
				}
				k++
			}
			c.AttachFile(key, key+".xml", bytes.NewReader(xmlSrc))
		}
		if verbose == true && (i%EPrintsExportBatchSize) == 0 {
			log.Printf("%d/%d uri processed, %d exported, %d unexported", i+1, count, j, k)
		}
	}
	if verbose == true {
		log.Printf("%d/%d uri processed, %d exported, %d unexported", len(uris), count, j, k)
	}
	if len(saveKeys) > 0 {
		if err := ioutil.WriteFile(saveKeys, []byte(strings.Join(exportedKeys, "\n")), 0664); err != nil {
			return fmt.Errorf("Failed to export %s, %s", saveKeys, err)
		}
	}
	return nil
}
<commit_msg>update code to use latest dataset CreateJSON(), and UpdateJSON() replacing CreateFrom(), and UpdateFrom()<commit_after>//
// Package eprinttools is a collection of structures and functions for working with the E-Prints REST API
//
// @author R. S. Doiel, <rsdoiel@caltech.edu>
//
// Copyright (c) 2017, Caltech
// All rights not granted herein are expressly reserved by Caltech.
//
// Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
package eprinttools

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"sort"
	"strings"
	"time"

	// CaltechLibrary packages
	"github.com/caltechlibrary/dataset"
)

// ExportEPrintsKeyList export a list of eprints from a list of keys
func (api *EPrintsAPI) ExportEPrintsKeyList(keys []string, saveKeys string, verbose bool) error {
	var (
		exportedKeys []string
		err          error
		src          []byte
	)

	c, err := dataset.Open(api.Dataset)
	if err != nil {
		return fmt.Errorf("ExportEPrintsKeyList() %s, %s", api.Dataset, err)
	}
	defer c.Close()

	uris := []string{}
	for _, key := range keys {
		uri := fmt.Sprintf("/rest/eprint/%s.xml", strings.TrimSpace(key))
		uris = append(uris, uri)
	}

	uriCount := len(uris)
	count := uriCount
	j := 0 // success count
	k := 0 // error count
	if verbose == true {
		log.Printf("Exporting %d of %d uris", count, uriCount)
	}
	for i := 0; i < uriCount && i < count; i++ {
		uri := uris[i]
		rec, xmlSrc, err := api.GetEPrint(uri)
		if err != nil {
			log.Printf("Failed, %s\n", err)
			k++
		} else {
			key := fmt.Sprintf("%d", rec.EPrintID)
			src, err = json.Marshal(rec)
			if err != nil {
				log.Printf("can't marshal key %s, %s", key, err)
			} else {
				// NOTE: Check to see if we're doing an update or create
				if c.HasKey(key) == true {
					err = c.UpdateJSON(key, src)
				} else {
					err = c.CreateJSON(key, src)
				}
			}
			if err == nil {
				if len(saveKeys) > 0 {
					exportedKeys = append(exportedKeys, key)
				}
				// We've exported a record successfully, now update select lists
				j++
			} else {
				if verbose == true {
					log.Printf("Failed to save eprint %s (%s) to %s, %s\n", key, uri, api.Dataset, err)
				}
				k++
			}
			c.AttachFile(key, key+".xml", bytes.NewReader(xmlSrc))
		}
		if verbose == true && (i%EPrintsExportBatchSize) == 0 {
			log.Printf("%d/%d uri processed, %d exported, %d unexported", i+1, count, j, k)
		}
	}
	if verbose == true {
		log.Printf("%d/%d uri processed, %d exported, %d unexported", len(uris), count, j, k)
	}
	if len(saveKeys) > 0 {
		if err := ioutil.WriteFile(saveKeys, []byte(strings.Join(exportedKeys, "\n")), 0664); err != nil {
			return fmt.Errorf("Failed to export %s, %s", saveKeys, err)
		}
	}
	return nil
}

// ExportEPrints from highest ID to lowest for cnt. Saves each record in a DB and indexes published ones
func (api *EPrintsAPI) ExportEPrints(count int, saveKeys string, verbose bool) error {
	var (
		exportedKeys []string
		err          error
		src          []byte
	)

	c, err := dataset.InitCollection(api.Dataset)
	if err != nil {
		return fmt.Errorf("ExportEPrints() %s, %s", api.Dataset, err)
	}
	defer c.Close()

	uris, err := api.ListEPrintsURI()
	if err != nil {
		return fmt.Errorf("Export %s failed, %s", api.URL.String(), err)
	}

	// NOTE: I am sorting the URI by decscending ID number so that the
	// newest articles are exported first
	sort.Sort(byURI(uris))

	uriCount := len(uris)
	if count < 0 {
		count = uriCount
	}
	j := 0 // success count
	k := 0 // error count
	if verbose == true {
		log.Printf("Exporting %d of %d uris", count, uriCount)
	}
	for i := 0; i < uriCount && i < count; i++ {
		uri := uris[i]
		rec, xmlSrc, err := api.GetEPrint(uri)
		if err != nil {
			log.Printf("Failed, %s\n", err)
			k++
		} else {
			key := fmt.Sprintf("%d", rec.EPrintID)
			src, err = json.Marshal(rec)
			if err != nil {
				log.Printf("Can't marshal key %s, %s", key, err)
			} else {
				// NOTE: Check to see if we're doing an update or create
				if c.HasKey(key) == true {
					err = c.UpdateJSON(key, src)
				} else {
					err = c.CreateJSON(key, src)
				}
			}
			if err == nil {
				if len(saveKeys) > 0 {
					exportedKeys = append(exportedKeys, key)
				}
				// We've exported a record successfully, now update select lists
				j++
			} else {
				if verbose == true {
					log.Printf("Failed to save eprint %s (%s) to %s, %s\n", key, uri, api.Dataset, err)
				}
				k++
			}
			c.AttachFile(key, key+".xml", bytes.NewReader(xmlSrc))
		}
		if verbose == true && (i%EPrintsExportBatchSize) == 0 {
			log.Printf("%d/%d uri processed, %d exported, %d unexported", i+1, count, j, k)
		}
	}
	if verbose == true {
		log.Printf("%d/%d uri processed, %d exported, %d unexported", len(uris), count, j, k)
	}
	if len(saveKeys) > 0 {
		if err := ioutil.WriteFile(saveKeys, []byte(strings.Join(exportedKeys, "\n")), 0664); err != nil {
			return fmt.Errorf("Failed to export %s, %s", saveKeys, err)
		}
	}
	return nil
}

// ExportModifiedEPrints returns a list of ids modified in one or between the start, end times
func (api *EPrintsAPI) ExportModifiedEPrints(start, end time.Time, saveKeys string, verbose bool) error {
	var (
		exportedKeys []string
		err          error
		src          []byte
	)

	c, err := dataset.InitCollection(api.Dataset)
	if err != nil {
		return fmt.Errorf("ExportModifiedEPrints() %s, %s", api.Dataset, err)
	}
	defer c.Close()

	uris, err := api.ListModifiedEPrintURI(start, end, verbose)
	if err != nil {
		return fmt.Errorf("Export modified %s to %s failed, %s", start, end, err)
	}

	// NOTE: I am sorting the URI by decscending ID number so that the
	// newest articles are exported first
	sort.Sort(byURI(uris))

	count := len(uris)
	j := 0 // success count
	k := 0 // error count
	if verbose == true {
		log.Printf("Exporting %d uris", count)
	}
	for i := 0; i < count; i++ {
		uri := uris[i]
		rec, xmlSrc, err := api.GetEPrint(uri)
		if err != nil {
			if verbose == true {
				log.Printf("Failed to get %s, %s\n", uri, err)
			}
			k++
		} else {
			key := fmt.Sprintf("%d", rec.EPrintID)
			src, err = json.Marshal(rec)
			if err != nil {
				log.Printf("Can't marshel key %s, %s", key, err)
			} else {
				// NOTE: Check to see if we're doing an update or create
				if c.HasKey(key) == true {
					err = c.UpdateJSON(key, src)
				} else {
					err = c.CreateJSON(key, src)
				}
			}
			if err == nil {
				if len(saveKeys) > 0 {
					exportedKeys = append(exportedKeys, key)
				}
				// We've exported a record successfully, now update select lists
				j++
			} else {
				if verbose == true {
					log.Printf("Failed to save eprint %s (%s) to %s, %s\n", key, uri, api.Dataset, err)
				}
				k++
			}
			c.AttachFile(key, key+".xml", bytes.NewReader(xmlSrc))
		}
		if verbose == true && (i%EPrintsExportBatchSize) == 0 {
			log.Printf("%d/%d uri processed, %d exported, %d unexported", i+1, count, j, k)
		}
	}
	if verbose == true {
		log.Printf("%d/%d uri processed, %d exported, %d unexported", len(uris), count, j, k)
	}
	if len(saveKeys) > 0 {
		if err := ioutil.WriteFile(saveKeys, []byte(strings.Join(exportedKeys, "\n")), 0664); err != nil {
			return fmt.Errorf("Failed to export %s, %s", saveKeys, err)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Naive implementation of Hashmap data structure.
package ghost

import (
	"errors"
	"sync"
)

const (
	initSize  uint32  = 64   // Default number of buckets
	threshold float32 = 0.75 // Threshold load factor to rehash table
)

type node struct {
	Key string
	Val string
}

type bucket struct {
	vector
}

type hashMap struct {
	Count   uint32       // Number of elements in hashmap
	Size    uint32       // Number of buckets in hashmap
	buckets []bucket     // Array of indiviual buckets in hashmap
	locks   []sync.Mutex // Array of locks. Used to syncronize bucket access
}

func NewHashMap() *hashMap {
	newTable := &hashMap{}

	newTable.buckets = make([]bucket, initSize)

	newTable.locks = make([]sync.Mutex, initSize)

	newTable.Size = initSize

	return newTable
}

// Set or update key.
func (h *hashMap) Set(key, val string) {
	if h.loadFactor() >= threshold {
		h.rehash()
	}

	index := h.getIndex(key)

	h.acquire(index)

	bucketIndex := h.buckets[index].Find(key)

	if bucketIndex < 0 {
		h.buckets[index].Push(node{key, val})
		h.Count++
	} else {
		h.buckets[index].Nodes[bucketIndex].Val = val
	}

	h.release(index)
}

// Get element from the hashmap.
// Return error if value is not found.
func (h *hashMap) Get(key string) (string, error) {
	index := h.getIndex(key)

	h.acquire(index)

	bucketIndex := h.buckets[index].Find(key)

	if bucketIndex < 0 {
		h.release(index)
		return "", errors.New("No value")
	} else {
		h.release(index)
		return h.buckets[index].Nodes[bucketIndex].Val, nil
	}
}

// Delete element from the hashmap.
func (h *hashMap) Del(key string) {
	index := h.getIndex(key)

	h.acquire(index)

	bucketIndex := h.buckets[index].Find(key)

	if bucketIndex < 0 {
		return
	}

	h.buckets[index].Pop(bucketIndex)
	h.Count--

	h.release(index)
}

// Get current load factor.
func (h *hashMap) loadFactor() float32 {
	return float32(h.Count) / float32(h.Size)
}

// Acquire control on the bucket.
func (h *hashMap) acquire(index uint32) {
	h.locks[index%uint32(len(h.locks))].Lock()
}

// Release control on the bucket.
func (h *hashMap) release(index uint32) {
	h.locks[index%uint32(len(h.locks))].Unlock()
}

// Allocate new bigger hashmap and rehash all keys.
func (h *hashMap) rehash() {
	oldSize := h.Size

	for i := 0; i < len(h.locks); i++ {
		h.locks[i].Lock()
	}

	if oldSize != h.Size {
		return // Someone beat us to it
	}

	h.Size <<= 1
	newBuckets := make([]bucket, h.Size)

	for n := range h.nodes() {
		newBuckets[h.getIndex(n.Key)].Push(n)
	}

	h.buckets = newBuckets

	for i := len(h.locks) - 1; i >= 0; i-- {
		h.locks[i].Unlock()
	}
}

// Navigate through all nodes
func (h *hashMap) nodes() <-chan node {
	ch := make(chan node)

	go func() {
		for _, b := range h.buckets {
			for i := 0; i < b.count; i++ {
				ch <- b.Nodes[i]
			}
		}
		close(ch)
	}()

	return ch
}

// Get index of bucket key belongs to.
func (h *hashMap) getIndex(key string) uint32 {
	return FNV1a_32([]byte(key)) % h.Size
}
<commit_msg>Protect hashMap.Count variable by mutex<commit_after>// Naive implementation of Hashmap data structure.
package ghost

import (
	"errors"
	"sync"
)

const (
	initSize  uint32  = 64   // Default number of buckets
	threshold float32 = 0.75 // Threshold load factor to rehash table
)

type node struct {
	Key string
	Val string
}

type bucket struct {
	vector
}

type hashMap struct {
	Count   uint32 // Number of elements in hashmap
	CountMu sync.Mutex
	Size    uint32       // Number of buckets in hashmap
	buckets []bucket     // Array of indiviual buckets in hashmap
	locks   []sync.Mutex // Array of locks. Used to syncronize bucket access
}

func NewHashMap() *hashMap {
	newTable := &hashMap{}

	newTable.buckets = make([]bucket, initSize)

	newTable.locks = make([]sync.Mutex, initSize)

	newTable.Size = initSize

	return newTable
}

// Set or update key.
func (h *hashMap) Set(key, val string) {
	if h.loadFactor() >= threshold {
		h.rehash()
	}

	index := h.getIndex(key)

	h.acquire(index)

	bucketIndex := h.buckets[index].Find(key)

	if bucketIndex < 0 {
		h.buckets[index].Push(node{key, val})

		h.CountMu.Lock()
		h.Count++
		h.CountMu.Unlock()
	} else {
		h.buckets[index].Nodes[bucketIndex].Val = val
	}

	h.release(index)
}

// Get element from the hashmap.
// Return error if value is not found.
func (h *hashMap) Get(key string) (string, error) {
	index := h.getIndex(key)

	h.acquire(index)

	bucketIndex := h.buckets[index].Find(key)

	if bucketIndex < 0 {
		h.release(index)
		return "", errors.New("No value")
	} else {
		val := h.buckets[index].Nodes[bucketIndex].Val
		h.release(index)

		return val, nil
	}
}

// Delete element from the hashmap.
func (h *hashMap) Del(key string) {
	index := h.getIndex(key)

	h.acquire(index)

	bucketIndex := h.buckets[index].Find(key)

	if bucketIndex < 0 {
		return
	}

	h.buckets[index].Pop(bucketIndex)

	h.CountMu.Lock()
	h.Count--
	h.CountMu.Unlock()

	h.release(index)
}

// Get current load factor.
func (h *hashMap) loadFactor() float32 {
	h.CountMu.Lock()
	factor := float32(h.Count) / float32(h.Size)
	h.CountMu.Unlock()

	return factor
}

// Acquire control on the bucket.
func (h *hashMap) acquire(index uint32) {
	h.locks[index%uint32(len(h.locks))].Lock()
}

// Release control on the bucket.
func (h *hashMap) release(index uint32) {
	h.locks[index%uint32(len(h.locks))].Unlock()
}

// Allocate new bigger hashmap and rehash all keys.
func (h *hashMap) rehash() {
	oldSize := h.Size

	for i := 0; i < len(h.locks); i++ {
		h.locks[i].Lock()
	}

	if oldSize != h.Size {
		return // Someone beat us to it
	}

	h.Size <<= 1
	newBuckets := make([]bucket, h.Size)

	for n := range h.nodes() {
		newBuckets[h.getIndex(n.Key)].Push(n)
	}

	h.buckets = newBuckets

	for i := len(h.locks) - 1; i >= 0; i-- {
		h.locks[i].Unlock()
	}
}

// Navigate through all nodes
func (h *hashMap) nodes() <-chan node {
	ch := make(chan node)

	go func() {
		for _, b := range h.buckets {
			for i := 0; i < b.count; i++ {
				ch <- b.Nodes[i]
			}
		}
		close(ch)
	}()

	return ch
}

// Get index of bucket key belongs to.
func (h *hashMap) getIndex(key string) uint32 {
	return FNV1a_32([]byte(key)) % h.Size
}
<|endoftext|>


<commit_before>// Shim for the Host Compute Service (HSC) to manage Windows Server
// containers and Hyper-V containers.

package hcsshim

import (
	"fmt"
	"syscall"
	"unsafe"

	"github.com/Sirupsen/logrus"
)

//go:generate go run mksyscall_windows.go -output zhcsshim.go hcsshim.go

//sys coTaskMemFree(buffer unsafe.Pointer) = ole32.CoTaskMemFree
//sys SetCurrentThreadCompartmentId(compartmentId uint32) (hr error) = iphlpapi.SetCurrentThreadCompartmentId

//sys activateLayer(info *driverInfo, id string) (hr error) = vmcompute.ActivateLayer?
//sys copyLayer(info *driverInfo, srcId string, dstId string, descriptors []WC_LAYER_DESCRIPTOR) (hr error) = vmcompute.CopyLayer?
//sys createLayer(info *driverInfo, id string, parent string) (hr error) = vmcompute.CreateLayer?
//sys createSandboxLayer(info *driverInfo, id string, parent string, descriptors []WC_LAYER_DESCRIPTOR) (hr error) = vmcompute.CreateSandboxLayer?
//sys expandSandboxSize(info *driverInfo, id string, size uint64) (hr error) = vmcompute.ExpandSandboxSize?
//sys deactivateLayer(info *driverInfo, id string) (hr error) = vmcompute.DeactivateLayer?
//sys destroyLayer(info *driverInfo, id string) (hr error) = vmcompute.DestroyLayer?
//sys exportLayer(info *driverInfo, id string, path string, descriptors []WC_LAYER_DESCRIPTOR) (hr error) = vmcompute.ExportLayer?
//sys getLayerMountPath(info *driverInfo, id string, length *uintptr, buffer *uint16) (hr error) = vmcompute.GetLayerMountPath?
//sys getBaseImages(buffer **uint16) (hr error) = vmcompute.GetBaseImages?
//sys importLayer(info *driverInfo, id string, path string, descriptors []WC_LAYER_DESCRIPTOR) (hr error) = vmcompute.ImportLayer?
//sys layerExists(info *driverInfo, id string, exists *uint32) (hr error) = vmcompute.LayerExists?
//sys nameToGuid(name string, guid *GUID) (hr error) = vmcompute.NameToGuid?
//sys prepareLayer(info *driverInfo, id string, descriptors []WC_LAYER_DESCRIPTOR) (hr error) = vmcompute.PrepareLayer?
//sys unprepareLayer(info *driverInfo, id string) (hr error) = vmcompute.UnprepareLayer?
//sys processBaseImage(path string) (hr error) = vmcompute.ProcessBaseImage?
//sys processUtilityImage(path string) (hr error) = vmcompute.ProcessUtilityImage?

//sys importLayerBegin(info *driverInfo, id string, descriptors []WC_LAYER_DESCRIPTOR, context *uintptr) (hr error) = vmcompute.ImportLayerBegin?
//sys importLayerNext(context uintptr, fileName string, fileInfo *winio.FileBasicInfo) (hr error) = vmcompute.ImportLayerNext?
//sys importLayerWrite(context uintptr, buffer []byte) (hr error) = vmcompute.ImportLayerWrite?
//sys importLayerEnd(context uintptr) (hr error) = vmcompute.ImportLayerEnd?

//sys exportLayerBegin(info *driverInfo, id string, descriptors []WC_LAYER_DESCRIPTOR, context *uintptr) (hr error) = vmcompute.ExportLayerBegin?
//sys exportLayerNext(context uintptr, fileName **uint16, fileInfo *winio.FileBasicInfo, fileSize *int64, deleted *uint32) (hr error) = vmcompute.ExportLayerNext?
//sys exportLayerRead(context uintptr, buffer []byte, bytesRead *uint32) (hr error) = vmcompute.ExportLayerRead?
//sys exportLayerEnd(context uintptr) (hr error) = vmcompute.ExportLayerEnd?

//sys hcsEnumerateComputeSystems(query string, computeSystems **uint16, result **uint16) (hr error) = vmcompute.HcsEnumerateComputeSystems?
//sys hcsCreateComputeSystem(id string, configuration string, identity syscall.Handle, computeSystem *hcsSystem, result **uint16) (hr error) = vmcompute.HcsCreateComputeSystem?
//sys hcsOpenComputeSystem(id string, computeSystem *hcsSystem, result **uint16) (hr error) = vmcompute.HcsOpenComputeSystem?
//sys hcsCloseComputeSystem(computeSystem hcsSystem) (hr error) = vmcompute.HcsCloseComputeSystem?
//sys hcsStartComputeSystem(computeSystem hcsSystem, options string, result **uint16) (hr error) = vmcompute.HcsStartComputeSystem?
//sys hcsShutdownComputeSystem(computeSystem hcsSystem, options string, result **uint16) (hr error) = vmcompute.HcsShutdownComputeSystem?
//sys hcsTerminateComputeSystem(computeSystem hcsSystem, options string, result **uint16) (hr error) = vmcompute.HcsTerminateComputeSystem?
//sys hcsPauseComputeSystem(computeSystem hcsSystem, options string, result **uint16) (hr error) = vmcompute.HcsPauseComputeSystem?
//sys hcsResumeComputeSystem(computeSystem hcsSystem, options string, result **uint16) (hr error) = vmcompute.HcsResumeComputeSystem?
//sys hcsGetComputeSystemProperties(computeSystem hcsSystem, propertyQuery string, properties **uint16, result **uint16) (hr error) = vmcompute.HcsGetComputeSystemProperties?
//sys hcsModifyComputeSystem(computeSystem hcsSystem, configuration string, result **uint16) (hr error) = vmcompute.HcsModifyComputeSystem?
//sys hcsRegisterComputeSystemCallback(computeSystem hcsSystem, callback uintptr, context uintptr, callbackHandle *hcsCallback) (hr error) = vmcompute.HcsRegisterComputeSystemCallback?
//sys hcsUnregisterComputeSystemCallback(callbackHandle hcsCallback) (hr error) = vmcompute.HcsUnregisterComputeSystemCallback?

//sys hcsCreateProcess(computeSystem hcsSystem, processParameters string, processInformation *hcsProcessInformation, process *hcsProcess, result **uint16) (hr error) = vmcompute.HcsCreateProcess?
//sys hcsOpenProcess(computeSystem hcsSystem, pid uint32, process *hcsProcess, result **uint16) (hr error) = vmcompute.HcsOpenProcess?
//sys hcsCloseProcess(process hcsProcess) (hr error) = vmcompute.HcsCloseProcess?
//sys hcsTerminateProcess(process hcsProcess, result **uint16) (hr error) = vmcompute.HcsTerminateProcess?
//sys hcsGetProcessInfo(process hcsProcess, processInformation *hcsProcessInformation, result **uint16) (hr error) = vmcompute.HcsGetProcessInfo?
//sys hcsGetProcessProperties(process hcsProcess, processProperties **uint16, result **uint16) (hr error) = vmcompute.HcsGetProcessProperties?
//sys hcsModifyProcess(process hcsProcess, settings string, result **uint16) (hr error) = vmcompute.HcsModifyProcess?
//sys hcsGetServiceProperties(propertyQuery string, properties **uint16, result **uint16) (hr error) = vmcompute.HcsGetServiceProperties?
//sys hcsRegisterProcessCallback(process hcsProcess, callback uintptr, context uintptr, callbackHandle *hcsCallback) (hr error) = vmcompute.HcsRegisterProcessCallback?
//sys hcsUnregisterProcessCallback(callbackHandle hcsCallback) (hr error) = vmcompute.HcsUnregisterProcessCallback?

//sys hcsModifyServiceSettings(settings string, result **uint16) (hr error) = vmcompute.HcsModifyServiceSettings?

//sys _hnsCall(method string, path string, object string, response **uint16) (hr error) = vmcompute.HNSCall?

const (
	// Specific user-visible exit codes
	WaitErrExecFailed = 32767

	ERROR_GEN_FAILURE          = syscall.Errno(31)
	ERROR_SHUTDOWN_IN_PROGRESS = syscall.Errno(1115)
	WSAEINVAL                  = syscall.Errno(10022)

	// Timeout on wait calls
	TimeoutInfinite = 0xFFFFFFFF
)

type HcsError struct {
	title string
	rest  string
	Err   error
}

type hcsSystem syscall.Handle
type hcsProcess syscall.Handle
type hcsCallback syscall.Handle

type hcsProcessInformation struct {
	ProcessId uint32
	Reserved  uint32
	StdInput  syscall.Handle
	StdOutput syscall.Handle
	StdError  syscall.Handle
}

func makeError(err error, title, rest string) error {
	// Pass through DLL errors directly since they do not originate from HCS.
	if _, ok := err.(*syscall.DLLError); ok {
		return err
	}
	return &HcsError{title, rest, err}
}

func makeErrorf(err error, title, format string, a ...interface{}) error {
	return makeError(err, title, fmt.Sprintf(format, a...))
}

func win32FromError(err error) uint32 {
	if herr, ok := err.(*HcsError); ok {
		return win32FromError(herr.Err)
	}
	if code, ok := err.(syscall.Errno); ok {
		return uint32(code)
	}
	return uint32(ERROR_GEN_FAILURE)
}

func win32FromHresult(hr uintptr) uintptr {
	if hr&0x1fff0000 == 0x00070000 {
		return hr & 0xffff
	}
	return hr
}

func (e *HcsError) Error() string {
	s := e.title
	if len(s) > 0 && s[len(s)-1] != ' ' {
		s += " "
	}
	s += fmt.Sprintf("failed in Win32: %s (0x%x)", e.Err, win32FromError(e.Err))
	if e.rest != "" {
		if e.rest[0] != ' ' {
			s += " "
		}
		s += e.rest
	}
	return s
}

func convertAndFreeCoTaskMemString(buffer *uint16) string {
	str := syscall.UTF16ToString((*[1 << 30]uint16)(unsafe.Pointer(buffer))[:])
	coTaskMemFree(unsafe.Pointer(buffer))
	return str
}

func convertAndFreeCoTaskMemBytes(buffer *uint16) []byte {
	return []byte(convertAndFreeCoTaskMemString(buffer))
}

func processHcsResult(err error, resultp *uint16) error {
	if resultp != nil {
		result := convertAndFreeCoTaskMemString(resultp)
		logrus.Debugf("Result: %s", result)
	}
	return err
}
<commit_msg>Update hcsshim.go<commit_after>// Shim for the Host Compute Service (HCS) to manage Windows Server
// containers and Hyper-V containers.

package hcsshim

import (
	"fmt"
	"syscall"
	"unsafe"

	"github.com/Sirupsen/logrus"
)

//go:generate go run mksyscall_windows.go -output zhcsshim.go hcsshim.go

//sys coTaskMemFree(buffer unsafe.Pointer) = ole32.CoTaskMemFree
//sys SetCurrentThreadCompartmentId(compartmentId uint32) (hr error) = iphlpapi.SetCurrentThreadCompartmentId

//sys activateLayer(info *driverInfo, id string) (hr error) = vmcompute.ActivateLayer?
//sys copyLayer(info *driverInfo, srcId string, dstId string, descriptors []WC_LAYER_DESCRIPTOR) (hr error) = vmcompute.CopyLayer?
//sys createLayer(info *driverInfo, id string, parent string) (hr error) = vmcompute.CreateLayer?
//sys createSandboxLayer(info *driverInfo, id string, parent string, descriptors []WC_LAYER_DESCRIPTOR) (hr error) = vmcompute.CreateSandboxLayer?
//sys expandSandboxSize(info *driverInfo, id string, size uint64) (hr error) = vmcompute.ExpandSandboxSize?
//sys deactivateLayer(info *driverInfo, id string) (hr error) = vmcompute.DeactivateLayer?
//sys destroyLayer(info *driverInfo, id string) (hr error) = vmcompute.DestroyLayer?
//sys exportLayer(info *driverInfo, id string, path string, descriptors []WC_LAYER_DESCRIPTOR) (hr error) = vmcompute.ExportLayer?
//sys getLayerMountPath(info *driverInfo, id string, length *uintptr, buffer *uint16) (hr error) = vmcompute.GetLayerMountPath?
//sys getBaseImages(buffer **uint16) (hr error) = vmcompute.GetBaseImages?
//sys importLayer(info *driverInfo, id string, path string, descriptors []WC_LAYER_DESCRIPTOR) (hr error) = vmcompute.ImportLayer?
//sys layerExists(info *driverInfo, id string, exists *uint32) (hr error) = vmcompute.LayerExists?
//sys nameToGuid(name string, guid *GUID) (hr error) = vmcompute.NameToGuid?
//sys prepareLayer(info *driverInfo, id string, descriptors []WC_LAYER_DESCRIPTOR) (hr error) = vmcompute.PrepareLayer?
//sys unprepareLayer(info *driverInfo, id string) (hr error) = vmcompute.UnprepareLayer?
//sys processBaseImage(path string) (hr error) = vmcompute.ProcessBaseImage?
//sys processUtilityImage(path string) (hr error) = vmcompute.ProcessUtilityImage?

//sys importLayerBegin(info *driverInfo, id string, descriptors []WC_LAYER_DESCRIPTOR, context *uintptr) (hr error) = vmcompute.ImportLayerBegin?
//sys importLayerNext(context uintptr, fileName string, fileInfo *winio.FileBasicInfo) (hr error) = vmcompute.ImportLayerNext?
//sys importLayerWrite(context uintptr, buffer []byte) (hr error) = vmcompute.ImportLayerWrite?
//sys importLayerEnd(context uintptr) (hr error) = vmcompute.ImportLayerEnd?

//sys exportLayerBegin(info *driverInfo, id string, descriptors []WC_LAYER_DESCRIPTOR, context *uintptr) (hr error) = vmcompute.ExportLayerBegin?
//sys exportLayerNext(context uintptr, fileName **uint16, fileInfo *winio.FileBasicInfo, fileSize *int64, deleted *uint32) (hr error) = vmcompute.ExportLayerNext?
//sys exportLayerRead(context uintptr, buffer []byte, bytesRead *uint32) (hr error) = vmcompute.ExportLayerRead?
//sys exportLayerEnd(context uintptr) (hr error) = vmcompute.ExportLayerEnd?

//sys hcsEnumerateComputeSystems(query string, computeSystems **uint16, result **uint16) (hr error) = vmcompute.HcsEnumerateComputeSystems?
//sys hcsCreateComputeSystem(id string, configuration string, identity syscall.Handle, computeSystem *hcsSystem, result **uint16) (hr error) = vmcompute.HcsCreateComputeSystem?
//sys hcsOpenComputeSystem(id string, computeSystem *hcsSystem, result **uint16) (hr error) = vmcompute.HcsOpenComputeSystem?
//sys hcsCloseComputeSystem(computeSystem hcsSystem) (hr error) = vmcompute.HcsCloseComputeSystem?
//sys hcsStartComputeSystem(computeSystem hcsSystem, options string, result **uint16) (hr error) = vmcompute.HcsStartComputeSystem?
//sys hcsShutdownComputeSystem(computeSystem hcsSystem, options string, result **uint16) (hr error) = vmcompute.HcsShutdownComputeSystem?
//sys hcsTerminateComputeSystem(computeSystem hcsSystem, options string, result **uint16) (hr error) = vmcompute.HcsTerminateComputeSystem?
//sys hcsPauseComputeSystem(computeSystem hcsSystem, options string, result **uint16) (hr error) = vmcompute.HcsPauseComputeSystem?
//sys hcsResumeComputeSystem(computeSystem hcsSystem, options string, result **uint16) (hr error) = vmcompute.HcsResumeComputeSystem?
//sys hcsGetComputeSystemProperties(computeSystem hcsSystem, propertyQuery string, properties **uint16, result **uint16) (hr error) = vmcompute.HcsGetComputeSystemProperties?
//sys hcsModifyComputeSystem(computeSystem hcsSystem, configuration string, result **uint16) (hr error) = vmcompute.HcsModifyComputeSystem?
//sys hcsRegisterComputeSystemCallback(computeSystem hcsSystem, callback uintptr, context uintptr, callbackHandle *hcsCallback) (hr error) = vmcompute.HcsRegisterComputeSystemCallback?
//sys hcsUnregisterComputeSystemCallback(callbackHandle hcsCallback) (hr error) = vmcompute.HcsUnregisterComputeSystemCallback?

//sys hcsCreateProcess(computeSystem hcsSystem, processParameters string, processInformation *hcsProcessInformation, process *hcsProcess, result **uint16) (hr error) = vmcompute.HcsCreateProcess?
//sys hcsOpenProcess(computeSystem hcsSystem, pid uint32, process *hcsProcess, result **uint16) (hr error) = vmcompute.HcsOpenProcess?
//sys hcsCloseProcess(process hcsProcess) (hr error) = vmcompute.HcsCloseProcess?
//sys hcsTerminateProcess(process hcsProcess, result **uint16) (hr error) = vmcompute.HcsTerminateProcess?
//sys hcsGetProcessInfo(process hcsProcess, processInformation *hcsProcessInformation, result **uint16) (hr error) = vmcompute.HcsGetProcessInfo?
//sys hcsGetProcessProperties(process hcsProcess, processProperties **uint16, result **uint16) (hr error) = vmcompute.HcsGetProcessProperties?
//sys hcsModifyProcess(process hcsProcess, settings string, result **uint16) (hr error) = vmcompute.HcsModifyProcess?
//sys hcsGetServiceProperties(propertyQuery string, properties **uint16, result **uint16) (hr error) = vmcompute.HcsGetServiceProperties?
//sys hcsRegisterProcessCallback(process hcsProcess, callback uintptr, context uintptr, callbackHandle *hcsCallback) (hr error) = vmcompute.HcsRegisterProcessCallback?
//sys hcsUnregisterProcessCallback(callbackHandle hcsCallback) (hr error) = vmcompute.HcsUnregisterProcessCallback?

//sys hcsModifyServiceSettings(settings string, result **uint16) (hr error) = vmcompute.HcsModifyServiceSettings?

//sys _hnsCall(method string, path string, object string, response **uint16) (hr error) = vmcompute.HNSCall?

const (
	// Specific user-visible exit codes
	WaitErrExecFailed = 32767

	ERROR_GEN_FAILURE          = syscall.Errno(31)
	ERROR_SHUTDOWN_IN_PROGRESS = syscall.Errno(1115)
	WSAEINVAL                  = syscall.Errno(10022)

	// Timeout on wait calls
	TimeoutInfinite = 0xFFFFFFFF
)

type HcsError struct {
	title string
	rest  string
	Err   error
}

type hcsSystem syscall.Handle
type hcsProcess syscall.Handle
type hcsCallback syscall.Handle

type hcsProcessInformation struct {
	ProcessId uint32
	Reserved  uint32
	StdInput  syscall.Handle
	StdOutput syscall.Handle
	StdError  syscall.Handle
}

func makeError(err error, title, rest string) error {
	// Pass through DLL errors directly since they do not originate from HCS.
	if _, ok := err.(*syscall.DLLError); ok {
		return err
	}
	return &HcsError{title, rest, err}
}

func makeErrorf(err error, title, format string, a ...interface{}) error {
	return makeError(err, title, fmt.Sprintf(format, a...))
}

func win32FromError(err error) uint32 {
	if herr, ok := err.(*HcsError); ok {
		return win32FromError(herr.Err)
	}
	if code, ok := err.(syscall.Errno); ok {
		return uint32(code)
	}
	return uint32(ERROR_GEN_FAILURE)
}

func win32FromHresult(hr uintptr) uintptr {
	if hr&0x1fff0000 == 0x00070000 {
		return hr & 0xffff
	}
	return hr
}

func (e *HcsError) Error() string {
	s := e.title
	if len(s) > 0 && s[len(s)-1] != ' ' {
		s += " "
	}
	s += fmt.Sprintf("failed in Win32: %s (0x%x)", e.Err, win32FromError(e.Err))
	if e.rest != "" {
		if e.rest[0] != ' ' {
			s += " "
		}
		s += e.rest
	}
	return s
}

func convertAndFreeCoTaskMemString(buffer *uint16) string {
	str := syscall.UTF16ToString((*[1 << 30]uint16)(unsafe.Pointer(buffer))[:])
	coTaskMemFree(unsafe.Pointer(buffer))
	return str
}

func convertAndFreeCoTaskMemBytes(buffer *uint16) []byte {
	return []byte(convertAndFreeCoTaskMemString(buffer))
}

func processHcsResult(err error, resultp *uint16) error {
	if resultp != nil {
		result := convertAndFreeCoTaskMemString(resultp)
		logrus.Debugf("Result: %s", result)
	}
	return err
}
<|endoftext|>


<commit_before>package gocql

import "reflect"

type rowData struct {
	Columns []string
	Values []interface{}
}

func (t *TypeInfo) New() interface{} {
	return reflect.New(goType(t)).Interface()
}

func goType(t *TypeInfo) reflect.Type {
	switch t.Type {
	case TypeVarchar, TypeAscii:
		return reflect.TypeOf(*new(string))
	case TypeBigInt, TypeCounter, TypeTimestamp:
		return reflect.TypeOf(*new(int64))
	case TypeBlob:
		return reflect.TypeOf(*new([]byte))
	case TypeBoolean:
		return reflect.TypeOf(*new(bool))
	case TypeFloat:
		return reflect.TypeOf(*new(float32))
	case TypeDouble:
		return reflect.TypeOf(*new(float64))
	case TypeInt:
		return reflect.TypeOf(*new(int))
	case TypeUUID, TypeTimeUUID:
		return reflect.TypeOf(*new(UUID))
	case TypeList, TypeSet:
		return reflect.SliceOf(goType(t.Elem))
	case TypeMap:
		return reflect.MapOf(goType(t.Key), goType(t.Elem))
	default:
		return nil
	}
}

func dereference(i interface{}) interface{} {
	return reflect.Indirect(reflect.ValueOf(i)).Interface()
}

func (r *rowData) Map(m map[string]interface{}) {
	for i, column := range r.Columns {
		m[column] = dereference(r.Values[i])
	}
}

func (iter *Iter) rowData() (rowData, error) {
	if iter.err != nil {
		return rowData{}, iter.err
	}
	columns := make([]string, 0)
	values := make([]interface{}, 0)
	for _, column := range iter.Columns() {
		val := column.TypeInfo.New()
		columns = append(columns, column.Name)
		values = append(values, val)
	}
	rowData := rowData{
		Columns: columns,
		Values: values,
	}
	return rowData, nil
}

// SliceMap is a helper function to make the API easier to use
// returns the data from the query in the form of []map[string]interface{}
func (iter *Iter) SliceMap() ([]map[string]interface{}, error) {
	if iter.err != nil {
		return nil, iter.err
	}

	// Not checking for the error because we just did
	rowData, _ := iter.rowData()
	dataToReturn := make([]map[string]interface{}, 0)
	for iter.Scan(rowData.Values...) {
		m := make(map[string]interface{})
		rowData.Map(m)
		dataToReturn = append(dataToReturn, m)
	}
	if iter.err != nil {
		return nil, iter.err
	}
	return dataToReturn, nil
}

func (iter *Iter) MapScan(m map[string]interface{}) bool {
	if iter.err != nil {
		return false
	}

	// Not checking for the error because we just did
	rowData, _ := iter.rowData()

	if iter.Scan(rowData.Values...) {
		rowData.Map(m)
		return true
	}
	return false
}
<commit_msg>changed the Map() function to rowMap()<commit_after>package gocql

import "reflect"

type rowData struct {
	Columns []string
	Values []interface{}
}

func (t *TypeInfo) New() interface{} {
	return reflect.New(goType(t)).Interface()
}

func goType(t *TypeInfo) reflect.Type {
	switch t.Type {
	case TypeVarchar, TypeAscii:
		return reflect.TypeOf(*new(string))
	case TypeBigInt, TypeCounter, TypeTimestamp:
		return reflect.TypeOf(*new(int64))
	case TypeBlob:
		return reflect.TypeOf(*new([]byte))
	case TypeBoolean:
		return reflect.TypeOf(*new(bool))
	case TypeFloat:
		return reflect.TypeOf(*new(float32))
	case TypeDouble:
		return reflect.TypeOf(*new(float64))
	case TypeInt:
		return reflect.TypeOf(*new(int))
	case TypeUUID, TypeTimeUUID:
		return reflect.TypeOf(*new(UUID))
	case TypeList, TypeSet:
		return reflect.SliceOf(goType(t.Elem))
	case TypeMap:
		return reflect.MapOf(goType(t.Key), goType(t.Elem))
	default:
		return nil
	}
}

func dereference(i interface{}) interface{} {
	return reflect.Indirect(reflect.ValueOf(i)).Interface()
}

func (r *rowData) rowMap(m map[string]interface{}) {
	for i, column := range r.Columns {
		m[column] = dereference(r.Values[i])
	}
}

func (iter *Iter) rowData() (rowData, error) {
	if iter.err != nil {
		return rowData{}, iter.err
	}
	columns := make([]string, 0)
	values := make([]interface{}, 0)
	for _, column := range iter.Columns() {
		val := column.TypeInfo.New()
		columns = append(columns, column.Name)
		values = append(values, val)
	}
	rowData := rowData{
		Columns: columns,
		Values: values,
	}
	return rowData, nil
}

// SliceMap is a helper function to make the API easier to use
// returns the data from the query in the form of []map[string]interface{}
func (iter *Iter) SliceMap() ([]map[string]interface{}, error) {
	if iter.err != nil {
		return nil, iter.err
	}

	// Not checking for the error because we just did
	rowData, _ := iter.rowData()
	dataToReturn := make([]map[string]interface{}, 0)
	for iter.Scan(rowData.Values...) {
		m := make(map[string]interface{})
		rowData.rowMap(m)
		dataToReturn = append(dataToReturn, m)
	}
	if iter.err != nil {
		return nil, iter.err
	}
	return dataToReturn, nil
}

func (iter *Iter) MapScan(m map[string]interface{}) bool {
	if iter.err != nil {
		return false
	}

	// Not checking for the error because we just did
	rowData, _ := iter.rowData()

	if iter.Scan(rowData.Values...) {
		rowData.rowMap(m)
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/rand"
	"fmt"
	"net/http"
	"testing"
	"time"
)

// HTTP ServeMux with an updateable handler so that tests can pass their own
// anonymous functions in to handle requests.
type CDNServeMux struct {
	Port    int
	handler func(w http.ResponseWriter, r *http.Request)
}

func (s *CDNServeMux) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Method == "HEAD" && r.URL.Path == "/" {
		w.Header().Set("PING", "PONG")
		return
	}

	s.handler(w, r)
}

func (s *CDNServeMux) SwitchHandler(h func(w http.ResponseWriter, r *http.Request)) {
	s.handler = h
}

// Start a new server and return the CDNServeMux used.
func StartServer(port int) *CDNServeMux {
	handler := func(w http.ResponseWriter, r *http.Request) {}
	mux := &CDNServeMux{port, handler}
	addr := fmt.Sprintf(":%d", port)

	go func() {
		err := http.ListenAndServe(addr, mux)
		if err != nil {
			panic(err)
		}
	}()

	return mux
}

// Return a v4 (random) UUID string.
// This might not be strictly RFC4122 compliant, but it will do. Credit:
// https://groups.google.com/d/msg/golang-nuts/Rn13T6BZpgE/dBaYVJ4hB5gJ
func NewUUID() string {
	bs := make([]byte, 16)
	rand.Read(bs)
	bs[6] = (bs[6] & 0x0f) | 0x40
	bs[8] = (bs[8] & 0x3f) | 0x80

	return fmt.Sprintf("%x-%x-%x-%x-%x", bs[0:4], bs[4:6], bs[6:8], bs[8:10], bs[10:])
}

// CDNServeMux helper should be ready to serve requests when test suite starts
// and then serve custom handlers each with their own status code.
func testHelpersCDNServeMuxHandlers(t *testing.T, mux *CDNServeMux) {
	url := fmt.Sprintf("http://localhost:%d/foo", mux.Port)
	req, _ := http.NewRequest("GET", url, nil)

	resp, err := client.RoundTrip(req)
	if err != nil {
		t.Fatal(err)
	}
	if resp.StatusCode != 200 {
		t.Error("First request to default handler failed")
	}

	for _, statusCode := range []int{301, 302, 403, 404} {
		mux.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
			w.WriteHeader(statusCode)
		})

		resp, err := client.RoundTrip(req)
		if err != nil {
			t.Fatal(err)
		}
		if resp.StatusCode != statusCode {
			t.Errorf("SwitchHandler didn't work. Got %d, expected %d", resp.StatusCode, statusCode)
		}
	}
}

// CDNServeMux should always respond to HEAD requests in order for the CDN to
// determine the health of our origin.
func testHelpersCDNServeMuxProbes(t *testing.T, mux *CDNServeMux) {
	mux.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		t.Error("HEAD request incorrectly served by CDNServeMux.handler")
	})

	url := fmt.Sprintf("http://localhost:%d/", mux.Port)
	req, _ := http.NewRequest("HEAD", url, nil)

	resp, err := client.RoundTrip(req)
	if err != nil {
		t.Fatal(err)
	}

	if resp.StatusCode != 200 || resp.Header.Get("PING") != "PONG" {
		t.Error("HEAD request for '/' served incorrectly")
	}
}

func confirmOriginIsEnabled(mux *CDNServeMux, edgeHost string) error {
	mux.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(200)
	})

	timeBetweenAttempts, _ := time.ParseDuration("2s")
	waitForCdnProbeToPropogate, _ := time.ParseDuration("5s")

	maxRetries := 20
	var sourceUrl string

	for try := 0; try <= maxRetries; try++ {
		uuid := NewUUID()
		sourceUrl = fmt.Sprintf("https://%s/confirm-cdn-ok-%s", edgeHost, uuid)
		req, _ := http.NewRequest("GET", sourceUrl, nil)
		resp, err := client.RoundTrip(req)
		if err != nil {
			return err
		}
		if resp.StatusCode == 200 {
			if try != 0 {
				time.Sleep(waitForCdnProbeToPropogate)
			}
			return nil // all is well!
		}
		time.Sleep(timeBetweenAttempts)
	}
	return fmt.Errorf("CDN still not available after %n attempts", maxRetries)

}
<commit_msg>Use 'GET /?cacheBuster=%s' for origin check req<commit_after>package main

import (
	"crypto/rand"
	"fmt"
	"net/http"
	"testing"
	"time"
)

// HTTP ServeMux with an updateable handler so that tests can pass their own
// anonymous functions in to handle requests.
type CDNServeMux struct {
	Port    int
	handler func(w http.ResponseWriter, r *http.Request)
}

func (s *CDNServeMux) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if r.Method == "HEAD" && r.URL.Path == "/" {
		w.Header().Set("PING", "PONG")
		return
	}

	s.handler(w, r)
}

func (s *CDNServeMux) SwitchHandler(h func(w http.ResponseWriter, r *http.Request)) {
	s.handler = h
}

// Start a new server and return the CDNServeMux used.
func StartServer(port int) *CDNServeMux {
	handler := func(w http.ResponseWriter, r *http.Request) {}
	mux := &CDNServeMux{port, handler}
	addr := fmt.Sprintf(":%d", port)

	go func() {
		err := http.ListenAndServe(addr, mux)
		if err != nil {
			panic(err)
		}
	}()

	return mux
}

// Return a v4 (random) UUID string.
// This might not be strictly RFC4122 compliant, but it will do. Credit:
// https://groups.google.com/d/msg/golang-nuts/Rn13T6BZpgE/dBaYVJ4hB5gJ
func NewUUID() string {
	bs := make([]byte, 16)
	rand.Read(bs)
	bs[6] = (bs[6] & 0x0f) | 0x40
	bs[8] = (bs[8] & 0x3f) | 0x80

	return fmt.Sprintf("%x-%x-%x-%x-%x", bs[0:4], bs[4:6], bs[6:8], bs[8:10], bs[10:])
}

// CDNServeMux helper should be ready to serve requests when test suite starts
// and then serve custom handlers each with their own status code.
func testHelpersCDNServeMuxHandlers(t *testing.T, mux *CDNServeMux) {
	url := fmt.Sprintf("http://localhost:%d/foo", mux.Port)
	req, _ := http.NewRequest("GET", url, nil)

	resp, err := client.RoundTrip(req)
	if err != nil {
		t.Fatal(err)
	}
	if resp.StatusCode != 200 {
		t.Error("First request to default handler failed")
	}

	for _, statusCode := range []int{301, 302, 403, 404} {
		mux.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
			w.WriteHeader(statusCode)
		})

		resp, err := client.RoundTrip(req)
		if err != nil {
			t.Fatal(err)
		}
		if resp.StatusCode != statusCode {
			t.Errorf("SwitchHandler didn't work. Got %d, expected %d", resp.StatusCode, statusCode)
		}
	}
}

// CDNServeMux should always respond to HEAD requests in order for the CDN to
// determine the health of our origin.
func testHelpersCDNServeMuxProbes(t *testing.T, mux *CDNServeMux) {
	mux.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		t.Error("HEAD request incorrectly served by CDNServeMux.handler")
	})

	url := fmt.Sprintf("http://localhost:%d/", mux.Port)
	req, _ := http.NewRequest("HEAD", url, nil)

	resp, err := client.RoundTrip(req)
	if err != nil {
		t.Fatal(err)
	}

	if resp.StatusCode != 200 || resp.Header.Get("PING") != "PONG" {
		t.Error("HEAD request for '/' served incorrectly")
	}
}

func confirmOriginIsEnabled(mux *CDNServeMux, edgeHost string) error {
	mux.SwitchHandler(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(200)
	})

	timeBetweenAttempts, _ := time.ParseDuration("2s")
	waitForCdnProbeToPropogate, _ := time.ParseDuration("5s")

	maxRetries := 20
	var sourceUrl string

	for try := 0; try <= maxRetries; try++ {
		uuid := NewUUID()
		sourceUrl = fmt.Sprintf("https://%s/?cacheBuster=%s", edgeHost, uuid)
		req, _ := http.NewRequest("GET", sourceUrl, nil)
		resp, err := client.RoundTrip(req)
		if err != nil {
			return err
		}
		if resp.StatusCode == 200 {
			if try != 0 {
				time.Sleep(waitForCdnProbeToPropogate)
			}
			return nil // all is well!
		}
		time.Sleep(timeBetweenAttempts)
	}
	return fmt.Errorf("CDN still not available after %n attempts", maxRetries)

}
<|endoftext|>


<commit_before>// Copyright (c) 2014, David Kitchen <david@buro9.com>
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// * Redistributions of source code must retain the above copyright notice, this
//   list of conditions and the following disclaimer.
//
// * Redistributions in binary form must reproduce the above copyright notice,
//   this list of conditions and the following disclaimer in the documentation
//   and/or other materials provided with the distribution.
//
// * Neither the name of the organisation (Microcosm) nor the names of its
//   contributors may be used to endorse or promote products derived from
//   this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

package bluemonday

import (
	"encoding/base64"
	"net/url"
	"regexp"
)

// A selection of regular expressions that can be used as .Matching() rules on
// HTML attributes.
var (
	// CellAlign handles the `align` attribute
	// https://developer.mozilla.org/en-US/docs/Web/HTML/Element/td#attr-align
	CellAlign = regexp.MustCompile(`(?i)^(center|justify|left|right|char)$`)

	// CellVerticalAlign handles the `valign` attribute
	// https://developer.mozilla.org/en-US/docs/Web/HTML/Element/td#attr-valign
	CellVerticalAlign = regexp.MustCompile(`(?i)^(baseline|bottom|middle|top)$`)

	// Direction handles the `dir` attribute
	// https://developer.mozilla.org/en-US/docs/Web/HTML/Element/bdo#attr-dir
	Direction = regexp.MustCompile(`(?i)^(rtl|ltr)$`)

	// ImageAlign handles the `align` attribute on the `image` tag
	// http://www.w3.org/MarkUp/Test/Img/imgtest.html
	ImageAlign = regexp.MustCompile(
		`(?i)^(left|right|top|texttop|middle|absmiddle|baseline|bottom|absbottom)$`,
	)

	// Integer describes whole positive integers (including 0) used in places
	// like td.colspan
	// https://developer.mozilla.org/en-US/docs/Web/HTML/Element/td#attr-colspan
	Integer = regexp.MustCompile(`^[0-9]+$`)

	// ISO8601 according to the W3 group is only a subset of the ISO8601
	// standard: http://www.w3.org/TR/NOTE-datetime
	//
	// Used in places like time.datetime
	// https://developer.mozilla.org/en-US/docs/Web/HTML/Element/time#attr-datetime
	//
	// Matches patterns:
	//  Year:
	//     YYYY (eg 1997)
	//  Year and month:
	//     YYYY-MM (eg 1997-07)
	//  Complete date:
	//     YYYY-MM-DD (eg 1997-07-16)
	//  Complete date plus hours and minutes:
	//     YYYY-MM-DDThh:mmTZD (eg 1997-07-16T19:20+01:00)
	//  Complete date plus hours, minutes and seconds:
	//     YYYY-MM-DDThh:mm:ssTZD (eg 1997-07-16T19:20:30+01:00)
	//  Complete date plus hours, minutes, seconds and a decimal fraction of a
	//  second
	//      YYYY-MM-DDThh:mm:ss.sTZD (eg 1997-07-16T19:20:30.45+01:00)
	ISO8601 = regexp.MustCompile(
		`^[0-9]{4}(-[0-9]{2}(-[0-9]{2}([ T][0-9]{2}(:[0-9]{2}){1,2}(.[0-9]{1,6})` +
			`?Z?([\+-][0-9]{2}:[0-9]{2})?)?)?)?$`,
	)

	// ListType encapsulates the common value as well as the latest spec
	// values for lists
	// https://developer.mozilla.org/en-US/docs/Web/HTML/Element/ol#attr-type
	ListType = regexp.MustCompile(`(?i)^(circle|disc|square|a|A|i|I|1)$`)

	// SpaceSeparatedTokens is used in places like `a.rel` and the common attribute
	// `class` which both contain space delimited lists of data tokens
	// http://www.w3.org/TR/html-markup/datatypes.html#common.data.tokens-def
	// Regexp: \p{L} matches unicode letters, \p{N} matches unicode numbers
	SpaceSeparatedTokens = regexp.MustCompile(`^([\s\p{L}\p{N}_-]+)$`)

	// Number is a double value used on HTML5 meter and progress elements
	// http://www.whatwg.org/specs/web-apps/current-work/multipage/the-button-element.html#the-meter-element
	Number = regexp.MustCompile(`^[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?$`)

	// NumberOrPercent is used predominantly as units of measurement in width
	// and height attributes
	// https://developer.mozilla.org/en-US/docs/Web/HTML/Element/img#attr-height
	NumberOrPercent = regexp.MustCompile(`^[0-9]+[%]?$`)

	// Paragraph of text in an attribute such as *.'title', img.alt, etc
	// https://developer.mozilla.org/en-US/docs/Web/HTML/Global_attributes#attr-title
	// Note that we are not allowing chars that could close tags like '>'
	Paragraph = regexp.MustCompile(`^[\p{L}\p{N}\s\-_',\[\]!\./\\\(\)]*$`)

	// dataURIImagePrefix is used by AllowDataURIImages to define the acceptable
	// prefix of data URIs that contain common web image formats.
	//
	// This is not exported as it's not useful by itself, and only has value
	// within the AllowDataURIImages func
	dataURIImagePrefix = regexp.MustCompile(
		`^image/(gif|jpeg|png|webp);base64,`,
	)
)

// AllowStandardURLs is a convenience function that will enable rel="nofollow"
// on "a", "area" and "link" (if you have allowed those elements) and will
// ensure that the URL values are parseable and either relative or belong to the
// "mailto", "http", or "https" schemes
func (p *Policy) AllowStandardURLs() {
	// URLs must be parseable by net/url.Parse()
	p.RequireParseableURLs(true)

	// !url.IsAbs() is permitted
	p.AllowRelativeURLs(true)

	// Most common URL schemes only
	p.AllowURLSchemes("mailto", "http", "https")

	// For all anchors we will add rel="nofollow" if it does not already exist
	// This applies to "a" "area" "link"
	p.RequireNoFollowOnLinks(true)
}

// AllowStandardAttributes will enable "id", "title" and the language specific
// attributes "dir" and "lang" on all elements that are whitelisted
func (p *Policy) AllowStandardAttributes() {
	// "dir" "lang" are permitted as both language attributes affect charsets
	// and direction of text.
	p.AllowAttrs("dir").Matching(Direction).Globally()
	p.AllowAttrs(
		"lang",
	).Matching(regexp.MustCompile(`[a-zA-Z]{2,20}`)).Globally()

	// "id" is permitted. This is pretty much as some HTML elements require this
	// to work well ("dfn" is an example of a "id" being value)
	// This does create a risk that JavaScript and CSS within your web page
	// might identify the wrong elements. Ensure that you select things
	// accurately
	p.AllowAttrs("id").Matching(
		regexp.MustCompile(`[a-zA-Z0-9\:\-_\.]+`),
	).Globally()

	// "title" is permitted as it improves accessibility.
	p.AllowAttrs("title").Matching(Paragraph).Globally()
}

// AllowStyling presently enables the class attribute globally.
//
// Note: When bluemonday ships a CSS parser and we can safely sanitise that,
// this will also allow sanitized styling of elements via the style attribute.
func (p *Policy) AllowStyling() {

	// "class" is permitted globally
	p.AllowAttrs("class").Matching(SpaceSeparatedTokens).Globally()
}

// AllowImages enables the img element and some popular attributes. It will also
// ensure that URL values are parseable
func (p *Policy) AllowImages() {

	// "img" is permitted
	p.AllowAttrs("align").Matching(ImageAlign).OnElements("img")
	p.AllowAttrs("alt").Matching(Paragraph).OnElements("img")
	p.AllowAttrs("height", "width").Matching(NumberOrPercent).OnElements("img")

	// Standard URLs enabled, which disables data URI images as the data scheme
	// isn't included in the policy
	p.AllowStandardURLs()
	p.AllowAttrs("src").OnElements("img")
}

// AllowDataURIImages permits the use of inline images defined in RFC2397
// http://tools.ietf.org/html/rfc2397
// http://en.wikipedia.org/wiki/Data_URI_scheme
//
// Images must have a mimetype matching:
//   image/gif
//   image/jpeg
//   image/png
//   image/webp
//
// NOTE: There is a potential security risk to allowing data URIs and you should
// only permit them on content you already trust.
// http://palizine.plynt.com/issues/2010Oct/bypass-xss-filters/
// https://capec.mitre.org/data/definitions/244.html
func (p *Policy) AllowDataURIImages() {

	// URLs must be parseable by net/url.Parse()
	p.RequireParseableURLs(true)

	// Supply a function to validate images contained within data URI
	p.AllowURLSchemeWithCustomPolicy(
		"data",
		func(url *url.URL) (allowUrl bool) {
			if url.RawQuery != "" || url.Fragment != "" {
				return false
			}

			matched := dataURIImagePrefix.FindString(url.Opaque)
			if matched == "" {
				return false
			}

			_, err := base64.StdEncoding.DecodeString(url.Opaque[len(matched):])
			if err != nil {
				return false
			}

			return true
		},
	)
}

// AllowLists will enabled ordered and unordered lists, as well as definition
// lists
func (p *Policy) AllowLists() {
	// "ol" "ul" are permitted
	p.AllowAttrs("type").Matching(ListType).OnElements("ol", "ul")

	// "li" is permitted
	p.AllowAttrs("type").Matching(ListType).OnElements("li")
	p.AllowAttrs("value").Matching(Integer).OnElements("li")

	// "dl" "dt" "dd" are permitted
	p.AllowElements("dl", "dt", "dd")
}

// AllowTables will enable a rich set of elements and attributes to describe
// HTML tables
func (p *Policy) AllowTables() {

	// "table" is permitted
	p.AllowAttrs("height", "width").Matching(NumberOrPercent).OnElements("table")
	p.AllowAttrs("summary").Matching(Paragraph).OnElements("table")

	// "caption" is permitted
	p.AllowElements("caption")

	// "col" "colgroup" are permitted
	p.AllowAttrs("align").Matching(CellAlign).OnElements("col", "colgroup")
	p.AllowAttrs("height", "width").Matching(
		NumberOrPercent,
	).OnElements("col", "colgroup")
	p.AllowAttrs("span").Matching(Integer).OnElements("colgroup", "col")
	p.AllowAttrs("valign").Matching(
		CellVerticalAlign,
	).OnElements("col", "colgroup")

	// "thead" "tr" are permitted
	p.AllowAttrs("align").Matching(CellAlign).OnElements("thead", "tr")
	p.AllowAttrs("valign").Matching(CellVerticalAlign).OnElements("thead", "tr")

	// "td" "th" are permitted
	p.AllowAttrs("abbr").Matching(Paragraph).OnElements("td", "th")
	p.AllowAttrs("align").Matching(CellAlign).OnElements("td", "th")
	p.AllowAttrs("colspan", "rowspan").Matching(Integer).OnElements("td", "th")
	p.AllowAttrs("headers").Matching(
		SpaceSeparatedTokens,
	).OnElements("td", "th")
	p.AllowAttrs("height", "width").Matching(
		NumberOrPercent,
	).OnElements("td", "th")
	p.AllowAttrs(
		"scope",
	).Matching(
		regexp.MustCompile(`(?i)(?:row|col)(?:group)?`),
	).OnElements("td", "th")
	p.AllowAttrs("valign").Matching(CellVerticalAlign).OnElements("td", "th")
	p.AllowAttrs("nowrap").Matching(
		regexp.MustCompile(`(?i)|nowrap`),
	).OnElements("td", "th")

	// "tbody" "tfoot"
	p.AllowAttrs("align").Matching(CellAlign).OnElements("tbody", "tfoot")
	p.AllowAttrs("valign").Matching(
		CellVerticalAlign,
	).OnElements("tbody", "tfoot")
}
<commit_msg>Edited comment to be clear that we aren't whitelisting data URI images within AllowImages<commit_after>// Copyright (c) 2014, David Kitchen <david@buro9.com>
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// * Redistributions of source code must retain the above copyright notice, this
//   list of conditions and the following disclaimer.
//
// * Redistributions in binary form must reproduce the above copyright notice,
//   this list of conditions and the following disclaimer in the documentation
//   and/or other materials provided with the distribution.
//
// * Neither the name of the organisation (Microcosm) nor the names of its
//   contributors may be used to endorse or promote products derived from
//   this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

package bluemonday

import (
	"encoding/base64"
	"net/url"
	"regexp"
)

// A selection of regular expressions that can be used as .Matching() rules on
// HTML attributes.
var (
	// CellAlign handles the `align` attribute
	// https://developer.mozilla.org/en-US/docs/Web/HTML/Element/td#attr-align
	CellAlign = regexp.MustCompile(`(?i)^(center|justify|left|right|char)$`)

	// CellVerticalAlign handles the `valign` attribute
	// https://developer.mozilla.org/en-US/docs/Web/HTML/Element/td#attr-valign
	CellVerticalAlign = regexp.MustCompile(`(?i)^(baseline|bottom|middle|top)$`)

	// Direction handles the `dir` attribute
	// https://developer.mozilla.org/en-US/docs/Web/HTML/Element/bdo#attr-dir
	Direction = regexp.MustCompile(`(?i)^(rtl|ltr)$`)

	// ImageAlign handles the `align` attribute on the `image` tag
	// http://www.w3.org/MarkUp/Test/Img/imgtest.html
	ImageAlign = regexp.MustCompile(
		`(?i)^(left|right|top|texttop|middle|absmiddle|baseline|bottom|absbottom)$`,
	)

	// Integer describes whole positive integers (including 0) used in places
	// like td.colspan
	// https://developer.mozilla.org/en-US/docs/Web/HTML/Element/td#attr-colspan
	Integer = regexp.MustCompile(`^[0-9]+$`)

	// ISO8601 according to the W3 group is only a subset of the ISO8601
	// standard: http://www.w3.org/TR/NOTE-datetime
	//
	// Used in places like time.datetime
	// https://developer.mozilla.org/en-US/docs/Web/HTML/Element/time#attr-datetime
	//
	// Matches patterns:
	//  Year:
	//     YYYY (eg 1997)
	//  Year and month:
	//     YYYY-MM (eg 1997-07)
	//  Complete date:
	//     YYYY-MM-DD (eg 1997-07-16)
	//  Complete date plus hours and minutes:
	//     YYYY-MM-DDThh:mmTZD (eg 1997-07-16T19:20+01:00)
	//  Complete date plus hours, minutes and seconds:
	//     YYYY-MM-DDThh:mm:ssTZD (eg 1997-07-16T19:20:30+01:00)
	//  Complete date plus hours, minutes, seconds and a decimal fraction of a
	//  second
	//      YYYY-MM-DDThh:mm:ss.sTZD (eg 1997-07-16T19:20:30.45+01:00)
	ISO8601 = regexp.MustCompile(
		`^[0-9]{4}(-[0-9]{2}(-[0-9]{2}([ T][0-9]{2}(:[0-9]{2}){1,2}(.[0-9]{1,6})` +
			`?Z?([\+-][0-9]{2}:[0-9]{2})?)?)?)?$`,
	)

	// ListType encapsulates the common value as well as the latest spec
	// values for lists
	// https://developer.mozilla.org/en-US/docs/Web/HTML/Element/ol#attr-type
	ListType = regexp.MustCompile(`(?i)^(circle|disc|square|a|A|i|I|1)$`)

	// SpaceSeparatedTokens is used in places like `a.rel` and the common attribute
	// `class` which both contain space delimited lists of data tokens
	// http://www.w3.org/TR/html-markup/datatypes.html#common.data.tokens-def
	// Regexp: \p{L} matches unicode letters, \p{N} matches unicode numbers
	SpaceSeparatedTokens = regexp.MustCompile(`^([\s\p{L}\p{N}_-]+)$`)

	// Number is a double value used on HTML5 meter and progress elements
	// http://www.whatwg.org/specs/web-apps/current-work/multipage/the-button-element.html#the-meter-element
	Number = regexp.MustCompile(`^[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?$`)

	// NumberOrPercent is used predominantly as units of measurement in width
	// and height attributes
	// https://developer.mozilla.org/en-US/docs/Web/HTML/Element/img#attr-height
	NumberOrPercent = regexp.MustCompile(`^[0-9]+[%]?$`)

	// Paragraph of text in an attribute such as *.'title', img.alt, etc
	// https://developer.mozilla.org/en-US/docs/Web/HTML/Global_attributes#attr-title
	// Note that we are not allowing chars that could close tags like '>'
	Paragraph = regexp.MustCompile(`^[\p{L}\p{N}\s\-_',\[\]!\./\\\(\)]*$`)

	// dataURIImagePrefix is used by AllowDataURIImages to define the acceptable
	// prefix of data URIs that contain common web image formats.
	//
	// This is not exported as it's not useful by itself, and only has value
	// within the AllowDataURIImages func
	dataURIImagePrefix = regexp.MustCompile(
		`^image/(gif|jpeg|png|webp);base64,`,
	)
)

// AllowStandardURLs is a convenience function that will enable rel="nofollow"
// on "a", "area" and "link" (if you have allowed those elements) and will
// ensure that the URL values are parseable and either relative or belong to the
// "mailto", "http", or "https" schemes
func (p *Policy) AllowStandardURLs() {
	// URLs must be parseable by net/url.Parse()
	p.RequireParseableURLs(true)

	// !url.IsAbs() is permitted
	p.AllowRelativeURLs(true)

	// Most common URL schemes only
	p.AllowURLSchemes("mailto", "http", "https")

	// For all anchors we will add rel="nofollow" if it does not already exist
	// This applies to "a" "area" "link"
	p.RequireNoFollowOnLinks(true)
}

// AllowStandardAttributes will enable "id", "title" and the language specific
// attributes "dir" and "lang" on all elements that are whitelisted
func (p *Policy) AllowStandardAttributes() {
	// "dir" "lang" are permitted as both language attributes affect charsets
	// and direction of text.
	p.AllowAttrs("dir").Matching(Direction).Globally()
	p.AllowAttrs(
		"lang",
	).Matching(regexp.MustCompile(`[a-zA-Z]{2,20}`)).Globally()

	// "id" is permitted. This is pretty much as some HTML elements require this
	// to work well ("dfn" is an example of a "id" being value)
	// This does create a risk that JavaScript and CSS within your web page
	// might identify the wrong elements. Ensure that you select things
	// accurately
	p.AllowAttrs("id").Matching(
		regexp.MustCompile(`[a-zA-Z0-9\:\-_\.]+`),
	).Globally()

	// "title" is permitted as it improves accessibility.
	p.AllowAttrs("title").Matching(Paragraph).Globally()
}

// AllowStyling presently enables the class attribute globally.
//
// Note: When bluemonday ships a CSS parser and we can safely sanitise that,
// this will also allow sanitized styling of elements via the style attribute.
func (p *Policy) AllowStyling() {

	// "class" is permitted globally
	p.AllowAttrs("class").Matching(SpaceSeparatedTokens).Globally()
}

// AllowImages enables the img element and some popular attributes. It will also
// ensure that URL values are parseable. This helper does not enable data URI
// images, for that you should also use the AllowDataURIImages() helper.
func (p *Policy) AllowImages() {

	// "img" is permitted
	p.AllowAttrs("align").Matching(ImageAlign).OnElements("img")
	p.AllowAttrs("alt").Matching(Paragraph).OnElements("img")
	p.AllowAttrs("height", "width").Matching(NumberOrPercent).OnElements("img")

	// Standard URLs enabled
	p.AllowStandardURLs()
	p.AllowAttrs("src").OnElements("img")
}

// AllowDataURIImages permits the use of inline images defined in RFC2397
// http://tools.ietf.org/html/rfc2397
// http://en.wikipedia.org/wiki/Data_URI_scheme
//
// Images must have a mimetype matching:
//   image/gif
//   image/jpeg
//   image/png
//   image/webp
//
// NOTE: There is a potential security risk to allowing data URIs and you should
// only permit them on content you already trust.
// http://palizine.plynt.com/issues/2010Oct/bypass-xss-filters/
// https://capec.mitre.org/data/definitions/244.html
func (p *Policy) AllowDataURIImages() {

	// URLs must be parseable by net/url.Parse()
	p.RequireParseableURLs(true)

	// Supply a function to validate images contained within data URI
	p.AllowURLSchemeWithCustomPolicy(
		"data",
		func(url *url.URL) (allowUrl bool) {
			if url.RawQuery != "" || url.Fragment != "" {
				return false
			}

			matched := dataURIImagePrefix.FindString(url.Opaque)
			if matched == "" {
				return false
			}

			_, err := base64.StdEncoding.DecodeString(url.Opaque[len(matched):])
			if err != nil {
				return false
			}

			return true
		},
	)
}

// AllowLists will enabled ordered and unordered lists, as well as definition
// lists
func (p *Policy) AllowLists() {
	// "ol" "ul" are permitted
	p.AllowAttrs("type").Matching(ListType).OnElements("ol", "ul")

	// "li" is permitted
	p.AllowAttrs("type").Matching(ListType).OnElements("li")
	p.AllowAttrs("value").Matching(Integer).OnElements("li")

	// "dl" "dt" "dd" are permitted
	p.AllowElements("dl", "dt", "dd")
}

// AllowTables will enable a rich set of elements and attributes to describe
// HTML tables
func (p *Policy) AllowTables() {

	// "table" is permitted
	p.AllowAttrs("height", "width").Matching(NumberOrPercent).OnElements("table")
	p.AllowAttrs("summary").Matching(Paragraph).OnElements("table")

	// "caption" is permitted
	p.AllowElements("caption")

	// "col" "colgroup" are permitted
	p.AllowAttrs("align").Matching(CellAlign).OnElements("col", "colgroup")
	p.AllowAttrs("height", "width").Matching(
		NumberOrPercent,
	).OnElements("col", "colgroup")
	p.AllowAttrs("span").Matching(Integer).OnElements("colgroup", "col")
	p.AllowAttrs("valign").Matching(
		CellVerticalAlign,
	).OnElements("col", "colgroup")

	// "thead" "tr" are permitted
	p.AllowAttrs("align").Matching(CellAlign).OnElements("thead", "tr")
	p.AllowAttrs("valign").Matching(CellVerticalAlign).OnElements("thead", "tr")

	// "td" "th" are permitted
	p.AllowAttrs("abbr").Matching(Paragraph).OnElements("td", "th")
	p.AllowAttrs("align").Matching(CellAlign).OnElements("td", "th")
	p.AllowAttrs("colspan", "rowspan").Matching(Integer).OnElements("td", "th")
	p.AllowAttrs("headers").Matching(
		SpaceSeparatedTokens,
	).OnElements("td", "th")
	p.AllowAttrs("height", "width").Matching(
		NumberOrPercent,
	).OnElements("td", "th")
	p.AllowAttrs(
		"scope",
	).Matching(
		regexp.MustCompile(`(?i)(?:row|col)(?:group)?`),
	).OnElements("td", "th")
	p.AllowAttrs("valign").Matching(CellVerticalAlign).OnElements("td", "th")
	p.AllowAttrs("nowrap").Matching(
		regexp.MustCompile(`(?i)|nowrap`),
	).OnElements("td", "th")

	// "tbody" "tfoot"
	p.AllowAttrs("align").Matching(CellAlign).OnElements("tbody", "tfoot")
	p.AllowAttrs("valign").Matching(
		CellVerticalAlign,
	).OnElements("tbody", "tfoot")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/nsf/termbox-go"
)

func tbprint(x, y int, fg, bg termbox.Attribute, msg string) {
	for _, c := range msg {
		termbox.SetCell(x, y, c, fg, bg)
		x++
	}
}

var current string
var curev termbox.Event

func mouse_button_num(k termbox.Key) int {
	switch k {
	case termbox.MouseLeft:
		return 0
	case termbox.MouseMiddle:
		return 1
	case termbox.MouseRight:
		return 2
	}
	return 0
}

func redraw_all() {
	const coldef = termbox.ColorDefault
	termbox.Clear(coldef, coldef)
	tbprint(0, 0, termbox.ColorMagenta, coldef, "Press 'q' to quit")
	tbprint(0, 1, coldef, coldef, current)
	switch curev.Type {
	case termbox.EventKey:
		tbprint(0, 2, coldef, coldef,
			fmt.Sprintf("EventKey: k: %d, c: %c, mod: %d", curev.Key, curev.Ch, curev.Mod))
	case termbox.EventMouse:
		tbprint(0, 2, coldef, coldef,
			fmt.Sprintf("EventMouse: x: %d, y: %d, b: %d", curev.MouseX, curev.MouseY, mouse_button_num(curev.Key)))
	case termbox.EventNone:
		tbprint(0, 2, coldef, coldef, "EventNone")
	}
	tbprint(0, 3, coldef, coldef, fmt.Sprintf("%d", curev.N))

        tbprint(0,4, termbox.ColorRed,    coldef,   "Meow");
        tbprint(0,5, termbox.ColorGreen,  coldef,   "Meow");
        tbprint(0,6, termbox.ColorBlue,   coldef,   "Meow");
        tbprint(0,7, termbox.ColorYellow, coldef,   "Meow");

	termbox.Flush()
}

func main() {
	err := termbox.Init()
	if err != nil {
		panic(err)
	}
	defer termbox.Close()
	termbox.SetInputMode(termbox.InputAlt | termbox.InputMouse)
	redraw_all()

	data := make([]byte, 0, 64)
mainloop:
	for {
		if cap(data)-len(data) < 32 {
			newdata := make([]byte, len(data), len(data)+32)
			copy(newdata, data)
			data = newdata
		}
		beg := len(data)
		d := data[beg : beg+32]
		switch ev := termbox.PollRawEvent(d); ev.Type {
		case termbox.EventRaw:
			data = data[:beg+ev.N]
			current = fmt.Sprintf("%q", data)
			if current == `"q"` {
				break mainloop
			}

			curev = termbox.ParseEvent(data)
			if curev.N > 0 {
				copy(data, data[curev.N:])
				data = data[:len(data)-curev.N]
			}
		case termbox.EventError:
			panic(ev.Err)
		}
		redraw_all()
	}
}
<commit_msg>removed cruft<commit_after>package main

import (
	"fmt"
	"github.com/nsf/termbox-go"
)

func tbprint(x, y int, fg, bg termbox.Attribute, msg string) {
	for _, c := range msg {
		termbox.SetCell(x, y, c, fg, bg)
		x++
	}
}

var current string
var curev termbox.Event

func mouse_button_num(k termbox.Key) int {
	switch k {
	case termbox.MouseLeft:
		return 0
	case termbox.MouseMiddle:
		return 1
	case termbox.MouseRight:
		return 2
	}
	return 0
}

func redraw_all() {
	const coldef = termbox.ColorDefault
	termbox.Clear(coldef, coldef)

	tbprint(0, 0, termbox.ColorMagenta, coldef, "Press 'q' to quit")
	tbprint(0, 1, coldef, coldef, current)
	tbprint(0, 3, coldef, coldef, fmt.Sprintf("%d", curev.N))

        tbprint(0,4, termbox.ColorRed,    coldef,   "Meow");
        tbprint(0,5, termbox.ColorGreen,  coldef,   "Meow");
        tbprint(0,6, termbox.ColorBlue,   coldef,   "Meow");
        tbprint(0,7, termbox.ColorYellow, coldef,   "Meow");

	termbox.Flush()
}

func main() {
	err := termbox.Init()
	if err != nil {
		panic(err)
	}
	defer termbox.Close()
	termbox.SetInputMode(termbox.InputAlt | termbox.InputMouse)
	redraw_all()

	data := make([]byte, 0, 64)
mainloop:
	for {
		if cap(data)-len(data) < 32 {
			newdata := make([]byte, len(data), len(data)+32)
			copy(newdata, data)
			data = newdata
		}
		beg := len(data)
		d := data[beg : beg+32]
		switch ev := termbox.PollRawEvent(d); ev.Type {
		case termbox.EventRaw:
			data = data[:beg+ev.N]
			current = fmt.Sprintf("%q", data)
			if current == `"q"` {
				break mainloop
			}

			curev = termbox.ParseEvent(data)
			if curev.N > 0 {
				copy(data, data[curev.N:])
				data = data[:len(data)-curev.N]
			}
		case termbox.EventError:
			panic(ev.Err)
		}
		redraw_all()
	}
}
<|endoftext|>


<commit_before>// httping 0.1 - A tool to measure RTT on HTTP/S requests
// This software is distributed AS IS and has no warranty. This is merely a learning exercise and should not be used in production under any circumstances.
// This is my own work and not that of my employer, not is endorsed or supported by them in any conceivable way.
// Pedro Perez - pjperez@outlook.com
// Based on https://github.com/giigame/httping (Thanks!)

package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"strings"
	"time"

	"github.com/mgutz/ansi"
)

func main() {

	fmt.Println("\nhttping 0.1 - A tool to measure RTT on HTTP/S requests")
	fmt.Println("Help: httping help")

	if (len(os.Args) <= 1) || (os.Args[1] == "help") {
		fmt.Println("\n\nUsage: httping [http://|https://]url[:port] [GET|HEAD]")
		os.Exit(1)
	}

	urlStr := os.Args[1]

	if urlStr[:7] != "http://" {
		if urlStr[:8] != "https://" {
			if strings.Contains(urlStr, "://") {
				fmt.Println("\n\nWrong protocol specified, httping supports HTTP and HTTPS")
				os.Exit(1)
			}
			fmt.Printf("\n\nNo protocol specified, falling back to HTTP\n\n")
			urlStr = "http://" + urlStr
		}
	}

	url, err := url.Parse(urlStr)
	if err != nil {
		fmt.Println("Cannot resolve: " + urlStr)
		os.Exit(1)
		return
	}

	httpVerb := "GET"

	if len(os.Args) >= 3 {
		if os.Args[2] == "HEAD" {
			httpVerb = os.Args[2]
		} else {
			httpVerb = "GET"
		}
	} else {
		fmt.Printf("No HTTP Verb specified, falling back to HTTP\n\n")
	}

	fmt.Printf("HTTP %s to %s (%s):\n", httpVerb, url.Host, urlStr)
	ping(httpVerb, url)
}

func ping(httpVerb string, url *url.URL) {
	// This function loops indefinitely (TODO- Select number of iterations) and prints result on screen after each loop

	timeTotal := time.Duration(0)
	timeout := time.Duration(2 * time.Second)
	client := http.Client{
		Timeout: timeout,
	}
	successfulProbes := 0
	ansiRed := ansi.ColorCode("red+h:black")
	ansiGreen := ansi.ColorCode("green+h:black")
	reset := ansi.ColorCode("reset")
	result, err := client.Get(url.String())

	for i := 1; ; i++ {

		timeStart := time.Now()
		responseTime := time.Since(timeStart)
		// Send GET or HEAD request
		if httpVerb == "GET" {
			timeStart = time.Now()
			result, err = client.Get(url.String())
			responseTime = time.Since(timeStart)
		} else {
			timeStart = time.Now()
			result, err = client.Head(url.String())
			responseTime = time.Since(timeStart)
		}

		if err != nil {
			fmt.Println(ansiRed, "Fatal error!", err, reset)
		}

		timeTotal += responseTime

		// Calculate the downloaded bytes
		body, _ := ioutil.ReadAll(result.Body)
		bytes := len(body)

		// Print result on screen
		fmt.Printf("connected to %s, seq=%d, httpVerb=%s, httpStatus=%d, bytes=%d, RTT=%.2f ms\n", url.Host, i, httpVerb, result.StatusCode, bytes, float32(responseTime)/1e6)

		// Count how many probes are successful, i.e. how many get a 200 HTTP StatusCode
		if result.StatusCode == 200 {
			successfulProbes++
		}

		time.Sleep(1e9)

		c := make(chan os.Signal, 1)
		signal.Notify(c, os.Interrupt)
		go func() {
			for sig := range c {
				timeAverage := time.Duration(int64(timeTotal) / int64(i))
				_ = sig
				fmt.Println(ansiGreen, "Probes sent:", i, "\nSuccessful responses:", successfulProbes, "\nAverage response time:", timeAverage, reset)
				os.Exit(1)
			}
		}()
	}
}
<commit_msg>Removed ansi coloring and external dependency<commit_after>// httping 0.1 - A tool to measure RTT on HTTP/S requests
// This software is distributed AS IS and has no warranty. This is merely a learning exercise and should not be used in production under any circumstances.
// This is my own work and not that of my employer, not is endorsed or supported by them in any conceivable way.
// Pedro Perez - pjperez@outlook.com
// Based on https://github.com/giigame/httping (Thanks!)

package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"strings"
	"time"
)

func main() {

	fmt.Println("\nhttping 0.1 - A tool to measure RTT on HTTP/S requests")
	fmt.Println("Help: httping help")

	if (len(os.Args) <= 1) || (os.Args[1] == "help") {
		fmt.Println("\n\nUsage: httping [http://|https://]url[:port] [GET|HEAD]")
		os.Exit(1)
	}

	urlStr := os.Args[1]

	if urlStr[:7] != "http://" {
		if urlStr[:8] != "https://" {
			if strings.Contains(urlStr, "://") {
				fmt.Println("\n\nWrong protocol specified, httping supports HTTP and HTTPS")
				os.Exit(1)
			}
			fmt.Printf("\n\nNo protocol specified, falling back to HTTP\n\n")
			urlStr = "http://" + urlStr
		}
	}

	url, err := url.Parse(urlStr)
	if err != nil {
		fmt.Println("Cannot resolve: " + urlStr)
		os.Exit(1)
		return
	}

	httpVerb := "GET"

	if len(os.Args) >= 3 {
		if os.Args[2] == "HEAD" {
			httpVerb = os.Args[2]
		} else {
			httpVerb = "GET"
		}
	} else {
		fmt.Printf("No HTTP Verb specified, falling back to HTTP\n\n")
	}

	fmt.Printf("HTTP %s to %s (%s):\n", httpVerb, url.Host, urlStr)
	ping(httpVerb, url)
}

func ping(httpVerb string, url *url.URL) {
	// This function loops indefinitely (TODO- Select number of iterations) and prints result on screen after each loop

	timeTotal := time.Duration(0)
	timeout := time.Duration(2 * time.Second)
	client := http.Client{
		Timeout: timeout,
	}
	successfulProbes := 0
	result, err := client.Get(url.String())

	for i := 1; ; i++ {

		timeStart := time.Now()
		responseTime := time.Since(timeStart)
		// Send GET or HEAD request
		if httpVerb == "GET" {
			timeStart = time.Now()
			result, err = client.Get(url.String())
			responseTime = time.Since(timeStart)
		} else {
			timeStart = time.Now()
			result, err = client.Head(url.String())
			responseTime = time.Since(timeStart)
		}

		if err != nil {
			fmt.Println("Fatal error!", err)
		}

		timeTotal += responseTime

		// Calculate the downloaded bytes
		body, _ := ioutil.ReadAll(result.Body)
		bytes := len(body)

		// Print result on screen
		fmt.Printf("connected to %s, seq=%d, httpVerb=%s, httpStatus=%d, bytes=%d, RTT=%.2f ms\n", url.Host, i, httpVerb, result.StatusCode, bytes, float32(responseTime)/1e6)

		// Count how many probes are successful, i.e. how many get a 200 HTTP StatusCode
		if result.StatusCode == 200 {
			successfulProbes++
		}

		time.Sleep(1e9)

		c := make(chan os.Signal, 1)
		signal.Notify(c, os.Interrupt)
		go func() {
			for sig := range c {
				timeAverage := time.Duration(int64(timeTotal) / int64(i))
				_ = sig
				fmt.Println("\nProbes sent:", i, "\nSuccessful responses:", successfulProbes, "\nAverage response time:", timeAverage)
				os.Exit(1)
			}
		}()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"errors"
	"flag"
	"fmt"
	"log"
	"math"
	"sort"
	"strings"
	"time"

	"github.com/octo/icestat/bahn"
)

var (
	interval    = flag.Duration("interval", 10*time.Second, "Interval in which to report statistics.")
	count       = flag.Int("count", -1, "Number of iterations.")
	destination = flag.String("destination", "", "Optional destination to anticipate.")
)

type speedDistribution struct {
	data []float64
}

func (s *speedDistribution) add(kmh float64) {
	s.data = append(s.data, kmh)
	sort.Float64s(s.data)
}

func (s speedDistribution) max() float64 {
	if len(s.data) == 0 {
		return math.NaN()
	}

	return s.data[len(s.data)-1]
}

func (s speedDistribution) percentile(p float64) float64 {
	if len(s.data) == 0 {
		return math.NaN()
	}

	idx := float64(len(s.data)) * p / 100.0
	return s.data[int(idx-0.5)]
}

func (s speedDistribution) median() float64 {
	return s.percentile(50.0)
}

func (s speedDistribution) average() float64 {
	var sum float64
	var num int

	for _, speed := range s.data {
		if !math.IsNaN(speed) {
			sum += speed
			num++
		}
	}

	if num == 0 {
		return math.NaN()
	}

	return sum / float64(num)
}

func formatDuration(d time.Duration) string {
	h, m := math.Modf(d.Hours())
	m *= 60.0

	return fmt.Sprintf("%.0f:%02.0f", h, m)
}

func printTrip(trip *bahn.Trip) error {
	if len(trip.Stops) == 0 {
		return errors.New("trip contains no strops")
	}

	finalStop := trip.Stops[len(trip.Stops)-1]
	nextStop := trip.NextStop
	if nextStop == nil {
		return fmt.Errorf("train arrived in %v", finalStop)
	}

	destinationStop := finalStop
	if *destination != "" {
		var ok bool
		destinationStop, ok = trip.FindStop(*destination)

		if !ok {
			var stops []string
			for _, stop := range trip.Stops {
				stops = append(stops, stop.Station.Name)
			}

			return fmt.Errorf("stop %q not found. Valid stops are: %s",
				*destination, strings.Join(stops, ", "))
		}
	}

	if destinationStop.Passed {
		return fmt.Errorf("train has passed %v", destinationStop)
	}

	if destinationStop != nextStop {
		fmt.Printf("%s%s to %q (via %q): "+
			"distance=%.0f(%.0f) km, "+
			"eta=%s(%s), "+
			"delay=%s(%s)",
			trip.TrainType, trip.TrainID, destinationStop.Station, nextStop.Station,
			trip.DistanceTo(destinationStop), trip.DistanceTo(nextStop),
			formatDuration(destinationStop.ETA()), formatDuration(nextStop.ETA()),
			formatDuration(destinationStop.Delay()), formatDuration(nextStop.Delay()))
	} else {
		fmt.Printf("%s%s to %q: "+
			"distance=%.0f km, "+
			"eta=%s, "+
			"delay=%s",
			trip.TrainType, trip.TrainID, destinationStop.Station,
			trip.DistanceTo(destinationStop),
			formatDuration(destinationStop.ETA()),
			formatDuration(destinationStop.Delay()))
	}

	return nil
}

var speed speedDistribution

func printSpeed(ctx context.Context) error {
	s, err := bahn.StatusInfo(ctx)
	if err != nil {
		return err
	}
	speed.add(s.Speed)

	fmt.Printf(", speed=%.0f/%.0f/%.0f [km/h] (cur/avg/max)",
		s.Speed, speed.average(), speed.max())

	return nil
}

func printUpdate(ctx context.Context) error {
	trip, err := bahn.TripInfo(ctx)
	if err != nil {
		return err
	}

	defer fmt.Println()

	if err := printTrip(trip); err != nil {
		return err
	}

	if err := printSpeed(ctx); err != nil {
		return err
	}

	return nil
}

func main() {
	flag.Parse()
	ctx := context.Background()

	for *count != 0 {
		if *count > 0 {
			*count -= 1
		}

		ctx, cancel := context.WithTimeout(ctx, *interval)
		if err := printUpdate(ctx); err != nil {
			cancel()
			log.Println(err)
			time.Sleep(*interval)
			continue
		}
		cancel()

		if *count != 0 {
			time.Sleep(*interval)
		}
	}
}
<commit_msg>icestat: Fix typo.<commit_after>package main

import (
	"context"
	"errors"
	"flag"
	"fmt"
	"log"
	"math"
	"sort"
	"strings"
	"time"

	"github.com/octo/icestat/bahn"
)

var (
	interval    = flag.Duration("interval", 10*time.Second, "Interval in which to report statistics.")
	count       = flag.Int("count", -1, "Number of iterations.")
	destination = flag.String("destination", "", "Optional destination to anticipate.")
)

type speedDistribution struct {
	data []float64
}

func (s *speedDistribution) add(kmh float64) {
	s.data = append(s.data, kmh)
	sort.Float64s(s.data)
}

func (s speedDistribution) max() float64 {
	if len(s.data) == 0 {
		return math.NaN()
	}

	return s.data[len(s.data)-1]
}

func (s speedDistribution) percentile(p float64) float64 {
	if len(s.data) == 0 {
		return math.NaN()
	}

	idx := float64(len(s.data)) * p / 100.0
	return s.data[int(idx-0.5)]
}

func (s speedDistribution) median() float64 {
	return s.percentile(50.0)
}

func (s speedDistribution) average() float64 {
	var sum float64
	var num int

	for _, speed := range s.data {
		if !math.IsNaN(speed) {
			sum += speed
			num++
		}
	}

	if num == 0 {
		return math.NaN()
	}

	return sum / float64(num)
}

func formatDuration(d time.Duration) string {
	h, m := math.Modf(d.Hours())
	m *= 60.0

	return fmt.Sprintf("%.0f:%02.0f", h, m)
}

func printTrip(trip *bahn.Trip) error {
	if len(trip.Stops) == 0 {
		return errors.New("trip contains no stops")
	}

	finalStop := trip.Stops[len(trip.Stops)-1]
	nextStop := trip.NextStop
	if nextStop == nil {
		return fmt.Errorf("train arrived in %v", finalStop)
	}

	destinationStop := finalStop
	if *destination != "" {
		var ok bool
		destinationStop, ok = trip.FindStop(*destination)

		if !ok {
			var stops []string
			for _, stop := range trip.Stops {
				stops = append(stops, stop.Station.Name)
			}

			return fmt.Errorf("stop %q not found. Valid stops are: %s",
				*destination, strings.Join(stops, ", "))
		}
	}

	if destinationStop.Passed {
		return fmt.Errorf("train has passed %v", destinationStop)
	}

	if destinationStop != nextStop {
		fmt.Printf("%s%s to %q (via %q): "+
			"distance=%.0f(%.0f) km, "+
			"eta=%s(%s), "+
			"delay=%s(%s)",
			trip.TrainType, trip.TrainID, destinationStop.Station, nextStop.Station,
			trip.DistanceTo(destinationStop), trip.DistanceTo(nextStop),
			formatDuration(destinationStop.ETA()), formatDuration(nextStop.ETA()),
			formatDuration(destinationStop.Delay()), formatDuration(nextStop.Delay()))
	} else {
		fmt.Printf("%s%s to %q: "+
			"distance=%.0f km, "+
			"eta=%s, "+
			"delay=%s",
			trip.TrainType, trip.TrainID, destinationStop.Station,
			trip.DistanceTo(destinationStop),
			formatDuration(destinationStop.ETA()),
			formatDuration(destinationStop.Delay()))
	}

	return nil
}

var speed speedDistribution

func printSpeed(ctx context.Context) error {
	s, err := bahn.StatusInfo(ctx)
	if err != nil {
		return err
	}
	speed.add(s.Speed)

	fmt.Printf(", speed=%.0f/%.0f/%.0f [km/h] (cur/avg/max)",
		s.Speed, speed.average(), speed.max())

	return nil
}

func printUpdate(ctx context.Context) error {
	trip, err := bahn.TripInfo(ctx)
	if err != nil {
		return err
	}

	defer fmt.Println()

	if err := printTrip(trip); err != nil {
		return err
	}

	if err := printSpeed(ctx); err != nil {
		return err
	}

	return nil
}

func main() {
	flag.Parse()
	ctx := context.Background()

	for *count != 0 {
		if *count > 0 {
			*count -= 1
		}

		ctx, cancel := context.WithTimeout(ctx, *interval)
		if err := printUpdate(ctx); err != nil {
			cancel()
			log.Println(err)
			time.Sleep(*interval)
			continue
		}
		cancel()

		if *count != 0 {
			time.Sleep(*interval)
		}
	}
}
<|endoftext|>