<commit_before>package gost

import (
	"crypto/rand"
	"crypto/rsa"
	"crypto/tls"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"errors"
	"io"
	"math/big"
	"net"
	"strings"
	"sync"
	"time"

	"github.com/go-log/log"
)

// Version is the gost version.
const Version = "2.7.1"

// Debug is a flag that enables the debug log.
var Debug bool

var (
	tinyBufferSize   = 128
	smallBufferSize  = 1 * 1024  // 1KB small buffer
	mediumBufferSize = 8 * 1024  // 8KB medium buffer
	largeBufferSize  = 32 * 1024 // 32KB large buffer
)

var (
	sPool = sync.Pool{
		New: func() interface{} {
			return make([]byte, smallBufferSize)
		},
	}
	mPool = sync.Pool{
		New: func() interface{} {
			return make([]byte, mediumBufferSize)
		},
	}
	lPool = sync.Pool{
		New: func() interface{} {
			return make([]byte, largeBufferSize)
		},
	}
)

var (
	// KeepAliveTime is the keep alive time period for TCP connection.
	KeepAliveTime = 180 * time.Second
	// DialTimeout is the timeout of dial.
	DialTimeout = 5 * time.Second
	// HandshakeTimeout is the timeout of handshake.
	HandshakeTimeout = 5 * time.Second
	// ConnectTimeout is the timeout for connect.
	ConnectTimeout = 5 * time.Second
	// ReadTimeout is the timeout for reading.
	ReadTimeout = 10 * time.Second
	// WriteTimeout is the timeout for writing.
	WriteTimeout = 10 * time.Second
	// PingTimeout is the timeout for pinging.
	PingTimeout = 30 * time.Second
	// PingRetries is the reties of ping.
	PingRetries = 1
	// default udp node TTL in second for udp port forwarding.
	defaultTTL = 60 * time.Second
)

var (
	// DefaultTLSConfig is a default TLS config for internal use.
	DefaultTLSConfig *tls.Config

	// DefaultUserAgent is the default HTTP User-Agent header used by HTTP and websocket.
	DefaultUserAgent = "Chrome/60.0.3112.90"
)

// SetLogger sets a new logger for internal log system.
func SetLogger(logger log.Logger) {
	log.DefaultLogger = logger
}

// GenCertificate generates a random TLS certificate.
func GenCertificate() (cert tls.Certificate, err error) {
	rawCert, rawKey, err := generateKeyPair()
	if err != nil {
		return
	}
	return tls.X509KeyPair(rawCert, rawKey)
}

func generateKeyPair() (rawCert, rawKey []byte, err error) {
	// Create private key and self-signed certificate
	// Adapted from https://golang.org/src/crypto/tls/generate_cert.go

	priv, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		return
	}
	validFor := time.Hour * 24 * 365 * 10 // ten years
	notBefore := time.Now()
	notAfter := notBefore.Add(validFor)
	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
	template := x509.Certificate{
		SerialNumber: serialNumber,
		Subject: pkix.Name{
			Organization: []string{"gost"},
		},
		NotBefore: notBefore,
		NotAfter:  notAfter,

		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
		BasicConstraintsValid: true,
	}
	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
	if err != nil {
		return
	}

	rawCert = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
	rawKey = pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})

	return
}

type readWriter struct {
	r io.Reader
	w io.Writer
}

func (rw *readWriter) Read(p []byte) (n int, err error) {
	return rw.r.Read(p)
}

func (rw *readWriter) Write(p []byte) (n int, err error) {
	return rw.w.Write(p)
}

var (
	nopClientConn = &nopConn{}
)

// a nop connection implements net.Conn,
// it does nothing.
type nopConn struct{}

func (c *nopConn) Read(b []byte) (n int, err error) {
	return 0, &net.OpError{Op: "read", Net: "nop", Source: nil, Addr: nil, Err: errors.New("read not supported")}
}

func (c *nopConn) Write(b []byte) (n int, err error) {
	return 0, &net.OpError{Op: "write", Net: "nop", Source: nil, Addr: nil, Err: errors.New("write not supported")}
}

func (c *nopConn) Close() error {
	return nil
}

func (c *nopConn) LocalAddr() net.Addr {
	return nil
}

func (c *nopConn) RemoteAddr() net.Addr {
	return nil
}

func (c *nopConn) SetDeadline(t time.Time) error {
	return &net.OpError{Op: "set", Net: "nop", Source: nil, Addr: nil, Err: errors.New("deadline not supported")}
}

func (c *nopConn) SetReadDeadline(t time.Time) error {
	return &net.OpError{Op: "set", Net: "nop", Source: nil, Addr: nil, Err: errors.New("deadline not supported")}
}

func (c *nopConn) SetWriteDeadline(t time.Time) error {
	return &net.OpError{Op: "set", Net: "nop", Source: nil, Addr: nil, Err: errors.New("deadline not supported")}
}

// splitLine splits a line text by white space, mainly used by config parser.
func splitLine(line string) []string {
	if line == "" {
		return nil
	}
	if n := strings.IndexByte(line, '#'); n >= 0 {
		line = line[:n]
	}
	line = strings.Replace(line, "\t", " ", -1)
	line = strings.TrimSpace(line)

	var ss []string
	for _, s := range strings.Split(line, " ") {
		if s = strings.TrimSpace(s); s != "" {
			ss = append(ss, s)
		}
	}
	return ss
}
<commit_msg>v2.7.2<commit_after>package gost

import (
	"crypto/rand"
	"crypto/rsa"
	"crypto/tls"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"errors"
	"io"
	"math/big"
	"net"
	"strings"
	"sync"
	"time"

	"github.com/go-log/log"
)

// Version is the gost version.
const Version = "2.7.2"

// Debug is a flag that enables the debug log.
var Debug bool

var (
	tinyBufferSize   = 128
	smallBufferSize  = 1 * 1024  // 1KB small buffer
	mediumBufferSize = 8 * 1024  // 8KB medium buffer
	largeBufferSize  = 32 * 1024 // 32KB large buffer
)

var (
	sPool = sync.Pool{
		New: func() interface{} {
			return make([]byte, smallBufferSize)
		},
	}
	mPool = sync.Pool{
		New: func() interface{} {
			return make([]byte, mediumBufferSize)
		},
	}
	lPool = sync.Pool{
		New: func() interface{} {
			return make([]byte, largeBufferSize)
		},
	}
)

var (
	// KeepAliveTime is the keep alive time period for TCP connection.
	KeepAliveTime = 180 * time.Second
	// DialTimeout is the timeout of dial.
	DialTimeout = 5 * time.Second
	// HandshakeTimeout is the timeout of handshake.
	HandshakeTimeout = 5 * time.Second
	// ConnectTimeout is the timeout for connect.
	ConnectTimeout = 5 * time.Second
	// ReadTimeout is the timeout for reading.
	ReadTimeout = 10 * time.Second
	// WriteTimeout is the timeout for writing.
	WriteTimeout = 10 * time.Second
	// PingTimeout is the timeout for pinging.
	PingTimeout = 30 * time.Second
	// PingRetries is the reties of ping.
	PingRetries = 1
	// default udp node TTL in second for udp port forwarding.
	defaultTTL = 60 * time.Second
)

var (
	// DefaultTLSConfig is a default TLS config for internal use.
	DefaultTLSConfig *tls.Config

	// DefaultUserAgent is the default HTTP User-Agent header used by HTTP and websocket.
	DefaultUserAgent = "Chrome/60.0.3112.90"
)

// SetLogger sets a new logger for internal log system.
func SetLogger(logger log.Logger) {
	log.DefaultLogger = logger
}

// GenCertificate generates a random TLS certificate.
func GenCertificate() (cert tls.Certificate, err error) {
	rawCert, rawKey, err := generateKeyPair()
	if err != nil {
		return
	}
	return tls.X509KeyPair(rawCert, rawKey)
}

func generateKeyPair() (rawCert, rawKey []byte, err error) {
	// Create private key and self-signed certificate
	// Adapted from https://golang.org/src/crypto/tls/generate_cert.go

	priv, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		return
	}
	validFor := time.Hour * 24 * 365 * 10 // ten years
	notBefore := time.Now()
	notAfter := notBefore.Add(validFor)
	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
	template := x509.Certificate{
		SerialNumber: serialNumber,
		Subject: pkix.Name{
			Organization: []string{"gost"},
		},
		NotBefore: notBefore,
		NotAfter:  notAfter,

		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
		BasicConstraintsValid: true,
	}
	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
	if err != nil {
		return
	}

	rawCert = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
	rawKey = pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})

	return
}

type readWriter struct {
	r io.Reader
	w io.Writer
}

func (rw *readWriter) Read(p []byte) (n int, err error) {
	return rw.r.Read(p)
}

func (rw *readWriter) Write(p []byte) (n int, err error) {
	return rw.w.Write(p)
}

var (
	nopClientConn = &nopConn{}
)

// a nop connection implements net.Conn,
// it does nothing.
type nopConn struct{}

func (c *nopConn) Read(b []byte) (n int, err error) {
	return 0, &net.OpError{Op: "read", Net: "nop", Source: nil, Addr: nil, Err: errors.New("read not supported")}
}

func (c *nopConn) Write(b []byte) (n int, err error) {
	return 0, &net.OpError{Op: "write", Net: "nop", Source: nil, Addr: nil, Err: errors.New("write not supported")}
}

func (c *nopConn) Close() error {
	return nil
}

func (c *nopConn) LocalAddr() net.Addr {
	return nil
}

func (c *nopConn) RemoteAddr() net.Addr {
	return nil
}

func (c *nopConn) SetDeadline(t time.Time) error {
	return &net.OpError{Op: "set", Net: "nop", Source: nil, Addr: nil, Err: errors.New("deadline not supported")}
}

func (c *nopConn) SetReadDeadline(t time.Time) error {
	return &net.OpError{Op: "set", Net: "nop", Source: nil, Addr: nil, Err: errors.New("deadline not supported")}
}

func (c *nopConn) SetWriteDeadline(t time.Time) error {
	return &net.OpError{Op: "set", Net: "nop", Source: nil, Addr: nil, Err: errors.New("deadline not supported")}
}

// splitLine splits a line text by white space, mainly used by config parser.
func splitLine(line string) []string {
	if line == "" {
		return nil
	}
	if n := strings.IndexByte(line, '#'); n >= 0 {
		line = line[:n]
	}
	line = strings.Replace(line, "\t", " ", -1)
	line = strings.TrimSpace(line)

	var ss []string
	for _, s := range strings.Split(line, " ") {
		if s = strings.TrimSpace(s); s != "" {
			ss = append(ss, s)
		}
	}
	return ss
}
<|endoftext|>


<commit_before>package cmd

import (
	"io/ioutil"

	. "gopkg.in/check.v1"
)

type StopTest struct{}

var _ = Suite(&StopTest{})

func (s *StopTest) TestStopBadSpec(c *C) {
	file, err := ioutil.TempFile("", "")
	c.Assert(err, IsNil)
	c.Assert(Stop.Flags().Parse([]string{"--json", file.Name()}), IsNil)
	_, err = file.WriteString("{")
	c.Assert(err, IsNil)
	c.Assert(file.Close(), IsNil)
	c.Assert(Stop.RunE(Stop, []string{}), ErrorMatches, "unexpected end of JSON input")
}

func (s *StopTest) TestStopJSONFlagNotProvided(c *C) {
	c.Assert(Stop.Flags().Parse([]string{}), IsNil)
	c.Assert(Stop.Flags().Set("json", ""), IsNil)
	c.Assert(Stop.RunE(Stop, []string{}), ErrorMatches, "--json not provided")
}
<commit_msg>fix test names<commit_after>package cmd

import (
	"io/ioutil"

	. "gopkg.in/check.v1"
)

type StopTest struct{}

var _ = Suite(&StopTest{})

func (s *StopTest) TestStop_BadSpec(c *C) {
	file, err := ioutil.TempFile("", "")
	c.Assert(err, IsNil)
	c.Assert(Stop.Flags().Parse([]string{"--json", file.Name()}), IsNil)
	_, err = file.WriteString("{")
	c.Assert(err, IsNil)
	c.Assert(file.Close(), IsNil)
	c.Assert(Stop.RunE(Stop, []string{}), ErrorMatches, "unexpected end of JSON input")
}

func (s *StopTest) TestStop_JSONFlagNotProvided(c *C) {
	c.Assert(Stop.Flags().Parse([]string{}), IsNil)
	c.Assert(Stop.Flags().Set("json", ""), IsNil)
	c.Assert(Stop.RunE(Stop, []string{}), ErrorMatches, "--json not provided")
}
<|endoftext|>


<commit_before>// web100_cli provides a simple CLI interface to web100lib functions.
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"os"

	"cloud.google.com/go/bigquery"
	"github.com/m-lab/etl/web100lib"
)

var (
	filename = flag.String("filename", "", "Trace filename.")
	tcpKis   = flag.String("tcp-kis", "tcp-kis.txt", "tcp-kis.txt filename.")
)

func PrettyPrint(results map[string]bigquery.Value) {
	b, err := json.MarshalIndent(results, "", "  ")
	if err != nil {
		fmt.Println("error:", err)
	}
	fmt.Print(string(b))
}

func main() {
	flag.Parse()

	// Parse tcp-kis.txt variable definitions.
	k, err := os.Open(*tcpKis)
	if err != nil {
		panic(err)
	}
	legacyNames, err := web100lib.ParseWeb100Definitions(k)
	if err != nil {
		panic(err)
	}

	// Open web100 snapshot log.
	w, err := web100lib.Open(*filename, legacyNames)
	if err != nil {
		panic(err)
	}
	defer w.Close()

	// Find all last web100 snapshot.
	for {
		err = w.Next()
		if err != nil {
			break
		}
		results, err := w.Values(legacyNames)
		if err != nil {
			panic(err)
		}
		PrettyPrint(results)
	}
	if err != io.EOF {
		panic(err)
	}
	results, err := w.Values()
	if err != nil {
		panic(err)
	}
	PrettyPrint(results)

	// Get results.
	// results, err = w.SnapValues(legacyNames)
	// if err != nil {
	// 	panic(err)
	// }
	// PrettyPrint(results)
	// fmt.Printf("%#v\n", w)
}
<commit_msg>Use the new Open interface.<commit_after>// web100_cli provides a simple CLI interface to web100lib functions.
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"os"

	"cloud.google.com/go/bigquery"
	"github.com/m-lab/etl/web100lib"
)

var (
	filename = flag.String("filename", "", "Trace filename.")
	tcpKis   = flag.String("tcp-kis", "tcp-kis.txt", "tcp-kis.txt filename.")
)

func PrettyPrint(results map[string]bigquery.Value) {
	b, err := json.MarshalIndent(results, "", "  ")
	if err != nil {
		fmt.Println("error:", err)
	}
	fmt.Print(string(b))
}

func main() {
	flag.Parse()

	// Parse tcp-kis.txt variable definitions.
	k, err := os.Open(*tcpKis)
	if err != nil {
		panic(err)
	}
	legacyNames, err := web100lib.ParseWeb100Definitions(k)
	if err != nil {
		panic(err)
	}

	// Open web100 snapshot log.
	w, err := web100lib.Open(*filename, legacyNames)
	if err != nil {
		panic(err)
	}
	defer w.Close()

	// Find all last web100 snapshot.
	for {
		err = w.Next()
		if err != nil {
			break
		}
	}
	if err != io.EOF {
		panic(err)
	}
	results, err := w.Values()
	if err != nil {
		panic(err)
	}
	PrettyPrint(results)

	// Get results.
	// results, err = w.SnapValues(legacyNames)
	// if err != nil {
	// 	panic(err)
	// }
	// PrettyPrint(results)
	// fmt.Printf("%#v\n", w)
}
<|endoftext|>


<commit_before>// +build freebsd

package net

import (
	"errors"
	"os/exec"
	"strconv"
	"strings"

	"github.com/shirou/gopsutil/internal/common"
)

func NetIOCounters(pernic bool) ([]NetIOCountersStat, error) {
	out, err := exec.Command("/usr/bin/netstat", "-ibdn").Output()
	if err != nil {
		return nil, err
	}

	lines := strings.Split(string(out), "\n")
	ret := make([]NetIOCountersStat, 0, len(lines)-1)
	exists := make([]string, 0, len(ret))

	for _, line := range lines {
		values := strings.Fields(line)
		if len(values) < 1 || values[0] == "Name" {
			continue
		}
		if common.StringsHas(exists, values[0]) {
			// skip if already get
			continue
		}
		exists = append(exists, values[0])

		base := 1
		// sometimes Address is ommitted
		if len(values) < 13 {
			base = 0
		}

		parsed := make([]uint64, 0, 8)
		vv := []string{
			values[base+3],  // PacketsRecv
			values[base+4],  // Errin
			values[base+5],  // Dropin
			values[base+6],  // BytesRecvn
			values[base+7],  // PacketSent
			values[base+8],  // Errout
			values[base+9],  // BytesSent
			values[base+11], // Dropout
		}
		for _, target := range vv {
			if target == "-" {
				parsed = append(parsed, 0)
				continue
			}

			t, err := strconv.ParseUint(target, 10, 64)
			if err != nil {
				return nil, err
			}
			parsed = append(parsed, t)
		}

		n := NetIOCountersStat{
			Name:        values[0],
			PacketsRecv: parsed[0],
			Errin:       parsed[1],
			Dropin:      parsed[2],
			BytesRecv:   parsed[3],
			PacketsSent: parsed[4],
			Errout:      parsed[5],
			BytesSent:   parsed[6],
			Dropout:     parsed[7],
		}
		ret = append(ret, n)
	}

	if pernic == false {
		return getNetIOCountersAll(ret)
	}

	return ret, nil
}

// NetProtoCounters returns network statistics for the entire system
// If protocols is empty then all protocols are returned, otherwise
// just the protocols in the list are returned.
// Not Implemented for FreeBSD
func NetProtoCounters(protocols []string) ([]NetProtoCountersStat, error) {
	return nil, errors.New("NetProtoCounters not implemented for freebsd")
}
<commit_msg>[net] fix panic on freebsd (netstat error with exit code 0)<commit_after>// +build freebsd

package net

import (
	"errors"
	"os/exec"
	"strconv"
	"strings"

	"github.com/shirou/gopsutil/internal/common"
)

func NetIOCounters(pernic bool) ([]NetIOCountersStat, error) {
	out, err := exec.Command("/usr/bin/netstat", "-ibdn").Output()
	if err != nil {
		return nil, err
	}

	lines := strings.Split(string(out), "\n")
	ret := make([]NetIOCountersStat, 0, len(lines)-1)
	exists := make([]string, 0, len(ret))

	for _, line := range lines {
		values := strings.Fields(line)
		if len(values) < 1 || values[0] == "Name" {
			continue
		}
		if common.StringsHas(exists, values[0]) {
			// skip if already get
			continue
		}
		exists = append(exists, values[0])

		if len(values) < 12 {
			continue
		}
		base := 1
		// sometimes Address is ommitted
		if len(values) < 13 {
			base = 0
		}

		parsed := make([]uint64, 0, 8)
		vv := []string{
			values[base+3],  // PacketsRecv
			values[base+4],  // Errin
			values[base+5],  // Dropin
			values[base+6],  // BytesRecvn
			values[base+7],  // PacketSent
			values[base+8],  // Errout
			values[base+9],  // BytesSent
			values[base+11], // Dropout
		}
		for _, target := range vv {
			if target == "-" {
				parsed = append(parsed, 0)
				continue
			}

			t, err := strconv.ParseUint(target, 10, 64)
			if err != nil {
				return nil, err
			}
			parsed = append(parsed, t)
		}

		n := NetIOCountersStat{
			Name:        values[0],
			PacketsRecv: parsed[0],
			Errin:       parsed[1],
			Dropin:      parsed[2],
			BytesRecv:   parsed[3],
			PacketsSent: parsed[4],
			Errout:      parsed[5],
			BytesSent:   parsed[6],
			Dropout:     parsed[7],
		}
		ret = append(ret, n)
	}

	if pernic == false {
		return getNetIOCountersAll(ret)
	}

	return ret, nil
}

// NetProtoCounters returns network statistics for the entire system
// If protocols is empty then all protocols are returned, otherwise
// just the protocols in the list are returned.
// Not Implemented for FreeBSD
func NetProtoCounters(protocols []string) ([]NetProtoCountersStat, error) {
	return nil, errors.New("NetProtoCounters not implemented for freebsd")
}
<|endoftext|>


<commit_before>package errorutil

import (
	"errors"
	"net/http"
)

// HTTPStatusCodeEr defines errors that should return a specific HTTP status code
type HTTPStatusCodeEr interface {
	HTTPStatusCode() int
}

// StatusCodeEr defines errors that should return a specific HTTP status code
type StatusCodeEr interface {
	StatusCode() int
}

// HTTPStatusCode returns the status code that a HTTP handler should return.
//
// If the error is nil, StatusOK is returned.
//
// If the error implements HTTPStatusCodeEr or StatusCodeEr, it returns the corresponding status code.
//
// It tries to check some stdlib errors (testing the error string, to avoid importing unwanted packages),
// and returns appropriate status codes.
//
// Otherwise, StatusInternalServerError is returned.
func HTTPStatusCode(err error) int {
	if err == nil {
		return http.StatusOK
	}
	type causer interface {
		Cause() error
	}

	for err != nil {
		if status, ok := err.(HTTPStatusCodeEr); ok {
			return status.HTTPStatusCode()
		}
		if status, ok := err.(StatusCodeEr); ok {
			return status.StatusCode()
		}
		// Check errors from stdlib. Test string to avoid importing packages
		switch err.Error() {
		// package os
		case "permission denied":
			return http.StatusForbidden
		case "file does not exist":
			return http.StatusNotFound
		case "storage: bucket doesn't exist":
			return http.StatusNotFound
		case "storage: object doesn't exist":
			return http.StatusNotFound
		// package database/sql
		case "sql: no rows in result set":
			return http.StatusNotFound
		case "i/o timeout":
			return http.StatusRequestTimeout
		}
		cause, ok := err.(causer)
		if ok {
			err = cause.Cause()
		} else {
			err = errors.Unwrap(err)
		}
	}
	return http.StatusInternalServerError
}

// HTTPError builds an error based on a http.Response. If status code is < 300 or 304, nil is returned.
// Otherwise, errors implementing the various interfaces (Retryabler, HTTPStatusCodeEr) are returned
func HTTPError(resp *http.Response) error {
	if resp.StatusCode < 300 || resp.StatusCode == http.StatusNotModified {
		return nil
	}
	return httpError(resp.StatusCode)
}

type httpError int

func (err httpError) Error() string {
	switch err {
	case 429:
		return "Too Many Requests"
	default:
		return http.StatusText(int(err))
	}
}

func (err httpError) HTTPStatusCode() int {
	return int(err)
}

func (err httpError) StatusCode() int {
	return int(err)
}
func (err httpError) Retryable() bool {
	switch int(err) {
	case http.StatusBadGateway, http.StatusGatewayTimeout, http.StatusServiceUnavailable, http.StatusInternalServerError:
		return true
	case 429:
		return true
	default:
		return false
	}
}

// NotFoundError marks an error as "not found". The calling http handler
// should return a StatusNotFound status code. It returns nil if the error is nil.
func NotFoundError(err error) error {
	if err == nil {
		return nil
	}
	return &notFoundError{err: err}
}

type notFoundError struct {
	err error
}

func (err *notFoundError) Error() string {
	return err.err.Error()
}

func (err *notFoundError) HTTPStatusCode() int {
	return http.StatusNotFound
}

func (err *notFoundError) IsRetryable() bool {
	return false
}

func (err *notFoundError) Cause() error {
	return err.err
}

// ForbiddenError marks an error as "access forbidden". The calling http handler
// should return a StatusForbidden status code. It returns nil if the error is nil.
func ForbiddenError(err error) error {
	if err == nil {
		return nil
	}
	return &forbiddenError{err: err}
}

type forbiddenError struct {
	err error
}

func (err *forbiddenError) Error() string {
	return err.err.Error()
}

func (err *forbiddenError) HTTPStatusCode() int {
	return http.StatusForbidden
}

func (err *forbiddenError) IsRetryable() bool {
	return false
}

func (err *forbiddenError) Cause() error {
	return err.err
}

// InvalidError marks an error as "invalid". The calling http handler
// should return a StatusBadRequest status code. It returns nil if the error is nil.
func InvalidError(err error) error {
	if err == nil {
		return nil
	}
	return &invalidError{err: err}
}

type invalidError struct {
	err error
}

func (err *invalidError) Error() string {
	return err.err.Error()
}

func (err *invalidError) HTTPStatusCode() int {
	return http.StatusBadRequest
}

func (err *invalidError) IsRetryable() bool {
	return false
}

func (err *invalidError) Cause() error {
	return err.err
}

// ConflictError marks an error as "conflict". The calling http handler
// should return a Conflict status code. It returns nil if the error is nil.
func ConflictError(err error) error {
	if err == nil {
		return nil
	}
	return &conflictError{err: err}
}

type conflictError struct {
	err error
}

func (err *conflictError) Error() string {
	return err.err.Error()
}

func (err *conflictError) HTTPStatusCode() int {
	return http.StatusConflict
}

func (err *conflictError) IsRetryable() bool {
	return false
}

func (err *conflictError) Cause() error {
	return err.err
}
<commit_msg>add TLS handshake timeout<commit_after>package errorutil

import (
	"errors"
	"net/http"
)

// HTTPStatusCodeEr defines errors that should return a specific HTTP status code
type HTTPStatusCodeEr interface {
	HTTPStatusCode() int
}

// StatusCodeEr defines errors that should return a specific HTTP status code
type StatusCodeEr interface {
	StatusCode() int
}

// HTTPStatusCode returns the status code that a HTTP handler should return.
//
// If the error is nil, StatusOK is returned.
//
// If the error implements HTTPStatusCodeEr or StatusCodeEr, it returns the corresponding status code.
//
// It tries to check some stdlib errors (testing the error string, to avoid importing unwanted packages),
// and returns appropriate status codes.
//
// Otherwise, StatusInternalServerError is returned.
func HTTPStatusCode(err error) int {
	if err == nil {
		return http.StatusOK
	}
	type causer interface {
		Cause() error
	}

	for err != nil {
		if status, ok := err.(HTTPStatusCodeEr); ok {
			return status.HTTPStatusCode()
		}
		if status, ok := err.(StatusCodeEr); ok {
			return status.StatusCode()
		}
		// Check errors from stdlib. Test string to avoid importing packages
		switch err.Error() {
		// package os
		case "permission denied":
			return http.StatusForbidden
		case "file does not exist":
			return http.StatusNotFound
		case "storage: bucket doesn't exist":
			return http.StatusNotFound
		case "storage: object doesn't exist":
			return http.StatusNotFound
		// package database/sql
		case "sql: no rows in result set":
			return http.StatusNotFound
		case "i/o timeout", "TLS handshake timeout":
			return http.StatusRequestTimeout
		}
		cause, ok := err.(causer)
		if ok {
			err = cause.Cause()
		} else {
			err = errors.Unwrap(err)
		}
	}
	return http.StatusInternalServerError
}

// HTTPError builds an error based on a http.Response. If status code is < 300 or 304, nil is returned.
// Otherwise, errors implementing the various interfaces (Retryabler, HTTPStatusCodeEr) are returned
func HTTPError(resp *http.Response) error {
	if resp.StatusCode < 300 || resp.StatusCode == http.StatusNotModified {
		return nil
	}
	return httpError(resp.StatusCode)
}

type httpError int

func (err httpError) Error() string {
	switch err {
	case 429:
		return "Too Many Requests"
	default:
		return http.StatusText(int(err))
	}
}

func (err httpError) HTTPStatusCode() int {
	return int(err)
}

func (err httpError) StatusCode() int {
	return int(err)
}
func (err httpError) Retryable() bool {
	switch int(err) {
	case http.StatusBadGateway, http.StatusGatewayTimeout, http.StatusServiceUnavailable, http.StatusInternalServerError:
		return true
	case 429:
		return true
	default:
		return false
	}
}

// NotFoundError marks an error as "not found". The calling http handler
// should return a StatusNotFound status code. It returns nil if the error is nil.
func NotFoundError(err error) error {
	if err == nil {
		return nil
	}
	return &notFoundError{err: err}
}

type notFoundError struct {
	err error
}

func (err *notFoundError) Error() string {
	return err.err.Error()
}

func (err *notFoundError) HTTPStatusCode() int {
	return http.StatusNotFound
}

func (err *notFoundError) IsRetryable() bool {
	return false
}

func (err *notFoundError) Cause() error {
	return err.err
}

// ForbiddenError marks an error as "access forbidden". The calling http handler
// should return a StatusForbidden status code. It returns nil if the error is nil.
func ForbiddenError(err error) error {
	if err == nil {
		return nil
	}
	return &forbiddenError{err: err}
}

type forbiddenError struct {
	err error
}

func (err *forbiddenError) Error() string {
	return err.err.Error()
}

func (err *forbiddenError) HTTPStatusCode() int {
	return http.StatusForbidden
}

func (err *forbiddenError) IsRetryable() bool {
	return false
}

func (err *forbiddenError) Cause() error {
	return err.err
}

// InvalidError marks an error as "invalid". The calling http handler
// should return a StatusBadRequest status code. It returns nil if the error is nil.
func InvalidError(err error) error {
	if err == nil {
		return nil
	}
	return &invalidError{err: err}
}

type invalidError struct {
	err error
}

func (err *invalidError) Error() string {
	return err.err.Error()
}

func (err *invalidError) HTTPStatusCode() int {
	return http.StatusBadRequest
}

func (err *invalidError) IsRetryable() bool {
	return false
}

func (err *invalidError) Cause() error {
	return err.err
}

// ConflictError marks an error as "conflict". The calling http handler
// should return a Conflict status code. It returns nil if the error is nil.
func ConflictError(err error) error {
	if err == nil {
		return nil
	}
	return &conflictError{err: err}
}

type conflictError struct {
	err error
}

func (err *conflictError) Error() string {
	return err.err.Error()
}

func (err *conflictError) HTTPStatusCode() int {
	return http.StatusConflict
}

func (err *conflictError) IsRetryable() bool {
	return false
}

func (err *conflictError) Cause() error {
	return err.err
}
<|endoftext|>


<commit_before>package typhon

import (
	"io"
	"net"
	"net/http"
	"net/textproto"
	"os"
	"sync"
	"syscall"

	"github.com/monzo/slog"
	"golang.org/x/net/http/httpguts"
)

const (
	// chunkThreshold is a byte threshold above which request and response bodies that result from using buffered I/O
	// within Typhon will be transferred with chunked encoding on the wire.
	chunkThreshold = 5 * 1000000 // 5 megabytes
)

var httpChunkBufPool = sync.Pool{
	New: func() interface{} {
		return make([]byte, 32*1024) // size is the same as io.Copy uses internally
	}}

func isStreamingRsp(rsp Response) bool {
	// Most straightforward: service may have set rsp.Body to a streamer
	if s, ok := rsp.Body.(*streamer); ok && s != nil {
		return true
	}

	// If the content length is unknown, it should stream
	if rsp.ContentLength <= 0 {
		return true
	}

	// If the response body is the same as the request body and the request is streaming, the response should be too
	if rsp.Request != nil && rsp.Request.ContentLength <= 0 && rsp.Body == rsp.Request.Body {
		return true
	}

	// Chunked transfer encoding (only in HTTP/1.1) gives us an additional clue
	if !rsp.ProtoAtLeast(2, 0) {
		if httpguts.HeaderValuesContainsToken(rsp.Header[textproto.CanonicalMIMEHeaderKey("Transfer-Encoding")], "chunked") {
			return true
		}
		// Annoyingly, this can be removed from headers by net/http and promoted to its own field
		for _, v := range rsp.TransferEncoding {
			if v == "chunked" {
				return true
			}
		}
	}

	return false
}

// copyErrSeverity returns a slog error severity that should be used to report an error from an io.Copy operation to
// send the response body to a client. This exists because these errors often do not indicate actual problems. For
// example, a client may disconnect before the response body is copied to it; this doesn't mean the server is
// misbehaving.
func copyErrSeverity(err error) slog.Severity {
	// Annoyingly, these errors can be deeply nested; &net.OpError{&os.SyscallError{syscall.Errno}}
	switch err := err.(type) {
	case syscall.Errno:
		return copyErrnoSeverity(err) // platform-specific

	case *os.SyscallError:
		return copyErrSeverity(err.Err)

	case *net.OpError:
		return copyErrSeverity(err.Err)

	default:
		return slog.WarnSeverity
	}
}

// HttpHandler transforms the given Service into a standard library HTTP handler. It is one of the main "bridges"
// between Typhon and net/http.
func HttpHandler(svc Service) http.Handler {
	return http.HandlerFunc(func(rw http.ResponseWriter, httpReq *http.Request) {
		if httpReq.Body != nil {
			defer httpReq.Body.Close()
		}

		req := Request{
			Context: httpReq.Context(),
			Request: *httpReq}
		if h, ok := rw.(http.Hijacker); ok {
			req.hijacker = h
		}
		rsp := svc(req)

		// If the connection was hijacked, we should not attempt to write anything out
		if rsp.hijacked {
			return
		}

		rwHeader := rw.Header()
		for k, v := range rsp.Header {
			rwHeader[k] = v
		}
		rw.WriteHeader(rsp.StatusCode)
		if rsp.Body != nil {
			defer rsp.Body.Close()
			buf := httpChunkBufPool.Get().([]byte)
			defer httpChunkBufPool.Put(buf)
			if isStreamingRsp(rsp) {
				// Streaming responses use copyChunked(), which takes care of flushing transparently
				if _, err := copyChunked(rw, rsp.Body, buf); err != nil {
					slog.Log(slog.Eventf(copyErrSeverity(err), req, "Couldn't send streaming response body: %v", err))
				}
			} else {
				if _, err := io.CopyBuffer(rw, rsp.Body, buf); err != nil {
					slog.Log(slog.Eventf(copyErrSeverity(err), req, "Couldn't send response body: %v", err))
				}
			}
		}
	})
}
<commit_msg>httpChunkBufPool return pointers to byte slices<commit_after>package typhon

import (
	"io"
	"net"
	"net/http"
	"net/textproto"
	"os"
	"sync"
	"syscall"

	"github.com/monzo/slog"
	"golang.org/x/net/http/httpguts"
)

const (
	// chunkThreshold is a byte threshold above which request and response bodies that result from using buffered I/O
	// within Typhon will be transferred with chunked encoding on the wire.
	chunkThreshold = 5 * 1000000 // 5 megabytes
)

var httpChunkBufPool = sync.Pool{
	New: func() interface{} {
		buf := make([]byte, 32*1024) // size is the same as io.Copy uses internally
		return &buf
	}}

func isStreamingRsp(rsp Response) bool {
	// Most straightforward: service may have set rsp.Body to a streamer
	if s, ok := rsp.Body.(*streamer); ok && s != nil {
		return true
	}

	// If the content length is unknown, it should stream
	if rsp.ContentLength <= 0 {
		return true
	}

	// If the response body is the same as the request body and the request is streaming, the response should be too
	if rsp.Request != nil && rsp.Request.ContentLength <= 0 && rsp.Body == rsp.Request.Body {
		return true
	}

	// Chunked transfer encoding (only in HTTP/1.1) gives us an additional clue
	if !rsp.ProtoAtLeast(2, 0) {
		if httpguts.HeaderValuesContainsToken(rsp.Header[textproto.CanonicalMIMEHeaderKey("Transfer-Encoding")], "chunked") {
			return true
		}
		// Annoyingly, this can be removed from headers by net/http and promoted to its own field
		for _, v := range rsp.TransferEncoding {
			if v == "chunked" {
				return true
			}
		}
	}

	return false
}

// copyErrSeverity returns a slog error severity that should be used to report an error from an io.Copy operation to
// send the response body to a client. This exists because these errors often do not indicate actual problems. For
// example, a client may disconnect before the response body is copied to it; this doesn't mean the server is
// misbehaving.
func copyErrSeverity(err error) slog.Severity {
	// Annoyingly, these errors can be deeply nested; &net.OpError{&os.SyscallError{syscall.Errno}}
	switch err := err.(type) {
	case syscall.Errno:
		return copyErrnoSeverity(err) // platform-specific

	case *os.SyscallError:
		return copyErrSeverity(err.Err)

	case *net.OpError:
		return copyErrSeverity(err.Err)

	default:
		return slog.WarnSeverity
	}
}

// HttpHandler transforms the given Service into a standard library HTTP handler. It is one of the main "bridges"
// between Typhon and net/http.
func HttpHandler(svc Service) http.Handler {
	return http.HandlerFunc(func(rw http.ResponseWriter, httpReq *http.Request) {
		if httpReq.Body != nil {
			defer httpReq.Body.Close()
		}

		req := Request{
			Context: httpReq.Context(),
			Request: *httpReq}
		if h, ok := rw.(http.Hijacker); ok {
			req.hijacker = h
		}
		rsp := svc(req)

		// If the connection was hijacked, we should not attempt to write anything out
		if rsp.hijacked {
			return
		}

		rwHeader := rw.Header()
		for k, v := range rsp.Header {
			rwHeader[k] = v
		}
		rw.WriteHeader(rsp.StatusCode)
		if rsp.Body != nil {
			defer rsp.Body.Close()
			buf := *httpChunkBufPool.Get().(*[]byte)
			defer httpChunkBufPool.Put(&buf)
			if isStreamingRsp(rsp) {
				// Streaming responses use copyChunked(), which takes care of flushing transparently
				if _, err := copyChunked(rw, rsp.Body, buf); err != nil {
					slog.Log(slog.Eventf(copyErrSeverity(err), req, "Couldn't send streaming response body: %v", err))
				}
			} else {
				if _, err := io.CopyBuffer(rw, rsp.Body, buf); err != nil {
					slog.Log(slog.Eventf(copyErrSeverity(err), req, "Couldn't send response body: %v", err))
				}
			}
		}
	})
}
<|endoftext|>


<commit_before>package nethttp

import (
	"bytes"
	"errors"
	"fmt"
	"go/format"
	"io"
	"log"
	"path/filepath"
	"sort"
	"strconv"
	"strings"

	"github.com/lestrrat/go-hsup/internal/genutil"
	"github.com/lestrrat/go-jshschema"
	"github.com/lestrrat/go-jsschema"
	"github.com/lestrrat/go-jsval"
	"github.com/lestrrat/go-pdebug"
)

type Builder struct {
	AppPkg       string
	ValidatorPkg string
}

type genctx struct {
	apppkg             string
	schema             *hschema.HyperSchema
	clientpkg          string
	validatorpkg       string
	methods            map[string]string
	methodPayloadType  map[string]string
	methodNames        []string
	pathToMethods      map[string]string
	requestValidators  map[string]*jsval.JSVal
	responseValidators map[string]*jsval.JSVal
}

func New() *Builder {
	return &Builder{
		AppPkg:       "app",
		ValidatorPkg: "validator",
	}
}

func (b *Builder) ProcessFile(f string) error {
	log.Printf(" ===> Using schema file '%s'", f)
	s, err := hschema.ReadFile(f)
	if err != nil {
		return err
	}
	return b.Process(s)
}

func (b *Builder) Process(s *hschema.HyperSchema) error {
	ctx := genctx{
		schema:             s,
		methodNames:        make([]string, len(s.Links)),
		apppkg:             b.AppPkg,
		clientpkg:          "client",
		validatorpkg:       b.ValidatorPkg,
		methods:            make(map[string]string),
		methodPayloadType:  make(map[string]string),
		pathToMethods:      make(map[string]string),
		requestValidators:  make(map[string]*jsval.JSVal),
		responseValidators: make(map[string]*jsval.JSVal),
	}

	if err := parse(&ctx, s); err != nil {
		return err
	}

	if err := generateFiles(&ctx); err != nil {
		return err
	}

	log.Printf(" <=== All files generated")
	return nil
}

func parse(ctx *genctx, s *hschema.HyperSchema) error {
	for i, link := range s.Links {
		methodName := genutil.TitleToName(link.Title)
		ctx.methodPayloadType[methodName] = "interface{}"

		// Got to do this first, because validators are used in makeMethod()
		if s := link.Schema; s != nil {
			v, err := genutil.MakeValidator(s, ctx.schema)
			if err != nil {
				return err
			}
			v.Name = fmt.Sprintf("HTTP%sRequest", methodName)
			ctx.requestValidators[methodName] = v
		}
		if s := link.TargetSchema; s != nil {
			v, err := genutil.MakeValidator(s, ctx.schema)
			if err != nil {
				return err
			}
			v.Name = fmt.Sprintf("HTTP%sResponse", methodName)
			ctx.responseValidators[methodName] = v
		}
		if ls := link.Schema; ls != nil {
			if !ls.IsResolved() {
				rs, err := ls.Resolve(ctx.schema)
				if err != nil {
					return err
				}
				ls = rs
			}
			if pdebug.Enabled {
				pdebug.Printf("checking extras for %s: %#v", link.Path(), ls.Extras)
			}
			if gt, ok := ls.Extras["gotype"]; ok {
				ctx.methodPayloadType[methodName] = gt.(string)
			}
		}
		ctx.methodNames[i] = methodName
		ctx.pathToMethods[link.Path()] = methodName
		methodBody, err := makeMethod(ctx, methodName, link)
		if err != nil {
			return err
		}
		ctx.methods[methodName] = methodBody
	}

	sort.Strings(ctx.methodNames)
	return nil
}

func makeMethod(ctx *genctx, name string, l *hschema.Link) (string, error) {
	buf := bytes.Buffer{}

	fmt.Fprintf(&buf, `func http%s(w http.ResponseWriter, r *http.Request) {`+"\n", name)
	method := strings.ToLower(l.Method)
	if method == "" {
		method = "get"
	}
	buf.WriteString("if strings.ToLower(r.Method) != `")
	fmt.Fprintf(&buf, "%s", method)
	buf.WriteString("` {\nhttp.Error(w, `Not Found`, http.StatusNotFound)\n}\n")

	if v := ctx.requestValidators[name]; v != nil {
		payloadType := ctx.methodPayloadType[name]
		if method == "get" {
			// If this is a get request, then we'd have to assemble
			// the incoming data from r.Form
			if payloadType == "interface{}" {
				buf.WriteString("\nvar payload map[string]interface{}")
				for k, v := range l.Schema.Properties {
					if !v.IsResolved() {
						rv, err := v.Resolve(ctx.schema)
						if err != nil {
							return "", err
						}
						v = rv
					}

					if len(v.Type) != 1 {
						return "", errors.New("'" + name + "' can't handle input parameters unless the type contains exactly 1 type")
					}

					qk := strconv.Quote(k)
					buf.WriteString("\n{")
					switch v.Type[0] {
					case schema.IntegerType:
						fmt.Fprintf(&buf, "\nv, err := getInteger(r.Form, %s)", qk)
						fmt.Fprintf(&buf, `i
if err != nil {
	http.Error(w, "Invalid parameter " + %s, http.StatusInternalServerError)
	return
}
`, strconv.Quote(k))
					case schema.StringType:
						fmt.Fprintf(&buf, "\nv := r.Form[%s]", qk)
					}
					fmt.Fprintf(&buf, `
switch len(v) {
case 0:
	continue
case 1:
	payload[%s] = v[0]
default:
	payload[%s] = v
}
}
`, qk, qk)
				}
			}
		} else {
			buf.WriteString("\nvar payload ")
			if genutil.LooksLikeStruct(payloadType) {
				buf.WriteRune('*')
			}
			buf.WriteString(payloadType)
		}
		buf.WriteString("\nif err := json.NewDecoder(r.Body).Decode(payload); err != nil {")
		buf.WriteString("\nhttp.Error(w, `Invalid input`, http.StatusInternalServerError)")
		buf.WriteString("\nreturn")
		buf.WriteString("\n}")
		fmt.Fprintf(&buf, "\n\nif err := %s.%s.Validate(payload); err != nil {", ctx.validatorpkg, v.Name)
		buf.WriteString("\nhttp.Error(w, `Invalid input`, http.StatusInternalServerError)")
		buf.WriteString("\nreturn")
		buf.WriteString("\n}")
	}

	fmt.Fprintf(&buf, "\ndo%s(context.Background(), w, r, payload)", name)
	buf.WriteString("\n}\n")

	return buf.String(), nil
}

func generateFile(ctx *genctx, fn string, cb func(io.Writer, *genctx) error) error {
	log.Printf(" + Generating file '%s'", fn)
	f, err := genutil.CreateFile(fn)
	if err != nil {
		return err
	}
	defer f.Close()
	return cb(f, ctx)
}

func generateFiles(ctxif interface{}) error {
	ctx, ok := ctxif.(*genctx)
	if !ok {
		return errors.New("expected genctx type")
	}

	{
		fn := filepath.Join(ctx.apppkg, fmt.Sprintf("%s.go", ctx.apppkg))
		if err := generateFile(ctx, fn, generateServerCode); err != nil {
			return err
		}
	}

	{
		fn := filepath.Join(ctx.apppkg, ctx.validatorpkg, fmt.Sprintf("%s.go", ctx.validatorpkg))
		if err := generateFile(ctx, fn, generateValidatorCode); err != nil {
			return err
		}
	}

	return nil
}

func generateServerCode(out io.Writer, ctx *genctx) error {
	buf := bytes.Buffer{}

	fmt.Fprintf(&buf, "package %s\n\n", ctx.apppkg)

	genutil.WriteImports(
		&buf,
		[]string{
			"encoding/json",
			"net/http",
			"net/url",
			"strconv",
			"strings",
		},
		[]string{
			"github.com/gorilla/mux",
			"golang.org/x/net/context",
		},
	)

	buf.WriteString(`
type Server struct {
	*mux.Router
}

func New() *Server {
	s := &Server{
		Router: mux.NewRouter(),
	}
	s.SetupRoutes()
	return s
}

func getInteger(v url.Values, f string) ([]int64, error) {
	x, ok := v[f]
	if !ok {
		return nil, nil
	}

	ret := make([]int64, len(x))
	for i, e := range x {
		p, err := strconv.ParseInt(e, 10, 64)
		if err != nil {
			return nil, err
		}
		ret[i] = p
	}

	return ret, nil
}

`)

	for _, methodName := range ctx.methodNames {
		buf.WriteString(ctx.methods[methodName])
		buf.WriteString("\n")
	}

	buf.WriteString("func (s *Server) SetupRoutes() {")
	buf.WriteString("\nr := s.Router")
	paths := make([]string, 0, len(ctx.pathToMethods))
	for path := range ctx.pathToMethods {
		paths = append(paths, path)
	}
	sort.Strings(paths)
	for _, path := range paths {
		method := ctx.pathToMethods[path]
		fmt.Fprintf(&buf, "\nr.HandleFunc(`%s`, %s)", path, method)
	}
	buf.WriteString("\n}\n")

	fsrc, err := format.Source(buf.Bytes())
	if err != nil {
		return err
	}

	if _, err := out.Write(fsrc); err != nil {
		return err
	}
	return nil
}

func generateValidatorCode(out io.Writer, ctx *genctx) error {
	g := jsval.NewGenerator()
	validators := make([]*jsval.JSVal, 0, len(ctx.requestValidators)+len(ctx.responseValidators))
	for _, v := range ctx.requestValidators {
		validators = append(validators, v)
	}
	for _, v := range ctx.responseValidators {
		validators = append(validators, v)
	}

	buf := bytes.Buffer{}
	buf.WriteString("package " + ctx.validatorpkg + "\n\n")

	genutil.WriteImports(
		&buf,
		nil,
		[]string{
			"github.com/lestrrat/go-jsval",
		},
	)
	if err := g.Process(&buf, validators...); err != nil {
		return err
	}
	buf.WriteString("\n\n")

	fsrc, err := format.Source(buf.Bytes())
	if err != nil {
		return err
	}

	if _, err := out.Write(fsrc); err != nil {
		return err
	}

	return nil
}
<commit_msg>Generate scaffold for callbacks<commit_after>package nethttp

import (
	"bytes"
	"errors"
	"fmt"
	"go/format"
	"io"
	"log"
	"path/filepath"
	"sort"
	"strconv"
	"strings"

	"github.com/lestrrat/go-hsup/internal/genutil"
	"github.com/lestrrat/go-jshschema"
	"github.com/lestrrat/go-jsschema"
	"github.com/lestrrat/go-jsval"
	"github.com/lestrrat/go-pdebug"
)

type Builder struct {
	AppPkg       string
	ValidatorPkg string
}

type genctx struct {
	apppkg             string
	schema             *hschema.HyperSchema
	clientpkg          string
	validatorpkg       string
	methods            map[string]string
	methodPayloadType  map[string]string
	methodNames        []string
	pathToMethods      map[string]string
	requestValidators  map[string]*jsval.JSVal
	responseValidators map[string]*jsval.JSVal
}

func New() *Builder {
	return &Builder{
		AppPkg:       "app",
		ValidatorPkg: "validator",
	}
}

func (b *Builder) ProcessFile(f string) error {
	log.Printf(" ===> Using schema file '%s'", f)
	s, err := hschema.ReadFile(f)
	if err != nil {
		return err
	}
	return b.Process(s)
}

func (b *Builder) Process(s *hschema.HyperSchema) error {
	ctx := genctx{
		schema:             s,
		methodNames:        make([]string, len(s.Links)),
		apppkg:             b.AppPkg,
		clientpkg:          "client",
		validatorpkg:       b.ValidatorPkg,
		methods:            make(map[string]string),
		methodPayloadType:  make(map[string]string),
		pathToMethods:      make(map[string]string),
		requestValidators:  make(map[string]*jsval.JSVal),
		responseValidators: make(map[string]*jsval.JSVal),
	}

	if err := parse(&ctx, s); err != nil {
		return err
	}

	if err := generateFiles(&ctx); err != nil {
		return err
	}

	log.Printf(" <=== All files generated")
	return nil
}

func parse(ctx *genctx, s *hschema.HyperSchema) error {
	for i, link := range s.Links {
		methodName := genutil.TitleToName(link.Title)
		ctx.methodPayloadType[methodName] = "interface{}"

		// Got to do this first, because validators are used in makeMethod()
		if s := link.Schema; s != nil {
			v, err := genutil.MakeValidator(s, ctx.schema)
			if err != nil {
				return err
			}
			v.Name = fmt.Sprintf("HTTP%sRequest", methodName)
			ctx.requestValidators[methodName] = v
		}
		if s := link.TargetSchema; s != nil {
			v, err := genutil.MakeValidator(s, ctx.schema)
			if err != nil {
				return err
			}
			v.Name = fmt.Sprintf("HTTP%sResponse", methodName)
			ctx.responseValidators[methodName] = v
		}
		if ls := link.Schema; ls != nil {
			if !ls.IsResolved() {
				rs, err := ls.Resolve(ctx.schema)
				if err != nil {
					return err
				}
				ls = rs
			}
			if pdebug.Enabled {
				pdebug.Printf("checking extras for %s: %#v", link.Path(), ls.Extras)
			}
			if gt, ok := ls.Extras["gotype"]; ok {
				ctx.methodPayloadType[methodName] = gt.(string)
			}
		}
		ctx.methodNames[i] = methodName
		ctx.pathToMethods[link.Path()] = methodName
		methodBody, err := makeMethod(ctx, methodName, link)
		if err != nil {
			return err
		}
		ctx.methods[methodName] = methodBody
	}

	sort.Strings(ctx.methodNames)
	return nil
}

func makeMethod(ctx *genctx, name string, l *hschema.Link) (string, error) {
	buf := bytes.Buffer{}

	fmt.Fprintf(&buf, `func http%s(w http.ResponseWriter, r *http.Request) {`+"\n", name)
	method := strings.ToLower(l.Method)
	if method == "" {
		method = "get"
	}
	buf.WriteString("if strings.ToLower(r.Method) != `")
	fmt.Fprintf(&buf, "%s", method)
	buf.WriteString("` {\nhttp.Error(w, `Not Found`, http.StatusNotFound)\n}\n")

	if v := ctx.requestValidators[name]; v != nil {
		payloadType := ctx.methodPayloadType[name]
		if method == "get" {
			// If this is a get request, then we'd have to assemble
			// the incoming data from r.Form
			if payloadType == "interface{}" {
				buf.WriteString("\nvar payload map[string]interface{}")
				for k, v := range l.Schema.Properties {
					if !v.IsResolved() {
						rv, err := v.Resolve(ctx.schema)
						if err != nil {
							return "", err
						}
						v = rv
					}

					if len(v.Type) != 1 {
						return "", errors.New("'" + name + "' can't handle input parameters unless the type contains exactly 1 type")
					}

					qk := strconv.Quote(k)
					buf.WriteString("\n{")
					switch v.Type[0] {
					case schema.IntegerType:
						fmt.Fprintf(&buf, "\nv, err := getInteger(r.Form, %s)", qk)
						fmt.Fprintf(&buf, `i
if err != nil {
	http.Error(w, "Invalid parameter " + %s, http.StatusInternalServerError)
	return
}
`, strconv.Quote(k))
					case schema.StringType:
						fmt.Fprintf(&buf, "\nv := r.Form[%s]", qk)
					}
					fmt.Fprintf(&buf, `
switch len(v) {
case 0:
case 1:
	payload[%s] = v[0]
default:
	payload[%s] = v
}
}
`, qk, qk)
				}
			}
		} else {
			buf.WriteString("\nvar payload ")
			if genutil.LooksLikeStruct(payloadType) {
				buf.WriteRune('*')
			}
			buf.WriteString(payloadType)
		}
		buf.WriteString("\nif err := json.NewDecoder(r.Body).Decode(payload); err != nil {")
		buf.WriteString("\nhttp.Error(w, `Invalid input`, http.StatusInternalServerError)")
		buf.WriteString("\nreturn")
		buf.WriteString("\n}")
		fmt.Fprintf(&buf, "\n\nif err := %s.%s.Validate(payload); err != nil {", ctx.validatorpkg, v.Name)
		buf.WriteString("\nhttp.Error(w, `Invalid input`, http.StatusInternalServerError)")
		buf.WriteString("\nreturn")
		buf.WriteString("\n}")
	}

	fmt.Fprintf(&buf, "\ndo%s(context.Background(), w, r, payload)", name)
	buf.WriteString("\n}\n")

	return buf.String(), nil
}

func generateFile(ctx *genctx, fn string, cb func(io.Writer, *genctx) error) error {
	log.Printf(" + Generating file '%s'", fn)
	f, err := genutil.CreateFile(fn)
	if err != nil {
		return err
	}
	defer f.Close()
	return cb(f, ctx)
}

func generateFiles(ctxif interface{}) error {
	ctx, ok := ctxif.(*genctx)
	if !ok {
		return errors.New("expected genctx type")
	}

	{
		fn := filepath.Join(ctx.apppkg, fmt.Sprintf("%s.go", ctx.apppkg))
		if err := generateFile(ctx, fn, generateServerCode); err != nil {
			return err
		}
	}

	{
		fn := filepath.Join(ctx.apppkg, ctx.validatorpkg, fmt.Sprintf("%s.go", ctx.validatorpkg))
		if err := generateFile(ctx, fn, generateValidatorCode); err != nil {
			return err
		}
	}

	return nil
}

func generateServerCode(out io.Writer, ctx *genctx) error {
	buf := bytes.Buffer{}

	fmt.Fprintf(&buf, "package %s\n\n", ctx.apppkg)

	genutil.WriteImports(
		&buf,
		[]string{
			"encoding/json",
			"net/http",
			"net/url",
			"strconv",
			"strings",
		},
		[]string{
			"github.com/gorilla/mux",
			"golang.org/x/net/context",
		},
	)

	buf.WriteString(`
type Server struct {
	*mux.Router
}

func New() *Server {
	s := &Server{
		Router: mux.NewRouter(),
	}
	s.SetupRoutes()
	return s
}

func getInteger(v url.Values, f string) ([]int64, error) {
	x, ok := v[f]
	if !ok {
		return nil, nil
	}

	ret := make([]int64, len(x))
	for i, e := range x {
		p, err := strconv.ParseInt(e, 10, 64)
		if err != nil {
			return nil, err
		}
		ret[i] = p
	}

	return ret, nil
}

`)

	for _, methodName := range ctx.methodNames {
		buf.WriteString(ctx.methods[methodName])
		buf.WriteString("\n")
	}

	buf.WriteString("func (s *Server) SetupRoutes() {")
	buf.WriteString("\nr := s.Router")
	paths := make([]string, 0, len(ctx.pathToMethods))
	for path := range ctx.pathToMethods {
		paths = append(paths, path)
	}
	sort.Strings(paths)
	for _, path := range paths {
		method := ctx.pathToMethods[path]
		fmt.Fprintf(&buf, "\nr.HandleFunc(`%s`, http%s)", path, method)
	}
	buf.WriteString("\n}\n")

	for _, methodName := range ctx.methodNames {
		payloadType := ctx.methodPayloadType[methodName]

		fmt.Fprintf(&buf, "\nfunc do%s(ctx context.Context, w http.ResponseWriter, r *http.Request, payload ", methodName)
		if genutil.LooksLikeStruct(payloadType) {
			buf.WriteRune('*')
		}
		buf.WriteString(payloadType)
		buf.WriteString(") {")
		buf.WriteString("\n}\n")
	}

	fsrc, err := format.Source(buf.Bytes())
	if err != nil {
		return err
	}

	if _, err := out.Write(fsrc); err != nil {
		return err
	}
	return nil
}

func generateValidatorCode(out io.Writer, ctx *genctx) error {
	g := jsval.NewGenerator()
	validators := make([]*jsval.JSVal, 0, len(ctx.requestValidators)+len(ctx.responseValidators))
	for _, v := range ctx.requestValidators {
		validators = append(validators, v)
	}
	for _, v := range ctx.responseValidators {
		validators = append(validators, v)
	}

	buf := bytes.Buffer{}
	buf.WriteString("package " + ctx.validatorpkg + "\n\n")

	genutil.WriteImports(
		&buf,
		nil,
		[]string{
			"github.com/lestrrat/go-jsval",
		},
	)
	if err := g.Process(&buf, validators...); err != nil {
		return err
	}
	buf.WriteString("\n\n")

	fsrc, err := format.Source(buf.Bytes())
	if err != nil {
		return err
	}

	if _, err := out.Write(fsrc); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// Package imap partially implements the Internet Message Access Protocol as
// defined in RFC 3501.
//
// This implementation is thread-safe, but does not take advantage of all the
// parallelism provided for in the standard. Commands will be performed
// sequentially.
//
// Untagged IMAP responses are parsed into a Mailbox struct, which tracks all
// currently known information concerning the state of the remote mailbox.

package main

import (
	"crypto/tls"
	"errors"
	"fmt"
	"net"
	tp "net/textproto"
	"strings"
)

type Client struct {
	Text *tp.Conn
	Box  *Mailbox
	conn net.Conn
	host string
}

// Represents the current known state of the remote mailbox.
type Mailbox struct {
	capabilities []string
}

func (m *Mailbox) Capable(c string) bool {
	for _, ca := range m.capabilities {
		if c == ca {
			return true
		}
	}
	return false
}

func Dial(addr string) (*Client, error) {
	conn, err := net.Dial("tcp", addr)
	if err != nil {
		return nil, err
	}
	host := addr[:strings.Index(addr, ":")]
	return NewClient(conn, host)
}

func DialTLS(addr string) (*Client, error) {
	conn, err := tls.Dial("tcp", addr, nil)
	if err != nil {
		return nil, err
	}
	host := addr[:strings.Index(addr, ":")]
	return NewClient(conn, host)
}

// NewClient returns a new Client using an existing connection.
func NewClient(conn net.Conn, host string) (*Client, error) {
	text := tp.NewConn(conn)
	client := &Client{
		Text: text,
		Box:  &Mailbox{
			capabilities: []string{},
		},
		conn: conn,
		host: host,
	}
	return client, nil
}

func (c *Client) handleUntagged(l string) {
	switch l[0:strings.Index(l, " ")] {
	case "CAPABILITY":
		c.Box.capabilities = strings.Split(l, " ")[1:]
	default:
		println(l)
	}
}

func (c *Client) Cmd(format string, args ...interface{}) error {
	t := c.Text
	id := t.Next()
	t.StartRequest(id)
	err := t.PrintfLine("x%d %s", id, fmt.Sprintf(format, args...))
	if err != nil {
		return err
	}
	t.EndRequest(id)

	t.StartResponse(id)
	defer t.EndResponse(id)

	l, err := t.ReadLine()
	if err != nil {
		return err
	}
	for isUntagged(l) {
		c.handleUntagged(l[2:])
		l, err = t.ReadLine()
		if err != nil {
			return err
		}
	}

	// TODO make sure the tagged response ends up in the right place. There
	// really should be a goroutine to handle all of this.
	l = strings.SplitN(l, " ", 2)[1]
	if l[0:2] == "OK" {
		return nil
	}
	return errors.New(l)
}

func isUntagged(l string) bool {
	return l[0:2] == "* "
}

// Noop sends a NOOP command to the server, which may be abused to test that
// the connection is still working, or keep it active.
func (c *Client) Noop() error {
	return c.Cmd("NOOP")
}

// Capability determines the server's capabilities.
func (c *Client) Capability() error {
	return c.Cmd("CAPABILITY")
}

// Login authenticates a client using the provided username and password. This
// method is only secure if TLS is being used. AUTHENTICATE and STARTTLS are
// not supported.
func (c *Client) Login(username, password string) error {
	return c.Cmd("LOGIN %s %s", username, password)
}

// Logout closes the connection, after instructing the server to do the same.
func (c *Client) Logout() error {
	return c.Cmd("LOGOUT")
}

// SELECT

// EXAMINE

// CREATE

// DELETE

// RENAME

// SUBSCRIBE

// UNSUBSCRIBE

// List lists all folder within basename that match the wildcard expression mb.
// The result is put into the Client's Mailbox struct.
func (c *Client) List(basename, mb string) error {
	return c.Cmd(`LIST "%s" "%s"`, basename, mb)
}

// LSUB

// STATUS

// APPEND

<commit_msg>Comments<commit_after>// Package imap partially implements the Internet Message Access Protocol as
// defined in RFC 3501.
//
// This implementation is thread-safe, but does not take advantage of all the
// parallelism provided for in the standard. Commands will be performed
// sequentially.
//
// Untagged IMAP responses are parsed into a Mailbox struct, which tracks all
// currently known information concerning the state of the remote mailbox.

package main

import (
	"crypto/tls"
	"errors"
	"fmt"
	"net"
	tp "net/textproto"
	"strings"
)

type Client struct {
	Text *tp.Conn
	Box  *Mailbox
	conn net.Conn
	host string
}

// Represents the current known state of the remote mailbox.
type Mailbox struct {
	capabilities []string
}

func (m *Mailbox) Capable(c string) bool {
	for _, ca := range m.capabilities {
		if c == ca {
			return true
		}
	}
	return false
}

// Dial creates an unsecured connection to the IMAP server at the given address
// and returns the corresponding Client.
func Dial(addr string) (*Client, error) {
	conn, err := net.Dial("tcp", addr)
	if err != nil {
		return nil, err
	}
	host := addr[:strings.Index(addr, ":")]
	return NewClient(conn, host)
}

// DIalTLS creates a TLS_secured connection to the IMAP server at the given
// address and returns the corresponding Client.
func DialTLS(addr string) (*Client, error) {
	conn, err := tls.Dial("tcp", addr, nil)
	if err != nil {
		return nil, err
	}
	host := addr[:strings.Index(addr, ":")]
	return NewClient(conn, host)
}

// NewClient returns a new Client using an existing connection.
func NewClient(conn net.Conn, host string) (*Client, error) {
	text := tp.NewConn(conn)
	client := &Client{
		Text: text,
		Box:  &Mailbox{
			capabilities: []string{},
		},
		conn: conn,
		host: host,
	}
	return client, nil
}

func (c *Client) handleUntagged(l string) {
	switch l[0:strings.Index(l, " ")] {
	case "CAPABILITY":
		c.Box.capabilities = strings.Split(l, " ")[1:]
	default:
		println(l)
	}
}

func (c *Client) Cmd(format string, args ...interface{}) error {
	t := c.Text
	id := t.Next()
	t.StartRequest(id)
	err := t.PrintfLine("x%d %s", id, fmt.Sprintf(format, args...))
	if err != nil {
		return err
	}
	t.EndRequest(id)

	t.StartResponse(id)
	defer t.EndResponse(id)

	l, err := t.ReadLine()
	if err != nil {
		return err
	}
	for isUntagged(l) {
		c.handleUntagged(l[2:])
		l, err = t.ReadLine()
		if err != nil {
			return err
		}
	}

	// TODO make sure the tagged response ends up in the right place. There
	// really should be a goroutine to handle all of this.
	l = strings.SplitN(l, " ", 2)[1]
	if l[0:2] == "OK" {
		return nil
	}
	return errors.New(l)
}

func isUntagged(l string) bool {
	return l[0:2] == "* "
}

// Noop sends a NOOP command to the server, which may be abused to test that
// the connection is still working, or keep it active.
func (c *Client) Noop() error {
	return c.Cmd("NOOP")
}

// Capability determines the server's capabilities.
func (c *Client) Capability() error {
	return c.Cmd("CAPABILITY")
}

// Login authenticates a client using the provided username and password. This
// method is only secure if TLS is being used. AUTHENTICATE and STARTTLS are
// not supported.
func (c *Client) Login(username, password string) error {
	return c.Cmd("LOGIN %s %s", username, password)
}

// Logout closes the connection, after instructing the server to do the same.
func (c *Client) Logout() error {
	return c.Cmd("LOGOUT")
}

// SELECT

// EXAMINE

// CREATE

// DELETE

// RENAME

// SUBSCRIBE

// UNSUBSCRIBE

// List lists all folder within basename that match the wildcard expression mb.
// The result is put into the Client's Mailbox struct.
func (c *Client) List(basename, mb string) error {
	return c.Cmd(`LIST "%s" "%s"`, basename, mb)
}

// LSUB

// STATUS

// APPEND

// CHECK

// CLOSE

// EXPUNGE

// SEARCH

// FETCH

// STORE

// COPY

// UID

<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"html/template"
	"io"
	"log"
	"strings"
	"time"

	"github.com/emersion/go-imap/client"
	message "github.com/emersion/go-message"
	"github.com/emersion/go-message/mail"
	"github.com/mmcdole/gofeed"
	"github.com/spf13/viper"
)

var mailTemplate = template.Must(template.New("mail").Parse(
	`

{{ .Title }} {{ .Author }}

{{ .Content }}

`))

type templatePayload struct {
	Link    string
	Title   string
	Author  string
	Content template.HTML
}

func FormatContent(item *gofeed.Item) (string, error) {
	var payload templatePayload

	payload.Link = item.Link
	payload.Title = item.Title

	if item.Author != nil {
		payload.Author = fmt.Sprintf("%s %s", item.Author.Name, item.Author.Email)
	}

	if len(item.Content) > 0 {
		payload.Content = template.HTML(item.Content)
	} else {
		payload.Content = template.HTML(item.Description)
	}

	var buffer bytes.Buffer

	err := mailTemplate.Execute(&buffer, payload)

	if err != nil {
		return "", err
	}

	return buffer.String(), nil
}

func NewMessage(item *gofeed.Item) (bytes.Buffer, error) {
	var b bytes.Buffer

	fromName := viper.GetString("imap.from.name")

	if item.Author != nil {
		fromName = fmt.Sprintf("%s %s", item.Author.Name, item.Author.Email)
	}

	from := []*mail.Address{{fromName, viper.GetString("imap.from.email")}}
	to := []*mail.Address{{viper.GetString("imap.to.name"), viper.GetString("imap.to.email")}}

	mediaParams := map[string]string{"charset": "utf-8"}

	h := mail.NewHeader()
	h.SetContentType("multipart/alternative", mediaParams)
	h.SetDate(*item.PublishedParsed)
	h.SetAddressList("From", from)
	h.SetAddressList("To", to)
	h.SetSubject(item.Title)

	// Create a new mail writer
	messageWriter, err := message.CreateWriter(&b, h.Header)
	defer messageWriter.Close()
	if err != nil {
		return b, err
	}

	htmlHeader := mail.NewHeader()
	htmlHeader.SetContentType("text/html", mediaParams)
	htmlWriter, err := messageWriter.CreatePart(htmlHeader.Header)
	defer htmlWriter.Close()
	if err != nil {
		return b, err
	}

	content, err := FormatContent(item)
	if err != nil {
		return b, err
	}

	io.WriteString(htmlWriter, content)

	return b, nil
}

func AppendNewItemsViaIMAP(items ItemsWithFolders) error {
	if viper.GetBool("debug") {
		log.Printf("Found %d new items", len(items))
	}

	hostPort := fmt.Sprintf("%s:%d", viper.GetString("imap.host"), viper.GetInt("imap.port"))
	c, err := client.DialTLS(hostPort, nil)
	if err != nil {
		return err
	}

	defer c.Logout()

	if err := c.Login(viper.GetString("imap.username"), viper.GetString("imap.password")); err != nil {
		return err
	}

	if viper.GetBool("debug") {
		log.Println("Logged in to IMAP")
	}

	for _, entry := range items {
		if entry.Item.PublishedParsed == nil {
			t := time.Now()
			entry.Item.PublishedParsed = &t
		}

		folderName := entry.Folder
		if viper.GetBool("imap.folder_capitalize") {
			folderName = strings.Title(folderName)
		}
		folder := fmt.Sprintf("%s/%s", viper.GetString("imap.folder_prefix"), folderName)

		_ = c.Create(folder)

		msg, err := NewMessage(entry.Item)
		if err != nil {
			return err
		}

		if viper.GetBool("debug") {
			log.Printf("Appending item to %s", folder)
		}

		literal := bytes.NewReader(msg.Bytes())
		err = c.Append(folder, []string{}, *entry.Item.PublishedParsed, literal)
		if err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>Fix encoding issues<commit_after>package main

import (
	"bytes"
	"fmt"
	"html/template"
	"io"
	"log"
	"strings"
	"time"

	"github.com/emersion/go-imap/client"
	"github.com/emersion/go-message/mail"
	"github.com/mmcdole/gofeed"
	"github.com/spf13/viper"
)

var mailTemplate = template.Must(template.New("mail").Parse(
	`

{{ .Title }} {{ .Author }}

{{ .Content }}

`))

type templatePayload struct {
	Link    string
	Title   string
	Author  string
	Content template.HTML
}

func FormatContent(item *gofeed.Item) (string, error) {
	var payload templatePayload

	payload.Link = item.Link
	payload.Title = item.Title

	if item.Author != nil {
		payload.Author = fmt.Sprintf("%s %s", item.Author.Name, item.Author.Email)
	}

	if len(item.Content) > 0 {
		payload.Content = template.HTML(item.Content)
	} else {
		payload.Content = template.HTML(item.Description)
	}

	var buffer bytes.Buffer

	err := mailTemplate.Execute(&buffer, payload)

	if err != nil {
		return "", err
	}

	return buffer.String(), nil
}

func NewMessage(item *gofeed.Item) (bytes.Buffer, error) {
	var b bytes.Buffer

	fromName := viper.GetString("imap.from.name")

	if item.Author != nil {
		fromName = fmt.Sprintf("%s %s", item.Author.Name, item.Author.Email)
	}

	from := []*mail.Address{{fromName, viper.GetString("imap.from.email")}}
	to := []*mail.Address{{viper.GetString("imap.to.name"), viper.GetString("imap.to.email")}}

	mediaParams := map[string]string{"charset": "utf-8"}

	h := mail.NewHeader()
	h.SetContentType("multipart/alternative", mediaParams)
	h.SetDate(*item.PublishedParsed)
	h.SetAddressList("From", from)
	h.SetAddressList("To", to)
	h.SetSubject(item.Title)

	messageWriter, err := mail.CreateWriter(&b, h)
	defer messageWriter.Close()
	if err != nil {
		return b, err
	}

	htmlHeader := mail.NewTextHeader()
	htmlHeader.SetContentType("text/html", mediaParams)
	htmlWriter, err := messageWriter.CreateSingleText(htmlHeader)
	defer htmlWriter.Close()
	if err != nil {
		return b, err
	}

	content, err := FormatContent(item)
	if err != nil {
		return b, err
	}

	io.WriteString(htmlWriter, content)

	return b, nil
}

func AppendNewItemsViaIMAP(items ItemsWithFolders) error {
	if viper.GetBool("debug") {
		log.Printf("Found %d new items", len(items))
	}

	hostPort := fmt.Sprintf("%s:%d", viper.GetString("imap.host"), viper.GetInt("imap.port"))
	c, err := client.DialTLS(hostPort, nil)
	if err != nil {
		return err
	}

	defer c.Logout()

	if err := c.Login(viper.GetString("imap.username"), viper.GetString("imap.password")); err != nil {
		return err
	}

	if viper.GetBool("debug") {
		log.Println("Logged in to IMAP")
	}

	for _, entry := range items {
		if entry.Item.PublishedParsed == nil {
			t := time.Now()
			entry.Item.PublishedParsed = &t
		}

		folderName := entry.Folder
		if viper.GetBool("imap.folder_capitalize") {
			folderName = strings.Title(folderName)
		}
		folder := fmt.Sprintf("%s/%s", viper.GetString("imap.folder_prefix"), folderName)

		_ = c.Create(folder)

		msg, err := NewMessage(entry.Item)
		if err != nil {
			return err
		}

		if viper.GetBool("debug") {
			log.Printf("Appending item to %s", folder)
		}

		literal := bytes.NewReader(msg.Bytes())
		err = c.Append(folder, []string{}, *entry.Item.PublishedParsed, literal)
		if err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2014, Ben Morgan. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.

package main

import (
	"bytes"
	"fmt"
	"os"
	"path"
	"sort"
	"strings"

	"github.com/goulash/pacman"
	"github.com/goulash/pr"
	"github.com/goulash/util"
)

// List displays all the packages available for the database.
// Note that they don't need to be registered with the database.
func List(c *Config) error {
	// Get packages in repo directory and database and find out duplicates.
	// Depinding on the configuration, some of this might be unecessary work,
	// so we have to declare the variabes here.
	var (
		db     map[string]*pacman.Package
		missed []*pacman.Package
		dups   map[string]int
	)
	pkgs, old := pacman.SplitOld(getRepoPkgs(c))
	if c.Pending {
		db, missed = getDatabasePkgs(c)
	}
	if c.Duplicates {
		dups = make(map[string]int)
		for _, p := range old {
			dups[p.Name]++
		}
	}

	// Create a list.
	var pkgnames []string
	if c.Pending {
		for _, p := range missed {
			pkgnames = append(pkgnames, fmt.Sprintf("-%s-", p.Name))
		}
	}
	for _, p := range pkgs {
		buf := bytes.NewBufferString(p.Name)

		if c.Pending {
			dbp, ok := db[p.Name]
			if !ok || dbp.OlderThan(p) {
				buf.WriteRune('*')
			}
		}
		if c.Versioned {
			buf.WriteRune(' ')
			buf.WriteString(p.Version)
		}
		if c.Synchronize {
			aurp, err := pacman.ReadAUR(p.Name)
			if err != nil {
				c.inform(fmt.Sprintf("searching for %s: %s", p.Name, err))
				buf.WriteString(" <?>")
			} else if aurp.NewerThan(p) {
				if c.Versioned {
					buf.WriteString(" -> ")
					buf.WriteString(aurp.Version)
				}
				buf.WriteString(" <!>")
			}
		}
		if c.Duplicates && dups[p.Name] > 0 {
			buf.WriteString(fmt.Sprintf(" (%v)", dups[p.Name]))
		}

		pkgnames = append(pkgnames, buf.String())
	}

	// Print packages to stdout
	printList(c, pkgnames)
	return nil
}

func Status(c *Config) error {
	pkgs, old := pacman.SplitOld(getRepoPkgs(c))
	db, missed := getDatabasePkgs(c)
	name := c.Database[:strings.IndexByte(c.Database, '.')]
	fmt.Printf("On repo %s\n", name)

	// We assume that there is nothing to do, and if there is,
	// then this is set to false.
	var nothing = true

	if len(missed) > 0 {
		filenames := make([]string, len(missed))
		for i, p := range missed {
			filenames[i] = p.Name
		}
		printStatus("Database entries to be removed:", filenames)
		nothing = false
	}

	if len(old) > 0 {
		filenames := make([]string, len(old))
		for i, p := range old {
			filenames[i] = p.Filename
		}
		printStatus("Obsolete packages to be removed:", filenames)
		nothing = false
	}

	{
		var pending []string
		for _, p := range pkgs {
			dbp, ok := db[p.Name]
			if !ok || dbp.OlderThan(p) {
				pending = append(pending, p.Name)
			}
		}

		if len(pending) > 0 {
			printStatus("Packages pending database addition:", pending)
			nothing = false
		}
	}

	{
		var updates []string
		for _, p := range pkgs {
			aurp, err := pacman.ReadAUR(p.Name)
			if err != nil {
				c.inform(fmt.Sprintf("error searching for %s: %s", p.Name, err))
			} else if aurp.NewerThan(p) {
				updates = append(updates, p.Name)
			}
		}

		if len(updates) > 0 {
			printStatus("Outdated packages:", updates)
			nothing = false
		}
	}

	if nothing {
		fmt.Println("everything up-to-date.")
	}
	return nil
}

func printStatus(heading string, items []string) {
	sort.Strings(items)
	fmt.Println()
	fmt.Println(heading)
	for _, i := range items {
		fmt.Println(" ", i)
	}
}

func getRepoPkgs(c *Config) []*pacman.Package {
	ch := make(chan error)
	go handleErrors("warning: %s\n", ch)
	dirPkgs := pacman.ReadDir(c.RepoPath, ch)
	close(ch)
	return dirPkgs
}

// handleErrors is meant to be launched as a separate goroutine to handle
// errors coming from ReadDir and the likes.
func handleErrors(format string, ch <-chan error) {
	for err := range ch {
		fmt.Fprintf(os.Stderr, format, err)
	}
}

func getDatabasePkgs(c *Config) (db map[string]*pacman.Package, missed []*pacman.Package) {
	db = make(map[string]*pacman.Package)
	pkgs, _ := pacman.ReadDatabase(path.Join(c.RepoPath, c.Database))
	for _, p := range pkgs {
		if ex, _ := util.FileExists(p.Filename); !ex {
			missed = append(missed, p)
			continue
		}
		db[p.Name] = p
	}
	return db, missed
}

func printList(c *Config, items []string) {
	sort.Strings(items)
	if c.OnePerLine {
		for _, i := range items {
			fmt.Println(i)
		}
	} else {
		pr.PrintFlex(items)
	}
}
<commit_msg>Fix Status() with new ConcurrentlyReadAUR function from pacman<commit_after>// Copyright (c) 2014, Ben Morgan. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.

package main

import (
	"bytes"
	"fmt"
	"os"
	"path"
	"sort"
	"strings"

	"github.com/goulash/pacman"
	"github.com/goulash/pr"
	"github.com/goulash/util"
)

// List displays all the packages available for the database.
// Note that they don't need to be registered with the database.
func List(c *Config) error {
	// Get packages in repo directory and database and find out duplicates.
	// Depinding on the configuration, some of this might be unecessary work,
	// so we have to declare the variabes here.
	var (
		db     map[string]*pacman.Package
		missed []*pacman.Package
		dups   map[string]int
	)
	pkgs, old := pacman.SplitOld(getRepoPkgs(c))
	if c.Pending {
		db, missed = getDatabasePkgs(c)
	}
	if c.Duplicates {
		dups = make(map[string]int)
		for _, p := range old {
			dups[p.Name]++
		}
	}

	// Create a list.
	var pkgnames []string
	if c.Pending {
		for _, p := range missed {
			pkgnames = append(pkgnames, fmt.Sprintf("-%s-", p.Name))
		}
	}
	for _, p := range pkgs {
		buf := bytes.NewBufferString(p.Name)

		if c.Pending {
			dbp, ok := db[p.Name]
			if !ok || dbp.OlderThan(p) {
				buf.WriteRune('*')
			}
		}
		if c.Versioned {
			buf.WriteRune(' ')
			buf.WriteString(p.Version)
		}
		if c.Synchronize {
			aurp, err := pacman.ReadAUR(p.Name)
			if err != nil {
				c.inform(fmt.Sprintf("searching for %s: %s", p.Name, err))
				buf.WriteString(" <?>")
			} else if aurp.NewerThan(p) {
				if c.Versioned {
					buf.WriteString(" -> ")
					buf.WriteString(aurp.Version)
				}
				buf.WriteString(" <!>")
			}
		}
		if c.Duplicates && dups[p.Name] > 0 {
			buf.WriteString(fmt.Sprintf(" (%v)", dups[p.Name]))
		}

		pkgnames = append(pkgnames, buf.String())
	}

	// Print packages to stdout
	printList(c, pkgnames)
	return nil
}

// Status prints a thorough status of the current repository.
func Status(c *Config) error {
	pkgs, old := pacman.SplitOld(getRepoPkgs(c))
	db, missed := getDatabasePkgs(c)
	name := c.Database[:strings.IndexByte(c.Database, '.')]
	fmt.Printf("On repo %s\n", name)

	// We assume that there is nothing to do, and if there is,
	// then this is set to false.
	var nothing = true

	if len(missed) > 0 {
		filenames := make([]string, len(missed))
		for i, p := range missed {
			filenames[i] = p.Name
		}
		printStatus("Database entries to be removed:", filenames)
		nothing = false
	}

	if len(old) > 0 {
		filenames := make([]string, len(old))
		for i, p := range old {
			filenames[i] = p.Filename
		}
		printStatus("Obsolete packages to be removed:", filenames)
		nothing = false
	}

	{
		var pending []string
		for _, p := range pkgs {
			dbp, ok := db[p.Name]
			if !ok || dbp.OlderThan(p) {
				pending = append(pending, p.Name)
			}
		}

		if len(pending) > 0 {
			printStatus("Packages pending database addition:", pending)
			nothing = false
		}
	}

	{
		names := make([]string, len(pkgs))
		for i, p := range pkgs {
			names[i] = p.Name
		}
		ch := make(chan error)
		go handleErrors("warning: %s\n", ch)
		aur := pacman.ConcurrentlyReadAUR(names, 16, ch)
		close(ch)

		var updates []string
		for _, p := range pkgs {
			if ap, ok := aur[p.Name]; ok {
				if ap != nil && ap.NewerThan(p) {
					updates = append(updates, p.Name)
				} else if ap == nil {
					updates = append(updates, fmt.Sprintf("-%s-", p.Name))
				}
			}
		}

		if len(updates) > 0 {
			printStatus("Outdated packages:", updates)
			nothing = false
		}
	}

	if nothing {
		fmt.Println("everything up-to-date.")
	}
	return nil
}

func printStatus(heading string, items []string) {
	sort.Strings(items)
	fmt.Println()
	fmt.Println(heading)
	for _, i := range items {
		fmt.Println(" ", i)
	}
}

func getRepoPkgs(c *Config) []*pacman.Package {
	ch := make(chan error)
	go handleErrors("warning: %s\n", ch)
	dirPkgs := pacman.ReadDir(c.RepoPath, ch)
	close(ch)
	return dirPkgs
}

// handleErrors is meant to be launched as a separate goroutine to handle
// errors coming from ReadDir and the likes.
func handleErrors(format string, ch <-chan error) {
	for err := range ch {
		fmt.Fprintf(os.Stderr, format, err)
	}
}

func getDatabasePkgs(c *Config) (db map[string]*pacman.Package, missed []*pacman.Package) {
	db = make(map[string]*pacman.Package)
	pkgs, _ := pacman.ReadDatabase(path.Join(c.RepoPath, c.Database))
	for _, p := range pkgs {
		if ex, _ := util.FileExists(p.Filename); !ex {
			missed = append(missed, p)
			continue
		}
		db[p.Name] = p
	}
	return db, missed
}

func printList(c *Config, items []string) {
	sort.Strings(items)
	if c.OnePerLine {
		for _, i := range items {
			fmt.Println(i)
		}
	} else {
		pr.PrintFlex(items)
	}
}
<|endoftext|>


<commit_before>package common

import (
	"fmt"
	"github.com/mitchellh/go-fs"
	"github.com/mitchellh/go-fs/fat"
	"github.com/mitchellh/multistep"
	"github.com/mitchellh/packer/packer"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
)

// StepCreateFloppy will create a floppy disk with the given files.
// The floppy disk doesn't support sub-directories. Only files at the
// root level are supported.
type StepCreateFloppy struct {
	Files []string

	floppyPath string

	FilesAdded map[string]bool
}

func (s *StepCreateFloppy) Run(state multistep.StateBag) multistep.StepAction {
	if len(s.Files) == 0 {
		log.Println("No floppy files specified. Floppy disk will not be made.")
		return multistep.ActionContinue
	}

	s.FilesAdded = make(map[string]bool)

	ui := state.Get("ui").(packer.Ui)
	ui.Say("Creating floppy disk...")

	// Create a temporary file to be our floppy drive
	floppyF, err := ioutil.TempFile("", "packer")
	if err != nil {
		state.Put("error",
			fmt.Errorf("Error creating temporary file for floppy: %s", err))
		return multistep.ActionHalt
	}
	defer floppyF.Close()

	// Set the path so we can remove it later
	s.floppyPath = floppyF.Name()

	log.Printf("Floppy path: %s", s.floppyPath)

	// Set the size of the file to be a floppy sized
	if err := floppyF.Truncate(1440 * 1024); err != nil {
		state.Put("error", fmt.Errorf("Error creating floppy: %s", err))
		return multistep.ActionHalt
	}

	// BlockDevice backed by the file for our filesystem
	log.Println("Initializing block device backed by temporary file")
	device, err := fs.NewFileDisk(floppyF)
	if err != nil {
		state.Put("error", fmt.Errorf("Error creating floppy: %s", err))
		return multistep.ActionHalt
	}

	// Format the block device so it contains a valid FAT filesystem
	log.Println("Formatting the block device with a FAT filesystem...")
	formatConfig := &fat.SuperFloppyConfig{
		FATType: fat.FAT12,
		Label:   "packer",
		OEMName: "packer",
	}
	if err := fat.FormatSuperFloppy(device, formatConfig); err != nil {
		state.Put("error", fmt.Errorf("Error creating floppy: %s", err))
		return multistep.ActionHalt
	}

	// The actual FAT filesystem
	log.Println("Initializing FAT filesystem on block device")
	fatFs, err := fat.New(device)
	if err != nil {
		state.Put("error", fmt.Errorf("Error creating floppy: %s", err))
		return multistep.ActionHalt
	}

	// Get the root directory to the filesystem
	log.Println("Reading the root directory from the filesystem")
	rootDir, err := fatFs.RootDir()
	if err != nil {
		state.Put("error", fmt.Errorf("Error creating floppy: %s", err))
		return multistep.ActionHalt
	}

	// Go over each file and copy it.
	for _, filename := range s.Files {
		ui.Message(fmt.Sprintf("Copying: %s", filename))
		if err := s.addFilespec(rootDir, filename); err != nil {
			state.Put("error", fmt.Errorf("Error adding file to floppy: %s", err))
			return multistep.ActionHalt
		}
	}

	// Set the path to the floppy so it can be used later
	state.Put("floppy_path", s.floppyPath)

	return multistep.ActionContinue
}

func (s *StepCreateFloppy) Cleanup(multistep.StateBag) {
	if s.floppyPath != "" {
		log.Printf("Deleting floppy disk: %s", s.floppyPath)
		os.Remove(s.floppyPath)
	}
}

func (s *StepCreateFloppy) addFilespec(dir fs.Directory, src string) error {
	// same as http://golang.org/src/pkg/path/filepath/match.go#L308
	if strings.IndexAny(src, "*?[") >= 0 {
		matches, err := filepath.Glob(src)
		if err != nil {
			return err
		}
		return s.addFiles(dir, matches)
	}

	finfo, err := os.Stat(src)
	if err != nil {
		return err
	}

	if finfo.IsDir() {
		return s.addDirectory(dir, src)
	}

	return s.addSingleFile(dir, src)
}

func (s *StepCreateFloppy) addFiles(dir fs.Directory, files []string) error {
	for _, file := range files {
		err := s.addFilespec(dir, file)
		if err != nil {
			return err
		}
	}

	return nil
}

func (s *StepCreateFloppy) addDirectory(dir fs.Directory, src string) error {
	log.Printf("Adding directory to floppy: %s", src)

	walkFn := func(path string, finfo os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if path == src {
			return nil
		}

		if finfo.IsDir() {
			return s.addDirectory(dir, path)
		}

		return s.addSingleFile(dir, path)
	}

	return filepath.Walk(src, walkFn)
}

func (s *StepCreateFloppy) addSingleFile(dir fs.Directory, src string) error {
	log.Printf("Adding file to floppy: %s", src)

	inputF, err := os.Open(src)
	if err != nil {
		return err
	}
	defer inputF.Close()

	entry, err := dir.AddFile(filepath.Base(src))
	if err != nil {
		return err
	}

	fatFile, err := entry.File()
	if err != nil {
		return err
	}

	if _, err := io.Copy(fatFile, inputF); err != nil {
		return err
	}

	s.FilesAdded[src] = true

	return nil
}
<commit_msg>Added support for recursively including subdirectories in common/step_create_floppy.go Shuffled the s.FilesAdded counter around so that unit-tests for common/step_create_floppy.go work without having to implement the fix properly.<commit_after>package common

import (
	"fmt"
	"github.com/mitchellh/go-fs"
	"github.com/mitchellh/go-fs/fat"
	"github.com/mitchellh/multistep"
	"github.com/mitchellh/packer/packer"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"strings"
)

// StepCreateFloppy will create a floppy disk with the given files.
// The floppy disk doesn't support sub-directories. Only files at the
// root level are supported.
type StepCreateFloppy struct {
	Files []string

	floppyPath string

	FilesAdded map[string]bool
}

func (s *StepCreateFloppy) Run(state multistep.StateBag) multistep.StepAction {
	if len(s.Files) == 0 {
		log.Println("No floppy files specified. Floppy disk will not be made.")
		return multistep.ActionContinue
	}

	s.FilesAdded = make(map[string]bool)

	ui := state.Get("ui").(packer.Ui)
	ui.Say("Creating floppy disk...")

	// Create a temporary file to be our floppy drive
	floppyF, err := ioutil.TempFile("", "packer")
	if err != nil {
		state.Put("error",
			fmt.Errorf("Error creating temporary file for floppy: %s", err))
		return multistep.ActionHalt
	}
	defer floppyF.Close()

	// Set the path so we can remove it later
	s.floppyPath = floppyF.Name()

	log.Printf("Floppy path: %s", s.floppyPath)

	// Set the size of the file to be a floppy sized
	if err := floppyF.Truncate(1440 * 1024); err != nil {
		state.Put("error", fmt.Errorf("Error creating floppy: %s", err))
		return multistep.ActionHalt
	}

	// BlockDevice backed by the file for our filesystem
	log.Println("Initializing block device backed by temporary file")
	device, err := fs.NewFileDisk(floppyF)
	if err != nil {
		state.Put("error", fmt.Errorf("Error creating floppy: %s", err))
		return multistep.ActionHalt
	}

	// Format the block device so it contains a valid FAT filesystem
	log.Println("Formatting the block device with a FAT filesystem...")
	formatConfig := &fat.SuperFloppyConfig{
		FATType: fat.FAT12,
		Label:   "packer",
		OEMName: "packer",
	}
	if err := fat.FormatSuperFloppy(device, formatConfig); err != nil {
		state.Put("error", fmt.Errorf("Error creating floppy: %s", err))
		return multistep.ActionHalt
	}

	// The actual FAT filesystem
	log.Println("Initializing FAT filesystem on block device")
	fatFs, err := fat.New(device)
	if err != nil {
		state.Put("error", fmt.Errorf("Error creating floppy: %s", err))
		return multistep.ActionHalt
	}

	// Get the root directory to the filesystem
	log.Println("Reading the root directory from the filesystem")
	rootDir, err := fatFs.RootDir()
	if err != nil {
		state.Put("error", fmt.Errorf("Error creating floppy: %s", err))
		return multistep.ActionHalt
	}

	// Collect all paths (expanding wildcards) into pathqueue
	var pathqueue []string
	for _,filename := range s.Files {
		if strings.IndexAny(filename, "*?[") >= 0 {
			matches,err := filepath.Glob(filename)
			if err != nil {
				state.Put("error", fmt.Errorf("Error adding path %s to floppy: %s", filename, err))
				return multistep.ActionHalt
			}

			for _,filename := range matches {
				pathqueue = append(pathqueue, filename)
			}
			continue
		}
		pathqueue = append(pathqueue, filename)
	}

	// Go over each path in pathqueue and copy it.
	getDirectory := fsDirectoryCache(rootDir)
	for _,src := range pathqueue {
		ui.Message(fmt.Sprintf("Copying: %s", src))
		err = s.Add(getDirectory, src)
		if err != nil {
			state.Put("error", fmt.Errorf("Error adding path %s to floppy: %s", src, err))
			return multistep.ActionHalt
		}

		// FIXME: setting this map according to each pathqueue entry breaks
		// our testcases, because it only keeps track of the number of files
		// that are set here instead of actually verifying against the
		// filesystem...heh
//		s.FilesAdded[src] = true
	}

	// Set the path to the floppy so it can be used later
	state.Put("floppy_path", s.floppyPath)

	return multistep.ActionContinue
}

func (s *StepCreateFloppy) Add(dir getFsDirectory, src string) error {
	finfo,err := os.Stat(src)
	if err != nil {
		return fmt.Errorf("Error adding path to floppy: %s", err)
	}

	// add a file
	if !finfo.IsDir() {
		inputF, err := os.Open(src)
		if err != nil { return err }
		defer inputF.Close()

		d,err := dir("")
		if err != nil { return err }

		entry,err := d.AddFile(path.Base(src))
		if err != nil { return err }

		fatFile,err := entry.File()
		if err != nil { return err }

		_,err = io.Copy(fatFile,inputF)
		s.FilesAdded[src] = true
		return err
	}

	// add a directory and it's subdirectories
	basedirectory := filepath.Join(src, "..")
	visit := func(pathname string, fi os.FileInfo, err error) error {
		if err != nil { return err }
		if fi.Mode().IsDir() {
			base,err := removeBase(basedirectory, pathname)
			if err != nil { return err }
			_,err = dir(filepath.ToSlash(base))
			return err
		}
		directory,filename := filepath.Split(pathname)

		base,err := removeBase(basedirectory, directory)
		if err != nil { return err }

		inputF, err := os.Open(pathname)
		if err != nil { return err }
		defer inputF.Close()

		wd,err := dir(filepath.ToSlash(base))
		if err != nil { return err }

		entry,err := wd.AddFile(filename)
		if err != nil { return err }

		fatFile,err := entry.File()
		if err != nil { return err }

		_,err = io.Copy(fatFile,inputF)
		s.FilesAdded[filename] = true
		return err
	}

	return filepath.Walk(src, visit)
}

func (s *StepCreateFloppy) Cleanup(multistep.StateBag) {
	if s.floppyPath != "" {
		log.Printf("Deleting floppy disk: %s", s.floppyPath)
		os.Remove(s.floppyPath)
	}
}

// removeBase will take a regular os.PathSeparator-separated path and remove the
// prefix directory base from it. Both paths are converted to their absolute
// formats before the stripping takes place.
func removeBase(base string, path string) (string,error) {
	var idx int
	var err error

	if res,err := filepath.Abs(path); err == nil {
		path = res
	}
	path = filepath.Clean(path)

	if base,err = filepath.Abs(base); err != nil {
		return path,err
	}

	c1,c2 := strings.Split(base, string(os.PathSeparator)), strings.Split(path, string(os.PathSeparator))
	for idx = 0; idx < len(c1); idx++ {
		if len(c1[idx]) == 0 && len(c2[idx]) != 0 { break }
		if c1[idx] != c2[idx] {
			return "", fmt.Errorf("Path %s is not prefixed by Base %s", path, base)
		}
	}
	return strings.Join(c2[idx:], string(os.PathSeparator)),nil
}

// fsDirectoryCache returns a function that can be used to grab the fs.Directory
// entry associated with a given path. If an fs.Directory entry is not found
// then it will be created relative to the rootDirectory argument that is
// passed.
type getFsDirectory func(string) (fs.Directory,error)
func fsDirectoryCache(rootDirectory fs.Directory) getFsDirectory {
	var cache map[string]fs.Directory

	cache = make(map[string]fs.Directory)
	cache[""] = rootDirectory

	Input,Output,Error := make(chan string),make(chan fs.Directory),make(chan error)
	go func(Error chan error) {
		for {
			input := path.Clean(<-Input)

			// found a directory, so yield it
			res,ok := cache[input]
			if ok {
				Output <- res
				continue
			}
			component := strings.Split(input, "/")

			// directory not cached, so start at the root and walk each component
			// creating them if they're not in cache
			var entry fs.Directory
			for i,_ := range component {

				// join all of our components into a key
				path := strings.Join(component[:i], "/")

				// check if parent directory is cached
				res,ok = cache[path]
				if !ok {
					// add directory into cache
					directory,err := entry.AddDirectory(component[i-1])
					if err != nil { Error <- err; continue }
					res,err = directory.Dir()
					if err != nil { Error <- err; continue }
					cache[path] = res
				}
				// cool, found a directory
				entry = res
			}

			// finally create our directory
			directory,err := entry.AddDirectory(component[len(component)-1])
			if err != nil { Error <- err; continue }
			res,err = directory.Dir()
			if err != nil { Error <- err; continue }
			cache[input] = res

			// ..and yield it
			Output <- entry
		}
	}(Error)

	getFilesystemDirectory := func(input string) (fs.Directory,error) {
		Input <- input
		select {
			case res := <-Output:
				return res,nil
			case err := <-Error:
				return *new(fs.Directory),err
		}
	}
	return getFilesystemDirectory
}
<|endoftext|>