<commit_before>package utp

/*
#include "utp.h"
*/
import "C"
import (
	"errors"
	"io"
	"net"
	"sync"
	"syscall"
	"time"
	"unsafe"
)

type Conn struct {
	s          *C.utp_socket
	cond       sync.Cond
	readBuf    []byte
	gotEOF     bool
	gotConnect bool
	// Set on state changed to UTP_STATE_DESTROYING. Not valid to refer to the
	// socket after getting this.
	destroyed bool
	// Conn.Close was called.
	closed   bool
	libError error

	writeDeadline time.Time
	readDeadline  time.Time
}

func (c *Conn) onLibError(codeName string) {
	if c.libError != nil {
		panic("multiple lib errors")
	}
	c.libError = errors.New(codeName)
	c.cond.Broadcast()
}

func (c *Conn) setConnected() {
	c.gotConnect = true
	c.cond.Broadcast()
}

func (c *Conn) waitForConnect() error {
	for {
		if c.libError != nil {
			return c.libError
		}
		if c.gotConnect {
			return nil
		}
		c.cond.Wait()
	}
}

func (c *Conn) Close() (err error) {
	mu.Lock()
	defer mu.Unlock()
	c.close()
	return nil
}

func (c *Conn) close() {
	if c.closed {
		return
	}
	if !c.destroyed {
		C.utp_close(c.s)
		c.s = nil
	}
	c.closed = true
	c.cond.Broadcast()
}

func (c *Conn) LocalAddr() net.Addr {
	mu.Lock()
	defer mu.Unlock()
	return getSocketForLibContext(C.utp_get_context(c.s)).pc.LocalAddr()
}

func (c *Conn) readNoWait(b []byte) (n int, err error) {
	n = copy(b, c.readBuf)
	c.readBuf = c.readBuf[n:]
	if n != 0 && len(c.readBuf) == 0 {
		// Can we call this if the utp_socket is closed, destroyed or errored?
		C.utp_read_drained(c.s)
	}
	err = func() error {
		switch {
		case c.gotEOF:
			return io.EOF
		case c.libError != nil:
			return c.libError
		case c.destroyed:
			return errors.New("destroyed")
		case c.closed:
			return errors.New("closed")
		case !c.readDeadline.IsZero() && !time.Now().Before(c.readDeadline):
			return errDeadlineExceeded{}
		default:
			return nil
		}
	}()
	return
}

func (c *Conn) Read(b []byte) (int, error) {
	mu.Lock()
	defer mu.Unlock()
	for {
		n, err := c.readNoWait(b)
		if n != 0 || len(b) == 0 || err != nil {
			return n, err
		}
		c.cond.Wait()
	}
}

func (c *Conn) writeNoWait(b []byte) (n int, err error) {
	err = func() error {
		switch {
		case c.libError != nil:
			return c.libError
		case c.closed:
			return errors.New("closed")
		case c.destroyed:
			return errors.New("destroyed")
		default:
			return nil
		}
	}()
	if err != nil {
		return
	}
	n = int(C.utp_write(c.s, unsafe.Pointer(&b[0]), C.size_t(len(b))))
	if n < 0 {
		panic(n)
	}
	return
}

func (c *Conn) Write(b []byte) (n int, err error) {
	mu.Lock()
	defer mu.Unlock()
	for len(b) != 0 {
		var n1 int
		n1, err = c.writeNoWait(b)
		b = b[n1:]
		n += n1
		if err != nil {
			break
		}
		if n1 != 0 {
			continue
		}
		if !c.writeDeadline.IsZero() && !time.Now().Before(c.writeDeadline) {
			err = errDeadlineExceeded{}
			break
		}
		c.cond.Wait()
	}
	return
}

func (c *Conn) RemoteAddr() net.Addr {
	var rsa syscall.RawSockaddrAny
	var addrlen C.socklen_t = syscall.SizeofSockaddrAny
	C.utp_getpeername(c.s, (*C.struct_sockaddr)(unsafe.Pointer(&rsa)), &addrlen)
	sa, err := anyToSockaddr(&rsa)
	if err != nil {
		panic(err)
	}
	return sockaddrToUDP(sa)
}

func (c *Conn) SetDeadline(t time.Time) error {
	mu.Lock()
	defer mu.Unlock()
	c.readDeadline = t
	c.writeDeadline = t
	c.cond.Broadcast()
	return nil
}
func (c *Conn) SetReadDeadline(t time.Time) error {
	mu.Lock()
	defer mu.Unlock()
	c.readDeadline = t
	c.cond.Broadcast()
	return nil
}
func (c *Conn) SetWriteDeadline(t time.Time) error {
	mu.Lock()
	defer mu.Unlock()
	c.writeDeadline = t
	c.cond.Broadcast()
	return nil
}

func (c *Conn) setGotEOF() {
	c.gotEOF = true
	c.cond.Broadcast()
}

func (c *Conn) onDestroyed() {
	c.destroyed = true
	c.s = nil
	c.cond.Broadcast()
}
<commit_msg>Give more info in multiple lib error panic<commit_after>package utp

/*
#include "utp.h"
*/
import "C"
import (
	"errors"
	"fmt"
	"io"
	"net"
	"sync"
	"syscall"
	"time"
	"unsafe"
)

type Conn struct {
	s          *C.utp_socket
	cond       sync.Cond
	readBuf    []byte
	gotEOF     bool
	gotConnect bool
	// Set on state changed to UTP_STATE_DESTROYING. Not valid to refer to the
	// socket after getting this.
	destroyed bool
	// Conn.Close was called.
	closed   bool
	libError error

	writeDeadline time.Time
	readDeadline  time.Time
}

func (c *Conn) onLibError(codeName string) {
	if c.libError != nil {
		panic(fmt.Sprintf("multiple lib errors: got %s, have %s", codeName, c.libError))
	}
	c.libError = errors.New(codeName)
	c.cond.Broadcast()
}

func (c *Conn) setConnected() {
	c.gotConnect = true
	c.cond.Broadcast()
}

func (c *Conn) waitForConnect() error {
	for {
		if c.libError != nil {
			return c.libError
		}
		if c.gotConnect {
			return nil
		}
		c.cond.Wait()
	}
}

func (c *Conn) Close() (err error) {
	mu.Lock()
	defer mu.Unlock()
	c.close()
	return nil
}

func (c *Conn) close() {
	if c.closed {
		return
	}
	if !c.destroyed {
		C.utp_close(c.s)
		c.s = nil
	}
	c.closed = true
	c.cond.Broadcast()
}

func (c *Conn) LocalAddr() net.Addr {
	mu.Lock()
	defer mu.Unlock()
	return getSocketForLibContext(C.utp_get_context(c.s)).pc.LocalAddr()
}

func (c *Conn) readNoWait(b []byte) (n int, err error) {
	n = copy(b, c.readBuf)
	c.readBuf = c.readBuf[n:]
	if n != 0 && len(c.readBuf) == 0 {
		// Can we call this if the utp_socket is closed, destroyed or errored?
		C.utp_read_drained(c.s)
	}
	err = func() error {
		switch {
		case c.gotEOF:
			return io.EOF
		case c.libError != nil:
			return c.libError
		case c.destroyed:
			return errors.New("destroyed")
		case c.closed:
			return errors.New("closed")
		case !c.readDeadline.IsZero() && !time.Now().Before(c.readDeadline):
			return errDeadlineExceeded{}
		default:
			return nil
		}
	}()
	return
}

func (c *Conn) Read(b []byte) (int, error) {
	mu.Lock()
	defer mu.Unlock()
	for {
		n, err := c.readNoWait(b)
		if n != 0 || len(b) == 0 || err != nil {
			return n, err
		}
		c.cond.Wait()
	}
}

func (c *Conn) writeNoWait(b []byte) (n int, err error) {
	err = func() error {
		switch {
		case c.libError != nil:
			return c.libError
		case c.closed:
			return errors.New("closed")
		case c.destroyed:
			return errors.New("destroyed")
		default:
			return nil
		}
	}()
	if err != nil {
		return
	}
	n = int(C.utp_write(c.s, unsafe.Pointer(&b[0]), C.size_t(len(b))))
	if n < 0 {
		panic(n)
	}
	return
}

func (c *Conn) Write(b []byte) (n int, err error) {
	mu.Lock()
	defer mu.Unlock()
	for len(b) != 0 {
		var n1 int
		n1, err = c.writeNoWait(b)
		b = b[n1:]
		n += n1
		if err != nil {
			break
		}
		if n1 != 0 {
			continue
		}
		if !c.writeDeadline.IsZero() && !time.Now().Before(c.writeDeadline) {
			err = errDeadlineExceeded{}
			break
		}
		c.cond.Wait()
	}
	return
}

func (c *Conn) RemoteAddr() net.Addr {
	var rsa syscall.RawSockaddrAny
	var addrlen C.socklen_t = syscall.SizeofSockaddrAny
	C.utp_getpeername(c.s, (*C.struct_sockaddr)(unsafe.Pointer(&rsa)), &addrlen)
	sa, err := anyToSockaddr(&rsa)
	if err != nil {
		panic(err)
	}
	return sockaddrToUDP(sa)
}

func (c *Conn) SetDeadline(t time.Time) error {
	mu.Lock()
	defer mu.Unlock()
	c.readDeadline = t
	c.writeDeadline = t
	c.cond.Broadcast()
	return nil
}
func (c *Conn) SetReadDeadline(t time.Time) error {
	mu.Lock()
	defer mu.Unlock()
	c.readDeadline = t
	c.cond.Broadcast()
	return nil
}
func (c *Conn) SetWriteDeadline(t time.Time) error {
	mu.Lock()
	defer mu.Unlock()
	c.writeDeadline = t
	c.cond.Broadcast()
	return nil
}

func (c *Conn) setGotEOF() {
	c.gotEOF = true
	c.cond.Broadcast()
}

func (c *Conn) onDestroyed() {
	c.destroyed = true
	c.s = nil
	c.cond.Broadcast()
}
<|endoftext|>


<commit_before>// Copyright 2010 Alexander Neumann. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The pgsql package partially implements a PostgreSQL frontend library.
// It is compatible with servers of version 7.4 and later.
package pgsql

import (
	"bufio"
	"bytes"
	"fmt"
	"encoding/binary"
	"log"
	"math"
	"net"
	"os"
	"runtime"
)

// LogLevel is used to control which messages are written to the log.
type LogLevel int

const (
	LogNothing LogLevel = iota
	LogFatal
	LogError
	LogWarning
	LogDebug
	LogVerbose
)

type ConnParams struct {
	Host     string
	Port     int
	User     string
	Password string
	Database string
}

// ConnStatus represents the status of a connection.
type ConnStatus int

const (
	StatusDisconnected ConnStatus = iota
	StatusConnected
	StatusAuthenticating
	StatusReady
	StatusProcessingQuery
)

func (s ConnStatus) String() string {
	switch s {
	case StatusDisconnected:
		return "Disconnected"

	case StatusConnected:
		return "Connected"

	case StatusAuthenticating:
		return "Authenticating"

	case StatusReady:
		return "Ready"

	case StatusProcessingQuery:
		return "Processing Query"
	}

	return "Unknown"
}

// Conn represents a connection to a database on a PostgreSQL server.
type Conn struct {
	LogLevel         LogLevel
	conn             net.Conn
	params           ConnParams
	state            state
	backendPID       int32
	backendSecretKey int32
	reader           *bufio.Reader
	writer           *bufio.Writer
}

// NewConn returns a new Conn initialized with the specified parameters.
func NewConn(params *ConnParams) (conn *Conn, err os.Error) {
	if params == nil {
		err = os.NewError("params cannot be nil")
		return
	}

	conn = new(Conn)
	conn.params = *params
	conn.state = disconnectedState{}

	if conn.params.Host == "" {
		conn.params.Host = "127.0.0.1"
	}
	if conn.params.Port == 0 {
		conn.params.Port = 5432
	}

	return
}

func (conn *Conn) log(level LogLevel, v ...interface{}) {
	log.Stdout(v)
}

func (conn *Conn) logf(level LogLevel, format string, v ...interface{}) {
	log.Stdoutf(format, v)
}

func (conn *Conn) logError(level LogLevel, err os.Error) {
	if conn.LogLevel >= level {
		conn.log(level, err)
	}
}

func (conn *Conn) logEnter(funcName string) string {
	conn.log(LogDebug, "entering: ", "pgsql."+funcName)
	return funcName
}

func (conn *Conn) logExit(funcName string) {
	conn.log(LogDebug, "exiting: ", "pgsql."+funcName)
}

func (conn *Conn) logAndConvertPanic(x interface{}) (err os.Error) {
	buf := bytes.NewBuffer(nil)

	buf.WriteString(fmt.Sprintf("Error: %v\nStack Trace:\n", x))
	buf.WriteString("=======================================================\n")

	i := 0
	for {
		pc, file, line, ok := runtime.Caller(i + 3)
		if !ok {
			break
		}
		if i > 0 {
			buf.WriteString("-------------------------------------------------------\n")
		}

		fun := runtime.FuncForPC(pc)
		name := fun.Name()

		buf.WriteString(fmt.Sprintf("%s (%s, Line %d)\n", name, file, line))

		i++
	}
	buf.WriteString("=======================================================\n")

	if conn.LogLevel >= LogError {
		conn.log(LogError, buf)
	}

	err, ok := x.(os.Error)
	if !ok {
		err = os.NewError(buf.String())
	}

	return
}

func (conn *Conn) flush() {
	err := conn.writer.Flush()
	if err != nil {
		panic(fmt.Sprintf("flush failed: %s", err))
	}
}

func (conn *Conn) read(b []byte) {
	_, err := conn.reader.Read(b)
	if err != nil {
		panic(fmt.Sprintf("read failed: %s", err))
	}
}

func (conn *Conn) readByte() byte {
	b, err := conn.reader.ReadByte()
	if err != nil {
		panic(fmt.Sprintf("readByte failed: %s", err))
	}

	return b
}

func (conn *Conn) readBytes(delim byte) []byte {
	b, err := conn.reader.ReadBytes(delim)
	if err != nil {
		panic(fmt.Sprintf("readBytes failed: %s", err))
	}

	return b
}

func (conn *Conn) readInt16() int16 {
	var buf [2]byte
	b := buf[0:]

	conn.read(b)
	return int16(binary.BigEndian.Uint16(b))
}

func (conn *Conn) readInt32() int32 {
	var buf [4]byte
	b := buf[0:]

	conn.read(b)
	return int32(binary.BigEndian.Uint32(b))
}

func (conn *Conn) readString() string {
	b := conn.readBytes(0)
	return string(b[0 : len(b)-1])
}

func (conn *Conn) write(b []byte) {
	_, err := conn.writer.Write(b)
	if err != nil {
		panic(fmt.Sprintf("write failed: %s", err))
	}
}

func (conn *Conn) writeByte(b byte) {
	err := conn.writer.WriteByte(b)
	if err != nil {
		panic(fmt.Sprintf("writeByte failed: %s", err))
	}
}

func (conn *Conn) writeFloat32(f float32) {
	var buf [4]byte
	b := buf[0:]

	binary.BigEndian.PutUint32(b, math.Float32bits(f))
	conn.write(b)
}

func (conn *Conn) writeFloat64(f float64) {
	var buf [8]byte
	b := buf[0:]

	binary.BigEndian.PutUint64(b, math.Float64bits(f))
	conn.write(b)
}

func (conn *Conn) writeFrontendMessageCode(code frontendMessageCode) {
	err := conn.writer.WriteByte(byte(code))
	if err != nil {
		panic(fmt.Sprintf("writeFrontendMessageCode failed: %s", err))
	}
}

func (conn *Conn) writeInt16(i int16) {
	var buf [2]byte
	b := buf[0:]

	binary.BigEndian.PutUint16(b, uint16(i))
	conn.write(b)
}

func (conn *Conn) writeInt32(i int32) {
	var buf [4]byte
	b := buf[0:]

	binary.BigEndian.PutUint32(b, uint32(i))
	conn.write(b)
}

func (conn *Conn) writeInt64(i int64) {
	var buf [8]byte
	b := buf[0:]

	binary.BigEndian.PutUint64(b, uint64(i))
	conn.write(b)
}

func (conn *Conn) writeString(s string) {
	_, err := conn.writer.WriteString(s)
	if err != nil {
		panic(fmt.Sprintf("writeString failed: %s", err))
	}
}

func (conn *Conn) writeString0(s string) {
	conn.writeString(s)
	conn.writeByte(0)
}

// Close closes the connection to the database.
func (conn *Conn) Close() (err os.Error) {
	defer func() {
		if x := recover(); x != nil {
			err = conn.logAndConvertPanic(x)
		}
	}()

	if conn.LogLevel >= LogDebug {
		defer conn.logExit(conn.logEnter("*Conn.Close"))
	}

	if conn.Status() == StatusDisconnected {
		err = os.NewError("connection already closed")
		conn.logError(LogWarning, err)
		return
	}

	conn.state.disconnect(conn)

	return
}

// Status returns the current connection status.
func (conn *Conn) Status() ConnStatus {
	return conn.state.code()
}

// Open opens a database connection and waits until it is ready to issue commands.
func (conn *Conn) Open() (err os.Error) {
	defer func() {
		if x := recover(); x != nil {
			err = conn.logAndConvertPanic(x)
		}
	}()

	if conn.LogLevel >= LogDebug {
		defer conn.logExit(conn.logEnter("*Conn.Open"))
	}

	conn.state.connect(conn)
	conn.state.startup(conn)

	return
}

// Query sends a SQL query to the server and returns a
// Reader for row-by-row retrieval of the results.
// The returned Reader must be closed before sending another
// query or command to the server over the same connection.
func (conn *Conn) Query(command string) (reader *Reader, err os.Error) {
	defer func() {
		if x := recover(); x != nil {
			err = conn.logAndConvertPanic(x)
		}
	}()

	if conn.LogLevel >= LogDebug {
		defer conn.logExit(conn.logEnter("*Conn.Query"))
	}

	r := newReader(conn)

	conn.state.query(conn, r, command)

	reader = r

	return
}

// Execute sends a SQL command to the server and returns the number
// of rows affected. If the results of a query are needed, use the
// Query method instead.
func (conn *Conn) Execute(command string) (rowsAffected int64, err os.Error) {
	defer func() {
		if x := recover(); x != nil {
			err = conn.logAndConvertPanic(x)
		}
	}()

	if conn.LogLevel >= LogDebug {
		defer conn.logExit(conn.logEnter("*Conn.Execute"))
	}

	reader, err := conn.Query(command)
	if err != nil {
		return
	}

	err = reader.Close()

	rowsAffected = reader.rowsAffected
	return
}

// Prepare returns a new prepared Statement, optimized to be executed multiple
// times with different parameter values.
func (conn *Conn) Prepare(command string, params []*Parameter) (stmt *Statement, err os.Error) {
	defer func() {
		if x := recover(); x != nil {
			err = conn.logAndConvertPanic(x)
		}
	}()

	if conn.LogLevel >= LogDebug {
		defer conn.logExit(conn.logEnter("*Conn.Prepare"))
	}

	statement := newStatement(conn, command, params)

	conn.state.prepare(statement)

	stmt = statement
	return
}
<commit_msg>Add *Conn.WithTransaction<commit_after>// Copyright 2010 Alexander Neumann. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The pgsql package partially implements a PostgreSQL frontend library.
// It is compatible with servers of version 7.4 and later.
package pgsql

import (
	"bufio"
	"bytes"
	"fmt"
	"encoding/binary"
	"log"
	"math"
	"net"
	"os"
	"runtime"
)

// LogLevel is used to control which messages are written to the log.
type LogLevel int

const (
	LogNothing LogLevel = iota
	LogFatal
	LogError
	LogWarning
	LogDebug
	LogVerbose
)

type ConnParams struct {
	Host     string
	Port     int
	User     string
	Password string
	Database string
}

// ConnStatus represents the status of a connection.
type ConnStatus int

const (
	StatusDisconnected ConnStatus = iota
	StatusConnected
	StatusAuthenticating
	StatusReady
	StatusProcessingQuery
)

func (s ConnStatus) String() string {
	switch s {
	case StatusDisconnected:
		return "Disconnected"

	case StatusConnected:
		return "Connected"

	case StatusAuthenticating:
		return "Authenticating"

	case StatusReady:
		return "Ready"

	case StatusProcessingQuery:
		return "Processing Query"
	}

	return "Unknown"
}

// Conn represents a connection to a database on a PostgreSQL server.
type Conn struct {
	LogLevel         LogLevel
	conn             net.Conn
	params           ConnParams
	state            state
	backendPID       int32
	backendSecretKey int32
	reader           *bufio.Reader
	writer           *bufio.Writer
}

// NewConn returns a new Conn initialized with the specified parameters.
func NewConn(params *ConnParams) (conn *Conn, err os.Error) {
	if params == nil {
		err = os.NewError("params cannot be nil")
		return
	}

	conn = new(Conn)
	conn.params = *params
	conn.state = disconnectedState{}

	if conn.params.Host == "" {
		conn.params.Host = "127.0.0.1"
	}
	if conn.params.Port == 0 {
		conn.params.Port = 5432
	}

	return
}

func (conn *Conn) log(level LogLevel, v ...interface{}) {
	log.Stdout(v)
}

func (conn *Conn) logf(level LogLevel, format string, v ...interface{}) {
	log.Stdoutf(format, v)
}

func (conn *Conn) logError(level LogLevel, err os.Error) {
	if conn.LogLevel >= level {
		conn.log(level, err)
	}
}

func (conn *Conn) logEnter(funcName string) string {
	conn.log(LogDebug, "entering: ", "pgsql."+funcName)
	return funcName
}

func (conn *Conn) logExit(funcName string) {
	conn.log(LogDebug, "exiting: ", "pgsql."+funcName)
}

func (conn *Conn) logAndConvertPanic(x interface{}) (err os.Error) {
	buf := bytes.NewBuffer(nil)

	buf.WriteString(fmt.Sprintf("Error: %v\nStack Trace:\n", x))
	buf.WriteString("=======================================================\n")

	i := 0
	for {
		pc, file, line, ok := runtime.Caller(i + 3)
		if !ok {
			break
		}
		if i > 0 {
			buf.WriteString("-------------------------------------------------------\n")
		}

		fun := runtime.FuncForPC(pc)
		name := fun.Name()

		buf.WriteString(fmt.Sprintf("%s (%s, Line %d)\n", name, file, line))

		i++
	}
	buf.WriteString("=======================================================\n")

	if conn.LogLevel >= LogError {
		conn.log(LogError, buf)
	}

	err, ok := x.(os.Error)
	if !ok {
		err = os.NewError(buf.String())
	}

	return
}

func (conn *Conn) flush() {
	err := conn.writer.Flush()
	if err != nil {
		panic(fmt.Sprintf("flush failed: %s", err))
	}
}

func (conn *Conn) read(b []byte) {
	_, err := conn.reader.Read(b)
	if err != nil {
		panic(fmt.Sprintf("read failed: %s", err))
	}
}

func (conn *Conn) readByte() byte {
	b, err := conn.reader.ReadByte()
	if err != nil {
		panic(fmt.Sprintf("readByte failed: %s", err))
	}

	return b
}

func (conn *Conn) readBytes(delim byte) []byte {
	b, err := conn.reader.ReadBytes(delim)
	if err != nil {
		panic(fmt.Sprintf("readBytes failed: %s", err))
	}

	return b
}

func (conn *Conn) readInt16() int16 {
	var buf [2]byte
	b := buf[0:]

	conn.read(b)
	return int16(binary.BigEndian.Uint16(b))
}

func (conn *Conn) readInt32() int32 {
	var buf [4]byte
	b := buf[0:]

	conn.read(b)
	return int32(binary.BigEndian.Uint32(b))
}

func (conn *Conn) readString() string {
	b := conn.readBytes(0)
	return string(b[0 : len(b)-1])
}

func (conn *Conn) write(b []byte) {
	_, err := conn.writer.Write(b)
	if err != nil {
		panic(fmt.Sprintf("write failed: %s", err))
	}
}

func (conn *Conn) writeByte(b byte) {
	err := conn.writer.WriteByte(b)
	if err != nil {
		panic(fmt.Sprintf("writeByte failed: %s", err))
	}
}

func (conn *Conn) writeFloat32(f float32) {
	var buf [4]byte
	b := buf[0:]

	binary.BigEndian.PutUint32(b, math.Float32bits(f))
	conn.write(b)
}

func (conn *Conn) writeFloat64(f float64) {
	var buf [8]byte
	b := buf[0:]

	binary.BigEndian.PutUint64(b, math.Float64bits(f))
	conn.write(b)
}

func (conn *Conn) writeFrontendMessageCode(code frontendMessageCode) {
	err := conn.writer.WriteByte(byte(code))
	if err != nil {
		panic(fmt.Sprintf("writeFrontendMessageCode failed: %s", err))
	}
}

func (conn *Conn) writeInt16(i int16) {
	var buf [2]byte
	b := buf[0:]

	binary.BigEndian.PutUint16(b, uint16(i))
	conn.write(b)
}

func (conn *Conn) writeInt32(i int32) {
	var buf [4]byte
	b := buf[0:]

	binary.BigEndian.PutUint32(b, uint32(i))
	conn.write(b)
}

func (conn *Conn) writeInt64(i int64) {
	var buf [8]byte
	b := buf[0:]

	binary.BigEndian.PutUint64(b, uint64(i))
	conn.write(b)
}

func (conn *Conn) writeString(s string) {
	_, err := conn.writer.WriteString(s)
	if err != nil {
		panic(fmt.Sprintf("writeString failed: %s", err))
	}
}

func (conn *Conn) writeString0(s string) {
	conn.writeString(s)
	conn.writeByte(0)
}

// Close closes the connection to the database.
func (conn *Conn) Close() (err os.Error) {
	defer func() {
		if x := recover(); x != nil {
			err = conn.logAndConvertPanic(x)
		}
	}()

	if conn.LogLevel >= LogDebug {
		defer conn.logExit(conn.logEnter("*Conn.Close"))
	}

	if conn.Status() == StatusDisconnected {
		err = os.NewError("connection already closed")
		conn.logError(LogWarning, err)
		return
	}

	conn.state.disconnect(conn)

	return
}

// Status returns the current connection status.
func (conn *Conn) Status() ConnStatus {
	return conn.state.code()
}

// Open opens a database connection and waits until it is ready to issue commands.
func (conn *Conn) Open() (err os.Error) {
	defer func() {
		if x := recover(); x != nil {
			err = conn.logAndConvertPanic(x)
		}
	}()

	if conn.LogLevel >= LogDebug {
		defer conn.logExit(conn.logEnter("*Conn.Open"))
	}

	conn.state.connect(conn)
	conn.state.startup(conn)

	return
}

// Query sends a SQL query to the server and returns a
// Reader for row-by-row retrieval of the results.
// The returned Reader must be closed before sending another
// query or command to the server over the same connection.
func (conn *Conn) Query(command string) (reader *Reader, err os.Error) {
	defer func() {
		if x := recover(); x != nil {
			err = conn.logAndConvertPanic(x)
		}
	}()

	if conn.LogLevel >= LogDebug {
		defer conn.logExit(conn.logEnter("*Conn.Query"))
	}

	r := newReader(conn)

	conn.state.query(conn, r, command)

	reader = r

	return
}

// Execute sends a SQL command to the server and returns the number
// of rows affected. If the results of a query are needed, use the
// Query method instead.
func (conn *Conn) Execute(command string) (rowsAffected int64, err os.Error) {
	defer func() {
		if x := recover(); x != nil {
			err = conn.logAndConvertPanic(x)
		}
	}()

	if conn.LogLevel >= LogDebug {
		defer conn.logExit(conn.logEnter("*Conn.Execute"))
	}

	reader, err := conn.Query(command)
	if err != nil {
		return
	}

	err = reader.Close()

	rowsAffected = reader.rowsAffected
	return
}

// Prepare returns a new prepared Statement, optimized to be executed multiple
// times with different parameter values.
func (conn *Conn) Prepare(command string, params []*Parameter) (stmt *Statement, err os.Error) {
	defer func() {
		if x := recover(); x != nil {
			err = conn.logAndConvertPanic(x)
		}
	}()

	if conn.LogLevel >= LogDebug {
		defer conn.logExit(conn.logEnter("*Conn.Prepare"))
	}

	statement := newStatement(conn, command, params)

	conn.state.prepare(statement)

	stmt = statement
	return
}

// WithTransaction starts a transaction, then calls function f.
// If f returns an error or panicks, the transaction is rolled back,
// otherwise it is committed.
func (conn *Conn) WithTransaction(f func() os.Error) (err os.Error) {
	defer func() {
		if x := recover(); x != nil {
			err = conn.logAndConvertPanic(x)
		}
		if err != nil {
			conn.Execute("ROLLBACK;")
		}
	}()

	if conn.LogLevel >= LogDebug {
		defer conn.logExit(conn.logEnter("*Conn.WithTransaction"))
	}

	_, err = conn.Execute("BEGIN;")
	if err != nil {
		panic(err)
	}

	err = f()
	if err != nil {
		panic(err)
	}

	_, err = conn.Execute("COMMIT;")
	return
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"strings"
	"time"
)

const (
	GET     = "GET"
	POST    = "POST"
	PUT     = "PUT"
	DELETE  = "DELETE"
	OPTIONS = "OPTIONS"
)

var all = strings.Join([]string{OPTIONS, GET, POST, PUT, DELETE}, ", ")

type GetMethod interface {
	Get(url.Values) (int, interface{})
}

type PostMethod interface {
	Post(url.Values) (int, interface{})
}

type PutMethod interface {
	Put(url.Values) (int, interface{})
}

type DeleteMethod interface {
	Delete(url.Values) (int, interface{})
}

// Dispatches an HTTP error with the given code
func Abort(rw *http.ResponseWriter, code int) {
	err := fmt.Sprintf("%d %s", code, http.StatusText(code))
	http.Error(*rw, err, code)
}

// Reflects the different HTTP verbs into the given interface, allows the
// following methods: `Get`, `Post`, `Put`, `Delete`.
//
// Based on a Doug Black code:
// https://github.com/dougblack/sleepy/blob/master/core.go
func RestController(c interface{}) http.HandlerFunc {
	return func(rw http.ResponseWriter, r *http.Request) {
		l := NewHttpLogger(*r)
		defer l.Print()
		// Add some usefull headers
		h := rw.Header()
		h.Add("Content-Type", "application/json; charset=utf-8")
		h.Add("Connection", "close")
		// CORS Headers
		h.Add("Access-Control-Allow-Headers", "Origin, X-Requested-With, "+
			"Content-Type, Accept")
		h.Add("Access-Control-Allow-Origin", "*")
		h.Add("Access-Control-Allow-Methods", all)

		var handler func(url.Values) (int, interface{})

		switch r.Method {
		case GET:
			if c, ok := c.(GetMethod); ok {
				handler = c.Get
			}
		case POST:
			if c, ok := c.(PostMethod); ok {
				handler = c.Post
			}
		case PUT:
			if c, ok := c.(PutMethod); ok {
				handler = c.Put
			}
		case DELETE:
			if c, ok := c.(DeleteMethod); ok {
				handler = c.Delete
			}
		case OPTIONS:
			handler = func(_ url.Values) (int, interface{}) {
				return http.StatusOK, ""
			}
			h.Add("Allow", all)
		}
		// Abort with a 405 status
		if handler == nil {
			Abort(&rw, http.StatusMethodNotAllowed)
			return
		}
		// Parse request data
		if err := r.ParseForm(); err != nil {
			Abort(&rw, http.StatusBadRequest)
			return
		}
		// Create the params from GET and POST values
		params, _ := url.ParseQuery(fmt.Sprintf("%s&%s",
			r.URL.Query().Encode(), r.Form.Encode()))
		// Call the handler
		code, data := handler(params)
		// Set a default
		if data == nil {
			data = map[string]interface{}{
				"status": code,
				"text":   http.StatusText(code),
			}
		}
		// Encode
		content, err := json.Marshal(data)
		if err != nil {
			Abort(&rw, http.StatusInternalServerError)
			return
		}
		// Write to response
		rw.WriteHeader(code)
		rw.Write(content)
	}
}

// Simple HTTP request logger
type HttpLogger struct {
	initTime time.Time
	request  http.Request
}

// Print with the method, url and request time
func (l *HttpLogger) Print() {
	log.Printf("%s \t %s %s",
		time.Since(l.initTime).String(),
		l.request.Method,
		l.request.URL.Path)
}

// Return a new instance of the HTTP logger
func NewHttpLogger(r http.Request) *HttpLogger {
	l := new(HttpLogger)
	l.initTime = time.Now()
	l.request = r
	return l
}
<commit_msg>Fix quoting at allowed CORS methods<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"strings"
	"time"
)

const (
	GET     = "GET"
	POST    = "POST"
	PUT     = "PUT"
	DELETE  = "DELETE"
	OPTIONS = "OPTIONS"
)

var all = strings.Join([]string{OPTIONS, GET, POST, PUT, DELETE}, ", ")

type GetMethod interface {
	Get(url.Values) (int, interface{})
}

type PostMethod interface {
	Post(url.Values) (int, interface{})
}

type PutMethod interface {
	Put(url.Values) (int, interface{})
}

type DeleteMethod interface {
	Delete(url.Values) (int, interface{})
}

// Dispatches an HTTP error with the given code
func Abort(rw *http.ResponseWriter, code int) {
	err := fmt.Sprintf("%d %s", code, http.StatusText(code))
	http.Error(*rw, err, code)
}

// Reflects the different HTTP verbs into the given interface, allows the
// following methods: `Get`, `Post`, `Put`, `Delete`.
//
// Based on a Doug Black code:
// https://github.com/dougblack/sleepy/blob/master/core.go
func RestController(c interface{}) http.HandlerFunc {
	return func(rw http.ResponseWriter, r *http.Request) {
		l := NewHttpLogger(*r)
		defer l.Print()
		// Add some usefull headers
		h := rw.Header()
		h.Add("Content-Type", "application/json; charset=utf-8")
		h.Add("Connection", "close")
		// CORS Headers
		h.Add("Access-Control-Allow-Headers", "Origin, X-Requested-With, "+
			"Content-Type, Accept")
		h.Add("Access-Control-Allow-Origin", "*")
		h.Add("Access-Control-Allow-Methods", "*")

		var handler func(url.Values) (int, interface{})

		switch r.Method {
		case GET:
			if c, ok := c.(GetMethod); ok {
				handler = c.Get
			}
		case POST:
			if c, ok := c.(PostMethod); ok {
				handler = c.Post
			}
		case PUT:
			if c, ok := c.(PutMethod); ok {
				handler = c.Put
			}
		case DELETE:
			if c, ok := c.(DeleteMethod); ok {
				handler = c.Delete
			}
		case OPTIONS:
			handler = func(_ url.Values) (int, interface{}) {
				return http.StatusOK, ""
			}
			h.Add("Allow", all)
		}
		// Abort with a 405 status
		if handler == nil {
			Abort(&rw, http.StatusMethodNotAllowed)
			return
		}
		// Parse request data
		if err := r.ParseForm(); err != nil {
			Abort(&rw, http.StatusBadRequest)
			return
		}
		// Create the params from GET and POST values
		params, _ := url.ParseQuery(fmt.Sprintf("%s&%s",
			r.URL.Query().Encode(), r.Form.Encode()))
		// Call the handler
		code, data := handler(params)
		// Set a default
		if data == nil {
			data = map[string]interface{}{
				"status": code,
				"text":   http.StatusText(code),
			}
		}
		// Encode
		content, err := json.Marshal(data)
		if err != nil {
			Abort(&rw, http.StatusInternalServerError)
			return
		}
		// Write to response
		rw.WriteHeader(code)
		rw.Write(content)
	}
}

// Simple HTTP request logger
type HttpLogger struct {
	initTime time.Time
	request  http.Request
}

// Print with the method, url and request time
func (l *HttpLogger) Print() {
	log.Printf("%s \t %s %s",
		time.Since(l.initTime).String(),
		l.request.Method,
		l.request.URL.Path)
}

// Return a new instance of the HTTP logger
func NewHttpLogger(r http.Request) *HttpLogger {
	l := new(HttpLogger)
	l.initTime = time.Now()
	l.request = r
	return l
}
<|endoftext|>


<commit_before>package cron

import (
	"log"
	"runtime"
	"sort"
	"time"
)

// Cron keeps track of any number of entries, invoking the associated func as
// specified by the schedule. It may be started, stopped, and the entries may
// be inspected while running.
type Cron struct {
	entries  []*Entry
	stop     chan struct{}
	add      chan *Entry
	snapshot chan []*Entry
	running  bool
	ErrorLog *log.Logger
	location *time.Location
}

// Job is an interface for submitted cron jobs.
type Job interface {
	Run()
}

// The Schedule describes a job's duty cycle.
type Schedule interface {
	// Return the next activation time, later than the given time.
	// Next is invoked initially, and then each time the job is run.
	Next(time.Time) time.Time
}

// Entry consists of a schedule and the func to execute on that schedule.
type Entry struct {
	// The schedule on which this job should be run.
	Schedule Schedule

	// The next time the job will run. This is the zero time if Cron has not been
	// started or this entry's schedule is unsatisfiable
	Next time.Time

	// The last time this job was run. This is the zero time if the job has never
	// been run.
	Prev time.Time

	// The Job to run.
	Job Job
}

// byTime is a wrapper for sorting the entry array by time
// (with zero time at the end).
type byTime []*Entry

func (s byTime) Len() int      { return len(s) }
func (s byTime) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s byTime) Less(i, j int) bool {
	// Two zero times should return false.
	// Otherwise, zero is "greater" than any other time.
	// (To sort it at the end of the list.)
	if s[i].Next.IsZero() {
		return false
	}
	if s[j].Next.IsZero() {
		return true
	}
	return s[i].Next.Before(s[j].Next)
}

// New returns a new Cron job runner, in the Local time zone.
func New() *Cron {
	return NewWithLocation(time.Now().Location())
}

// NewWithLocation returns a new Cron job runner.
func NewWithLocation(location *time.Location) *Cron {
	return &Cron{
		entries:  nil,
		add:      make(chan *Entry),
		stop:     make(chan struct{}),
		snapshot: make(chan []*Entry),
		running:  false,
		ErrorLog: nil,
		location: location,
	}
}

// A wrapper that turns a func() into a cron.Job
type FuncJob func()

func (f FuncJob) Run() { f() }

// AddFunc adds a func to the Cron to be run on the given schedule.
func (c *Cron) AddFunc(spec string, cmd func()) error {
	return c.AddJob(spec, FuncJob(cmd))
}

// AddJob adds a Job to the Cron to be run on the given schedule.
func (c *Cron) AddJob(spec string, cmd Job) error {
	schedule, err := Parse(spec)
	if err != nil {
		return err
	}
	c.Schedule(schedule, cmd)
	return nil
}

// Schedule adds a Job to the Cron to be run on the given schedule.
func (c *Cron) Schedule(schedule Schedule, cmd Job) {
	entry := &Entry{
		Schedule: schedule,
		Job:      cmd,
	}
	if !c.running {
		c.entries = append(c.entries, entry)
		return
	}

	c.add <- entry
}

// Entries returns a snapshot of the cron entries.
func (c *Cron) Entries() []*Entry {
	if c.running {
		c.snapshot <- nil
		x := <-c.snapshot
		return x
	}
	return c.entrySnapshot()
}

// Location gets the time zone location
func (c *Cron) Location() *time.Location {
	return c.location
}

// Start the cron scheduler in its own go-routine, or no-op if already started.
func (c *Cron) Start() {
	if c.running {
		return
	}
	c.running = true
	go c.run()
}

// Run the cron scheduler, or no-op if already running.
func (c *Cron) Run() {
	if c.running {
		return
	}
	c.running = true
	c.run()
}

func (c *Cron) runWithRecovery(j Job) {
	defer func() {
		if r := recover(); r != nil {
			const size = 64 << 10
			buf := make([]byte, size)
			buf = buf[:runtime.Stack(buf, false)]
			c.logf("cron: panic running job: %v\n%s", r, buf)
		}
	}()
	j.Run()
}

// Run the scheduler. this is private just due to the need to synchronize
// access to the 'running' state variable.
func (c *Cron) run() {
	// Figure out the next activation times for each entry.
	now := c.now()
	for _, entry := range c.entries {
		entry.Next = entry.Schedule.Next(now)
	}

	for {
		// Determine the next entry to run.
		sort.Sort(byTime(c.entries))

		var timer *time.Timer
		if len(c.entries) == 0 || c.entries[0].Next.IsZero() {
			// If there are no entries yet, just sleep - it still handles new entries
			// and stop requests.
			timer = time.NewTimer(100000 * time.Hour)
		} else {
			timer = time.NewTimer(c.entries[0].Next.Sub(now))
		}

		select {
		case now = <-timer.C:
			now = now.In(c.location)
			// Run every entry whose next time was less than now
			for _, e := range c.entries {
				if e.Next.After(now) || e.Next.IsZero() {
					break
				}
				go c.runWithRecovery(e.Job)
				e.Prev = e.Next
				e.Next = e.Schedule.Next(now)
			}
			continue

		case newEntry := <-c.add:
			c.entries = append(c.entries, newEntry)
			newEntry.Next = newEntry.Schedule.Next(time.Now().In(c.location))

		case <-c.snapshot:
			c.snapshot <- c.entrySnapshot()

		case <-c.stop:
			timer.Stop()
			return
		}

		// 'now' should be updated after newEntry and snapshot cases.
		now = time.Now().In(c.location)
		timer.Stop()
	}
}

// Logs an error to stderr or to the configured error log
func (c *Cron) logf(format string, args ...interface{}) {
	if c.ErrorLog != nil {
		c.ErrorLog.Printf(format, args...)
	} else {
		log.Printf(format, args...)
	}
}

// Stop stops the cron scheduler if it is running; otherwise it does nothing.
func (c *Cron) Stop() {
	if !c.running {
		return
	}
	c.stop <- struct{}{}
	c.running = false
}

// entrySnapshot returns a copy of the current cron entry list.
func (c *Cron) entrySnapshot() []*Entry {
	entries := []*Entry{}
	for _, e := range c.entries {
		entries = append(entries, &Entry{
			Schedule: e.Schedule,
			Next:     e.Next,
			Prev:     e.Prev,
			Job:      e.Job,
		})
	}
	return entries
}

// now returns current time in c location
func (c *Cron) now() time.Time {
	return time.Now().In(c.location)
}
<commit_msg>did not reset timer on snapshot requests<commit_after>package cron

import (
	"log"
	"runtime"
	"sort"
	"time"
)

// Cron keeps track of any number of entries, invoking the associated func as
// specified by the schedule. It may be started, stopped, and the entries may
// be inspected while running.
type Cron struct {
	entries  []*Entry
	stop     chan struct{}
	add      chan *Entry
	snapshot chan []*Entry
	running  bool
	ErrorLog *log.Logger
	location *time.Location
}

// Job is an interface for submitted cron jobs.
type Job interface {
	Run()
}

// The Schedule describes a job's duty cycle.
type Schedule interface {
	// Return the next activation time, later than the given time.
	// Next is invoked initially, and then each time the job is run.
	Next(time.Time) time.Time
}

// Entry consists of a schedule and the func to execute on that schedule.
type Entry struct {
	// The schedule on which this job should be run.
	Schedule Schedule

	// The next time the job will run. This is the zero time if Cron has not been
	// started or this entry's schedule is unsatisfiable
	Next time.Time

	// The last time this job was run. This is the zero time if the job has never
	// been run.
	Prev time.Time

	// The Job to run.
	Job Job
}

// byTime is a wrapper for sorting the entry array by time
// (with zero time at the end).
type byTime []*Entry

func (s byTime) Len() int      { return len(s) }
func (s byTime) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s byTime) Less(i, j int) bool {
	// Two zero times should return false.
	// Otherwise, zero is "greater" than any other time.
	// (To sort it at the end of the list.)
	if s[i].Next.IsZero() {
		return false
	}
	if s[j].Next.IsZero() {
		return true
	}
	return s[i].Next.Before(s[j].Next)
}

// New returns a new Cron job runner, in the Local time zone.
func New() *Cron {
	return NewWithLocation(time.Now().Location())
}

// NewWithLocation returns a new Cron job runner.
func NewWithLocation(location *time.Location) *Cron {
	return &Cron{
		entries:  nil,
		add:      make(chan *Entry),
		stop:     make(chan struct{}),
		snapshot: make(chan []*Entry),
		running:  false,
		ErrorLog: nil,
		location: location,
	}
}

// A wrapper that turns a func() into a cron.Job
type FuncJob func()

func (f FuncJob) Run() { f() }

// AddFunc adds a func to the Cron to be run on the given schedule.
func (c *Cron) AddFunc(spec string, cmd func()) error {
	return c.AddJob(spec, FuncJob(cmd))
}

// AddJob adds a Job to the Cron to be run on the given schedule.
func (c *Cron) AddJob(spec string, cmd Job) error {
	schedule, err := Parse(spec)
	if err != nil {
		return err
	}
	c.Schedule(schedule, cmd)
	return nil
}

// Schedule adds a Job to the Cron to be run on the given schedule.
func (c *Cron) Schedule(schedule Schedule, cmd Job) {
	entry := &Entry{
		Schedule: schedule,
		Job:      cmd,
	}
	if !c.running {
		c.entries = append(c.entries, entry)
		return
	}

	c.add <- entry
}

// Entries returns a snapshot of the cron entries.
func (c *Cron) Entries() []*Entry {
	if c.running {
		c.snapshot <- nil
		x := <-c.snapshot
		return x
	}
	return c.entrySnapshot()
}

// Location gets the time zone location
func (c *Cron) Location() *time.Location {
	return c.location
}

// Start the cron scheduler in its own go-routine, or no-op if already started.
func (c *Cron) Start() {
	if c.running {
		return
	}
	c.running = true
	go c.run()
}

// Run the cron scheduler, or no-op if already running.
func (c *Cron) Run() {
	if c.running {
		return
	}
	c.running = true
	c.run()
}

func (c *Cron) runWithRecovery(j Job) {
	defer func() {
		if r := recover(); r != nil {
			const size = 64 << 10
			buf := make([]byte, size)
			buf = buf[:runtime.Stack(buf, false)]
			c.logf("cron: panic running job: %v\n%s", r, buf)
		}
	}()
	j.Run()
}

// Run the scheduler. this is private just due to the need to synchronize
// access to the 'running' state variable.
func (c *Cron) run() {
	// Figure out the next activation times for each entry.
	now := c.now()
	for _, entry := range c.entries {
		entry.Next = entry.Schedule.Next(now)
	}

	for {
		// Determine the next entry to run.
		sort.Sort(byTime(c.entries))

		var timer *time.Timer
		if len(c.entries) == 0 || c.entries[0].Next.IsZero() {
			// If there are no entries yet, just sleep - it still handles new entries
			// and stop requests.
			timer = time.NewTimer(100000 * time.Hour)
		} else {
			timer = time.NewTimer(c.entries[0].Next.Sub(now))
		}

		for {
			select {
			case now = <-timer.C:
				now = now.In(c.location)
				// Run every entry whose next time was less than now
				for _, e := range c.entries {
					if e.Next.After(now) || e.Next.IsZero() {
						break
					}
					go c.runWithRecovery(e.Job)
					e.Prev = e.Next
					e.Next = e.Schedule.Next(now)
				}

			case newEntry := <-c.add:
				timer.Stop()
				now = c.now()
				newEntry.Next = newEntry.Schedule.Next(now)
				c.entries = append(c.entries, newEntry)

			case <-c.snapshot:
				c.snapshot <- c.entrySnapshot()
				continue

			case <-c.stop:
				timer.Stop()
				return
			}

			break
		}
	}
}

// Logs an error to stderr or to the configured error log
func (c *Cron) logf(format string, args ...interface{}) {
	if c.ErrorLog != nil {
		c.ErrorLog.Printf(format, args...)
	} else {
		log.Printf(format, args...)
	}
}

// Stop stops the cron scheduler if it is running; otherwise it does nothing.
func (c *Cron) Stop() {
	if !c.running {
		return
	}
	c.stop <- struct{}{}
	c.running = false
}

// entrySnapshot returns a copy of the current cron entry list.
func (c *Cron) entrySnapshot() []*Entry {
	entries := []*Entry{}
	for _, e := range c.entries {
		entries = append(entries, &Entry{
			Schedule: e.Schedule,
			Next:     e.Next,
			Prev:     e.Prev,
			Job:      e.Job,
		})
	}
	return entries
}

// now returns current time in c location
func (c *Cron) now() time.Time {
	return time.Now().In(c.location)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"code.google.com/p/go.net/html" //Tokenizer für HTML
	"database/sql"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"io"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"
)

const MaxOutstanding_Fetcher = 100 // Maximale Anzahl gleichzeitger Fetcher
const DEBUG = 1                    // Debugausgabe an/aus
const MaxLinkDepth = 1             // Maximale Linktiefe

type URLINDEX struct {
	URL   string
	WORDS map[string]int
}
type HTTPRESP struct {
	URL       string
	LINKDEPTH int
	FD        io.Reader
}
type URL struct {
	URL       string
	LINKDEPTH int
}

var crwldurls map[string]bool // globale URL Map - um doppeltes HTTP GET zu vermeiden

//Channels
var chan_urls = make(chan URL, 100000) // buffered channel of strings
//bei aktueller implementierung des Parser aufrufs, müsste der io_reader channel nicht buffered sein
var chan_ioreaders = make(chan HTTPRESP, 10)   // buffered channel of structs
var chan_urlindexes = make(chan URLINDEX, 100) // buffered channel of structs

//Semaphore Channel
var sem_Fetcher = make(chan int, MaxOutstanding_Fetcher)

func debugausgabe(msg string) {
	if DEBUG == 1 {
		fmt.Printf("%s DEBUG: %s\n", time.Now(), msg)
	}
}

func handleFetcher(url URL) {
	<-sem_Fetcher // Eine Ressource verbrauchen: Lock falls bereits alle verbraucht
	fetchURL(url)
	sem_Fetcher <- 1 // Eine Ressource wieder freigeben
}

func starten() {
	// Initiales setzen der Ressourcen
	for i := 0; i < MaxOutstanding_Fetcher; i++ {
		sem_Fetcher <- 1
	}
	// Endless (as long as the channel is not empty) Fetcher Spawning
	for {
		go handleFetcher(<-chan_urls)
	}
}

// parseHTML bekommt eine komplette HTML Seite
// und legt eine Map mit Wörtern und (viele) einzelne Links in entsprechende Channcd els
func parseHtml(a HTTPRESP) {
	//start := time.Now()
	d := html.NewTokenizer(a.FD)
	var words map[string]int
	words = make(map[string]int)

	for {
		// token type
		tokenType := d.Next()
		// ErrorToken kommt (auch) beim Ende der Daten
		if tokenType == html.ErrorToken {
			chan_urlindexes <- URLINDEX{a.URL, words} // WORD-Map in den Channel legen
			//fmt.Printf("Parse-Dauer : [%.2fs]  URL: %s\n", time.Since(start).Seconds(), a.URL)
			return
		}
		token := d.Token()
		switch tokenType {
		case html.StartTagToken: // <tag>
			// Links finden
			if token.Data == "a" {
				for _, element := range token.Attr {
					if element.Key == "href" {
						// Link normalisieren
						url, err := url.Parse(element.Val)
						// Nur Absolute Links die nicht in der globalen Link Map sind
						if url.IsAbs() && err == nil && url.Scheme == "http" && crwldurls[url.String()] != true && a.LINKDEPTH <= MaxLinkDepth {
							debugausgabe("Linktiefe: " + string(a.LINKDEPTH))
							chan_urls <- URL{url.String(), a.LINKDEPTH + 1} // Die URL in den Channel legen und Linktiefe hochzählen
						}
					}
				}
			}

		case html.TextToken: // text between start and end tag
			//Map mit Wörtern erstellen
			temp := strings.Fields(token.Data) //Aufteilen in Einzelne Wörter, trennen bei Whitespace
			for _, element := range temp {
				//TODO: einzelne Örter noch besser von Sonderzeichen trennen z.b. mit TRIM()
				words[element] = words[element] + 1
			}

			//fmt.Printf("%q\n", temp)
		case html.EndTagToken: // </tag>
		case html.SelfClosingTagToken: // <tag/>

		}
	}
}

// fetchURL Bekommt eine URL
// lädt die Seite herunter
// und legt ein struct vom Typ HTTPRESP in den Channel chan_ioreaders
func fetchURL(url URL) {
	//start := time.Now()
	crwldurls[url.URL] = true //URL in die globale URL Liste aufnehmen damit sie nicht nochmal in den Work Queue kommt.
	response, err := http.Get(url.URL)
	if err != nil {
		fmt.Printf("Fehler: %s beim HTTP GET von: %s\n", err, url.URL)
		return
	}
	//fmt.Printf("%T", response.Body)
	chan_ioreaders <- HTTPRESP{url.URL, url.LINKDEPTH, response.Body}
	//fmt.Printf("Dauer : [%.2fs]  URL: %s\n", time.Since(start).Seconds(), url)
	//TODO io.reader channel umstellen auf direkte übergabe des structs
	go parseHtml(<-chan_ioreaders) // Für jeden Fetcher wird ein Parser aufgerufen, der unabhängig läuft
	return
}

//save holt sich daten (structs vom Typ URLINDEX) aus dem Channel chan_urlindexes
// und schreibt die Daten in eine Datei "output.txt"
func save() {

	//fo, err := os.OpenFile("output.txt", os.O_APPEND, 0777)
	fo, err := os.Create("output.txt")
	if err != nil {
		panic(err)
	}
	// close fo on exit and check for its returned error
	defer func() {
		if err := fo.Close(); err != nil {
			panic(err)
		}
	}()
	// make a write buffer
	w := bufio.NewWriter(fo)
	for {
		data, ok := <-chan_urlindexes

		if ok == false {
			return // Abbruch wenn der Channel geschlossen ist
		}

		fmt.Fprintf(w, "\nURL: %s\nMAP:\n%v\n\n\n\n", data.URL, data.WORDS)
		w.Flush()
	}
}

//writeDB holt sich daten (structs vom Typ URLINDEX) aus dem Channel chan_urlindexes
// und schreibt die Daten in eine sqlite3 Datenbank: crwld.db
func writeDB() {
	os.Remove("./crwld.db")

	db, err := sql.Open("sqlite3", "./crwld.db")
	if err != nil {
		fmt.Println(err)
		return
	}
	defer db.Close()

	//Neue Tabelle(n) erstellen
	sqls := []string{
		"create table data (url text not null, word text, count int)",
		"delete from data",
	}
	for _, sql := range sqls {
		_, err = db.Exec(sql)
		if err != nil {
			fmt.Printf("%q: %s\n", err, sql)
			return
		}
	}

	// Endlosworker
	for {
		index, ok := <-chan_urlindexes //Neue Arbeit aus dem Channel holen

		if ok == false {
			return // Abbruch wenn der Channel geschlossen ist
		}

		tx, err := db.Begin()
		if err != nil {
			fmt.Println(err)
			return
		}

		stmt, err := tx.Prepare("insert into data(url, word, count) values(?, ?, ?)") //Maske für das SQL Statement setzen
		if err != nil {
			fmt.Println(err)
			return
		}
		defer stmt.Close()

		// Die gesamte MAP durchlaufen und für jeden Key ein SQl Statement zusammensetzen
		for element := range index.WORDS {
			_, err = stmt.Exec(index.URL, element, index.WORDS[element])
			if err != nil {
				fmt.Println(err)
				return
			}
		}
		tx.Commit()
	}
}

func main() {
	starturl := URL{"http://www.ebay.com", 0} // Start URL mit Linktiefe 0 festlegen
	chan_urls <- starturl                     // URL in den Channel legen
	crwldurls = make(map[string]bool)
	//go save() // File Writer starten
	debugausgabe("Starte DB Writer")
	go writeDB() //DB Writer starten
	debugausgabe("Starte Crawler")
	starten() // rawler starten

}
<commit_msg>Kommentare<commit_after>package main

import (
	"bufio"
	"code.google.com/p/go.net/html" //Tokenizer für HTML
	"database/sql"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"io"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"
)

const MaxOutstanding_Fetcher = 100 // Maximale Anzahl gleichzeitger Fetcher
const DEBUG = 1                    // Debugausgabe an/aus
const MaxLinkDepth = 1             // Maximale Linktiefe

type URLINDEX struct {
	URL   string
	WORDS map[string]int
}
type HTTPRESP struct {
	URL       string
	LINKDEPTH int
	FD        io.Reader
}
type URL struct {
	URL       string
	LINKDEPTH int
}

var crwldurls map[string]bool // globale URL Map - um doppeltes HTTP GET zu vermeiden

//Channels
var chan_urls = make(chan URL, 100000) // buffered channel of strings
//bei aktueller implementierung des Parser aufrufs, müsste der io_reader channel nicht buffered sein
var chan_ioreaders = make(chan HTTPRESP, 10)   // buffered channel of structs
var chan_urlindexes = make(chan URLINDEX, 100) // buffered channel of structs

//Semaphore Channel
var sem_Fetcher = make(chan int, MaxOutstanding_Fetcher)

func debugausgabe(msg string) {
	if DEBUG == 1 {
		fmt.Printf("%s DEBUG: %s\n", time.Now(), msg)
	}
}

func handleFetcher(url URL) {
	<-sem_Fetcher // Eine Ressource verbrauchen: Lock falls bereits alle verbraucht
	fetchURL(url)
	sem_Fetcher <- 1 // Eine Ressource wieder freigeben
}

func starten() {
	// Initiales setzen der Ressourcen
	for i := 0; i < MaxOutstanding_Fetcher; i++ {
		sem_Fetcher <- 1
	}
	// Endless (as long as the channel is not empty) Fetcher Spawning
	for {
		go handleFetcher(<-chan_urls)
	}
}

// parseHTML bekommt eine komplette HTML Seite
// und legt eine Map mit Wörtern und (viele) einzelne Links in entsprechende Channcd els
func parseHtml(a HTTPRESP) {
	//start := time.Now()
	d := html.NewTokenizer(a.FD)
	var words map[string]int
	words = make(map[string]int)

	for {
		// token type
		tokenType := d.Next()
		// ErrorToken kommt (auch) beim Ende der Daten
		if tokenType == html.ErrorToken {
			chan_urlindexes <- URLINDEX{a.URL, words} // WORD-Map in den Channel legen
			//fmt.Printf("Parse-Dauer : [%.2fs]  URL: %s\n", time.Since(start).Seconds(), a.URL)
			return
		}
		token := d.Token()
		switch tokenType {
		case html.StartTagToken: // <tag>
			// Links finden
			if token.Data == "a" {
				for _, element := range token.Attr {
					if element.Key == "href" {
						// Link normalisieren
						url, err := url.Parse(element.Val)
						// Nur Absolute Links die nicht in der globalen Link Map sind
						if url.IsAbs() && err == nil && url.Scheme == "http" && crwldurls[url.String()] != true && a.LINKDEPTH < MaxLinkDepth {
							chan_urls <- URL{url.String(), a.LINKDEPTH + 1} // Die URL in den Channel legen und Linktiefe hochzählen
						}
					}
				}
			}

		case html.TextToken: // text between start and end tag
			//Map mit Wörtern erstellen
			temp := strings.Fields(token.Data) //Aufteilen in Einzelne Wörter, trennen bei Whitespace
			for _, element := range temp {
				//TODO: einzelne Örter noch besser von Sonderzeichen trennen z.b. mit TRIM()
				words[element] = words[element] + 1
			}

			//fmt.Printf("%q\n", temp)
		case html.EndTagToken: // </tag>
		case html.SelfClosingTagToken: // <tag/>

		}
	}
}

// fetchURL Bekommt eine URL
// lädt die Seite herunter
// und legt ein struct vom Typ HTTPRESP in den Channel chan_ioreaders
func fetchURL(url URL) {
	start := time.Now()
	crwldurls[url.URL] = true //URL in die globale URL Liste aufnehmen damit sie nicht nochmal in den Work Queue kommt.
	response, err := http.Get(url.URL)
	if err != nil {
		fmt.Printf("Fehler: %s beim HTTP GET von: %s\n", err, url.URL)
		return
	}
	//fmt.Printf("%T", response.Body)
	chan_ioreaders <- HTTPRESP{url.URL, url.LINKDEPTH, response.Body}
	fmt.Printf("HTTP GET Dauer: [%.2fs]  URL: %s Tiefe: %d\n", time.Since(start).Seconds(), url.URL, url.LINKDEPTH)
	//TODO io.reader channel umstellen auf direkte übergabe des structs
	go parseHtml(<-chan_ioreaders) // Für jeden Fetcher wird ein Parser aufgerufen, der unabhängig läuft
	return
}

//save holt sich daten (structs vom Typ URLINDEX) aus dem Channel chan_urlindexes
// und schreibt die Daten in eine Datei "output.txt"
func save() {

	//fo, err := os.OpenFile("output.txt", os.O_APPEND, 0777)
	fo, err := os.Create("output.txt")
	if err != nil {
		panic(err)
	}
	// close fo on exit and check for its returned error
	defer func() {
		if err := fo.Close(); err != nil {
			panic(err)
		}
	}()
	// make a write buffer
	w := bufio.NewWriter(fo)
	for {
		data, ok := <-chan_urlindexes

		if ok == false {
			return // Abbruch wenn der Channel geschlossen ist
		}

		fmt.Fprintf(w, "\nURL: %s\nMAP:\n%v\n\n\n\n", data.URL, data.WORDS)
		w.Flush()
	}
}

//writeDB holt sich daten (structs vom Typ URLINDEX) aus dem Channel chan_urlindexes
// und schreibt die Daten in eine sqlite3 Datenbank: crwld.db
func writeDB() {
	os.Remove("./crwld.db")

	db, err := sql.Open("sqlite3", "./crwld.db")
	if err != nil {
		fmt.Println(err)
		return
	}
	defer db.Close()

	//Neue Tabelle(n) erstellen
	sqls := []string{
		"create table data (url text not null, word text, count int)",
		"delete from data",
	}
	for _, sql := range sqls {
		_, err = db.Exec(sql)
		if err != nil {
			fmt.Printf("%q: %s\n", err, sql)
			return
		}
	}

	// Endlosworker
	for {
		index, ok := <-chan_urlindexes //Neue Arbeit aus dem Channel holen

		if ok == false {
			return // Abbruch wenn der Channel geschlossen ist
		}

		tx, err := db.Begin()
		if err != nil {
			fmt.Println(err)
			return
		}

		stmt, err := tx.Prepare("insert into data(url, word, count) values(?, ?, ?)") //Maske für das SQL Statement setzen
		if err != nil {
			fmt.Println(err)
			return
		}
		defer stmt.Close()

		// Die gesamte MAP durchlaufen und für jeden Key ein SQl Statement zusammensetzen
		for element := range index.WORDS {
			_, err = stmt.Exec(index.URL, element, index.WORDS[element])
			if err != nil {
				fmt.Println(err)
				return
			}
		}
		tx.Commit()
	}
}

func main() {
	starturl := URL{"http://blogs.technet.com/", 0} // Start URL mit Linktiefe 0 festlegen
	chan_urls <- starturl                           // URL in den Channel legen
	crwldurls = make(map[string]bool)
	//go save() // File Writer starten
	debugausgabe("Starte DB Writer")
	go writeDB() //DB Writer starten
	debugausgabe("Starte Crawler")
	starten() // rawler starten

}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"code.google.com/p/go.net/html" //Tokenizer für HTML
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"
)

type URLINDEX struct {
	URL   string
	WORDS map[string]int
}

type HTTPRESP struct {
	URL string
	FD  io.Reader
}

const MaxOutstanding_URL = 100
const MaxOutstanding_RESP = 4
const DEBUG = 1

//Channels
var chan_urls = make(chan string, 100000)      // buffered channel of strings
var chan_ioreaders = make(chan HTTPRESP, 200)  // buffered channel of structs
var chan_urlindexes = make(chan URLINDEX, 100) // buffered channel of structs
//Semaphore Channels
var sem_URL = make(chan int, MaxOutstanding_URL)
var sem_RESP = make(chan int, MaxOutstanding_RESP)

func debugausgabe(msg string) {
	if DEBUG == 1 {

		fmt.Printf("\n%s DEBUG: %s\n", time.Now(), msg)
	}
}

func handleFetcher(url string) {
	//debugausgabe("Fetcher starten")
	<-sem_URL     // Wait for active queue to drain.
	fetchURL(url) // May take a long time.
	sem_URL <- 1  // Done; enable next request to run.
}

func handleParser(a HTTPRESP) {
	//debugausgabe("Parser starten")
	<-sem_RESP    // Wait for active queue to drain.
	parseHtml(a)  // May take a long time.
	sem_RESP <- 1 // Done; enable next request to run.
}

func starten() {
	//Initiales setzen der Ressourcen
	for i := 0; i < MaxOutstanding_URL; i++ {
		sem_URL <- 1
	}
	//Initiales setzen der Ressourcen
	for i := 0; i < MaxOutstanding_RESP; i++ {
		sem_RESP <- 1
	}
	for {
		work := <-chan_urls
		go handleFetcher(work) // Don't wait for handle to finish.
		work2 := <-chan_ioreaders
		go handleParser(work2)
	}
}

// parseHTML bekommt eine komplette HTML Seite
// und gibt je eine Map mit Links und Wörtern zurück
func parseHtml(a HTTPRESP) {
	start := time.Now()
	d := html.NewTokenizer(a.FD)
	var links map[string]int
	var words map[string]int
	links = make(map[string]int)
	words = make(map[string]int)

	for {
		// token type
		tokenType := d.Next()
		if d.Err() != nil {
			fmt.Printf("TokenERR vorher: %v", d.Err())
		}

		if tokenType == html.ErrorToken {
			chan_urlindexes <- URLINDEX{a.URL, words}
			fmt.Printf("Parse-Dauer : [%.2fs]  URL: %s\n", time.Since(start).Seconds(), a.URL)
			return
		}
		token := d.Token()
		switch tokenType {
		case html.StartTagToken: // <tag>
			//Eine Map mit Links erstellen
			if token.Data == "a" {
				for _, element := range token.Attr {
					if element.Key == "href" {
						//Link normalisieren
						url, err := url.Parse(element.Val)
						//debugausgabe(url.Scheme)
						if url.IsAbs() && err == nil && url.Scheme == "http" {
							//neuen Link zur Map hinzufügen und hochzählen
							links[element.Val] = links[element.Val] + 1
							if links[element.Val] == 1 {
								chan_urls <- element.Val //URL CHANNEL füllen
							}

						}
					}
				}
			}

		case html.TextToken: // text between start and end tag
			//Map mit Wörtern erstellen
			temp := strings.Fields(token.Data)
			for _, element := range temp {
				//TODO: einzelne Örter noch besser von Sonderzeichen trennen z.b. mit TRIM()
				words[element] = words[element] + 1
			}

			//fmt.Printf("%q\n", temp)
		case html.EndTagToken: // </tag>
		case html.SelfClosingTagToken: // <tag/>

		}
	}
}

// fetchURL Bekommt eine URL
// lädt die Seite herunter
// Gibt die komplette Seite zurück
func fetchURL(url string) {
	//start := time.Now()
	response, err := http.Get(url)
	if err != nil {
		fmt.Printf("Fehler: %s beim HTTP GET von: %s\n", err, url)
		return
	}
	//fmt.Printf("%T", response.Body)
	chan_ioreaders <- HTTPRESP{url, response.Body}
	//fmt.Printf("Dauer : [%.2fs]  URL: %s\n", time.Since(start).Seconds(), url)
	return
}

//save holt sich daten (structs vom Typ urlindexes) aus dem Channel chan_urlindexes
// und schreibt die Daten in eine Datei "output.txt"
func save() {

	//fo, err := os.OpenFile("output.txt", os.O_APPEND, 0777)
	fo, err := os.Create("output.txt")
	if err != nil {
		panic(err)
	}
	// close fo on exit and check for its returned error
	defer func() {
		if err := fo.Close(); err != nil {
			panic(err)
		}
	}()
	// make a write buffer
	w := bufio.NewWriter(fo)
	for {
		data := <-chan_urlindexes
		fmt.Fprintf(w, "\nURL: %s\nMAP:\n%v\n\n\n\n", data.URL, data.WORDS)
		w.Flush()
	}
}

func main() {

	chan_urls <- "http://www.ebay.com"

	go save()
	starten()

	//TODO paralleles schreiben in db
}
<commit_msg>Einfaches schreiben in SQLiteDB doppelte Wörter werden noch nicht herausgefiltert oder hochgezählt DB Struktur ist nur eine Tabelle mit allen Daten<commit_after>package main

import (
	"bufio"
	"code.google.com/p/go.net/html" //Tokenizer für HTML
	"database/sql"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"io"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"
)

type URLINDEX struct {
	URL   string
	WORDS map[string]int
}

type HTTPRESP struct {
	URL string
	FD  io.Reader
}

const MaxOutstanding_URL = 100
const MaxOutstanding_RESP = 4
const DEBUG = 1

//Channels
var chan_urls = make(chan string, 100000)      // buffered channel of strings
var chan_ioreaders = make(chan HTTPRESP, 200)  // buffered channel of structs
var chan_urlindexes = make(chan URLINDEX, 100) // buffered channel of structs
//Semaphore Channels
var sem_URL = make(chan int, MaxOutstanding_URL)
var sem_RESP = make(chan int, MaxOutstanding_RESP)

func debugausgabe(msg string) {
	if DEBUG == 1 {

		fmt.Printf("\n%s DEBUG: %s\n", time.Now(), msg)
	}
}

func handleFetcher(url string) {
	//debugausgabe("Fetcher starten")
	<-sem_URL     // Wait for active queue to drain.
	fetchURL(url) // May take a long time.
	sem_URL <- 1  // Done; enable next request to run.
}

func handleParser(a HTTPRESP) {
	//debugausgabe("Parser starten")
	<-sem_RESP    // Wait for active queue to drain.
	parseHtml(a)  // May take a long time.
	sem_RESP <- 1 // Done; enable next request to run.
}

func starten() {
	//Initiales setzen der Ressourcen
	for i := 0; i < MaxOutstanding_URL; i++ {
		sem_URL <- 1
	}
	//Initiales setzen der Ressourcen
	for i := 0; i < MaxOutstanding_RESP; i++ {
		sem_RESP <- 1
	}
	for {
		work := <-chan_urls
		go handleFetcher(work) // Don't wait for handle to finish.
		work2 := <-chan_ioreaders
		go handleParser(work2)
	}
}

// parseHTML bekommt eine komplette HTML Seite
// und gibt je eine Map mit Links und Wörtern zurück
func parseHtml(a HTTPRESP) {
	start := time.Now()
	d := html.NewTokenizer(a.FD)
	var links map[string]int
	var words map[string]int
	links = make(map[string]int)
	words = make(map[string]int)

	for {
		// token type
		tokenType := d.Next()
		// if d.Err() != nil {
		// 	fmt.Printf("TokenERR vorher: %v", d.Err())
		// }

		if tokenType == html.ErrorToken {
			chan_urlindexes <- URLINDEX{a.URL, words}
			fmt.Printf("Parse-Dauer : [%.2fs]  URL: %s\n", time.Since(start).Seconds(), a.URL)
			return
		}
		token := d.Token()
		switch tokenType {
		case html.StartTagToken: // <tag>
			//Eine Map mit Links erstellen
			if token.Data == "a" {
				for _, element := range token.Attr {
					if element.Key == "href" {
						//Link normalisieren
						url, err := url.Parse(element.Val)
						//debugausgabe(url.Scheme)
						if url.IsAbs() && err == nil && url.Scheme == "http" {
							//neuen Link zur Map hinzufügen und hochzählen
							links[element.Val] = links[element.Val] + 1
							if links[element.Val] == 1 {
								chan_urls <- element.Val //URL CHANNEL füllen
							}

						}
					}
				}
			}

		case html.TextToken: // text between start and end tag
			//Map mit Wörtern erstellen
			temp := strings.Fields(token.Data)
			for _, element := range temp {
				//TODO: einzelne Örter noch besser von Sonderzeichen trennen z.b. mit TRIM()
				words[element] = words[element] + 1
			}

			//fmt.Printf("%q\n", temp)
		case html.EndTagToken: // </tag>
		case html.SelfClosingTagToken: // <tag/>

		}
	}
}

// fetchURL Bekommt eine URL
// lädt die Seite herunter
// Gibt die komplette Seite zurück
func fetchURL(url string) {
	//start := time.Now()
	response, err := http.Get(url)
	if err != nil {
		fmt.Printf("Fehler: %s beim HTTP GET von: %s\n", err, url)
		return
	}
	//fmt.Printf("%T", response.Body)
	chan_ioreaders <- HTTPRESP{url, response.Body}
	//fmt.Printf("Dauer : [%.2fs]  URL: %s\n", time.Since(start).Seconds(), url)
	return
}

//save holt sich daten (structs vom Typ urlindexes) aus dem Channel chan_urlindexes
// und schreibt die Daten in eine Datei "output.txt"
func save() {

	//fo, err := os.OpenFile("output.txt", os.O_APPEND, 0777)
	fo, err := os.Create("output.txt")
	if err != nil {
		panic(err)
	}
	// close fo on exit and check for its returned error
	defer func() {
		if err := fo.Close(); err != nil {
			panic(err)
		}
	}()
	// make a write buffer
	w := bufio.NewWriter(fo)
	for {
		data := <-chan_urlindexes
		fmt.Fprintf(w, "\nURL: %s\nMAP:\n%v\n\n\n\n", data.URL, data.WORDS)
		w.Flush()
	}
}

func writeDB() {
	os.Remove("./crwld.db")

	db, err := sql.Open("sqlite3", "./crwld.db")
	if err != nil {
		fmt.Println(err)
		return
	}
	defer db.Close()

	sqls := []string{
		"create table data (url text not null, word text, count int)",
		"delete from data",
	}
	for _, sql := range sqls {
		_, err = db.Exec(sql)
		if err != nil {
			fmt.Printf("%q: %s\n", err, sql)
			return
		}
	}

	tx, err := db.Begin()
	if err != nil {
		fmt.Println(err)
		return
	}

	// Endlosworker
	for {
		//Holt sich neue Arbeit aus dem Channel
		index := <-chan_urlindexes

		stmt, err := tx.Prepare("insert into data(url, word, count) values(?, ?, ?)")
		if err != nil {
			fmt.Println(err)
			return
		}
		defer stmt.Close()

		for element := range index.WORDS {
			_, err = stmt.Exec(index.URL, element, index.WORDS[element])
			if err != nil {
				fmt.Println(err)
				return
			}
		}
		tx.Commit()
	}
}

func main() {

	chan_urls <- "http://www.ebay.com"

	//go save()
	go writeDB()
	starten()

	//TODO paralleles schreiben in db
}
<|endoftext|>


<commit_before>// JSON Data Storage
package golb

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"sort"
	"strings"
	"time"
)

type Comments []Comment

type Comment struct {
	Date    time.Time
	Name    string
	Email   string
	URL     string
	Comment string
	Enabled bool
}

type Articles []Article

type Article struct {
	Date     time.Time
	Title    string
	Slug     string
	Body     string
	Tags     []string
	Enabled  bool
	Author   string
	Comments Comments
}

type Data struct {
	Articles Articles
	Name     string
}

func Open(name string) *Data {
	d := new(Data)
	d.Name = name
	return d
}

func (d *Data) Read() error {
	data, err := ioutil.ReadFile(d.Name)
	if err != nil {
		return err
	}
	return json.Unmarshal(data, &d.Articles)
}

func (d *Data) Write() error {
	sort.Sort(d.Articles)
	for i, _ := range d.Articles {
		sort.Sort(d.Articles[i].Comments)
	}
	data, err := json.MarshalIndent(d.Articles, "", "\t")
	if err != nil {
		return err
	}
	return ioutil.WriteFile(d.Name, data, 0644)
}

func (d *Data) FindArticle(slug string) (Article, error) {
	for _, a := range d.Articles {
		if a.Slug == slug {
			return a, nil
		}
	}
	return Article{}, errors.New("not found")
}

func (a Article) MakeSlug() {
	r := strings.NewReplacer(" ", "-")
	a.Slug = strings.ToLower(r.Replace(a.Title))
}

func (a Article) AddComment(c Comment) {
	a.Comments = append(a.Comments, c)
}

func (a Article) Enable() {
	a.Date = time.Now()
	a.Enabled = true
}

func (a Article) Disable() {
	a.Enabled = false
}

func (a Articles) Len() int {
	return len(a)
}

func (a Articles) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}

func (a Articles) Less(i, j int) bool {
	return a[i].Date.Before(a[j].Date)
}

func (c Comment) Enable() {
	c.Enabled = true
}

func (c Comment) Disable() {
	c.Enabled = false
}

func (c Comments) Len() int {
	return len(c)
}

func (c Comments) Swap(i, j int) {
	c[i], c[j] = c[j], c[i]
}

func (c Comments) Less(i, j int) bool {
	return c[i].Date.Before(c[j].Date)
}
<commit_msg>reverse sort order<commit_after>// JSON Data Storage
package golb

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"sort"
	"strings"
	"time"
)

type Comments []Comment

type Comment struct {
	Date    time.Time
	Name    string
	Email   string
	URL     string
	Comment string
	Enabled bool
}

type Articles []Article

type Article struct {
	Date     time.Time
	Title    string
	Slug     string
	Body     string
	Tags     []string
	Enabled  bool
	Author   string
	Comments Comments
}

type Data struct {
	Articles Articles
	Name     string
}

func Open(name string) *Data {
	d := new(Data)
	d.Name = name
	return d
}

func (d *Data) Read() error {
	data, err := ioutil.ReadFile(d.Name)
	if err != nil {
		return err
	}
	return json.Unmarshal(data, &d.Articles)
}

func (d *Data) Write() error {
	sort.Sort(d.Articles)
	for i, _ := range d.Articles {
		sort.Sort(d.Articles[i].Comments)
	}
	data, err := json.MarshalIndent(d.Articles, "", "\t")
	if err != nil {
		return err
	}
	return ioutil.WriteFile(d.Name, data, 0644)
}

func (d *Data) FindArticle(slug string) (Article, error) {
	for _, a := range d.Articles {
		if a.Slug == slug {
			return a, nil
		}
	}
	return Article{}, errors.New("not found")
}

func (a Article) MakeSlug() {
	r := strings.NewReplacer(" ", "-")
	a.Slug = strings.ToLower(r.Replace(a.Title))
}

func (a Article) AddComment(c Comment) {
	a.Comments = append(a.Comments, c)
}

func (a Article) Enable() {
	a.Date = time.Now()
	a.Enabled = true
}

func (a Article) Disable() {
	a.Enabled = false
}

func (a Articles) Len() int {
	return len(a)
}

func (a Articles) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}

func (a Articles) Less(i, j int) bool {
	return a[i].Date.After(a[j].Date)
}

func (c Comment) Enable() {
	c.Enabled = true
}

func (c Comment) Disable() {
	c.Enabled = false
}

func (c Comments) Len() int {
	return len(c)
}

func (c Comments) Swap(i, j int) {
	c[i], c[j] = c[j], c[i]
}

func (c Comments) Less(i, j int) bool {
	return c[i].Date.After(c[j].Date)
}
<|endoftext|>


<commit_before>// Copyright 2013 The imageproxy authors.
// SPDX-License-Identifier: Apache-2.0

package imageproxy

import (
	"fmt"
	"net/http"
	"net/url"
	"regexp"
	"sort"
	"strconv"
	"strings"
)

const (
	optFit             = "fit"
	optFlipVertical    = "fv"
	optFlipHorizontal  = "fh"
	optFormatJPEG      = "jpeg"
	optFormatPNG       = "png"
	optFormatTIFF      = "tiff"
	optRotatePrefix    = "r"
	optQualityPrefix   = "q"
	optSignaturePrefix = "s"
	optSizeDelimiter   = "x"
	optScaleUp         = "scaleUp"
	optCropX           = "cx"
	optCropY           = "cy"
	optCropWidth       = "cw"
	optCropHeight      = "ch"
	optSmartCrop       = "sc"
)

// URLError reports a malformed URL error.
type URLError struct {
	Message string
	URL     *url.URL
}

func (e URLError) Error() string {
	return fmt.Sprintf("malformed URL %q: %s", e.URL, e.Message)
}

// Options specifies transformations to be performed on the requested image.
type Options struct {
	// See ParseOptions for interpretation of Width and Height values
	Width  float64
	Height float64

	// If true, resize the image to fit in the specified dimensions.  Image
	// will not be cropped, and aspect ratio will be maintained.
	Fit bool

	// Rotate image the specified degrees counter-clockwise.  Valid values
	// are 90, 180, 270.
	Rotate int

	FlipVertical   bool
	FlipHorizontal bool

	// Quality of output image
	Quality int

	// HMAC Signature for signed requests.
	Signature string

	// Allow image to scale beyond its original dimensions.  This value
	// will always be overwritten by the value of Proxy.ScaleUp.
	ScaleUp bool

	// Desired image format. Valid values are "jpeg", "png", "tiff".
	Format string

	// Crop rectangle params
	CropX      float64
	CropY      float64
	CropWidth  float64
	CropHeight float64

	// Automatically find good crop points based on image content.
	SmartCrop bool
}

func (o Options) String() string {
	opts := []string{fmt.Sprintf("%v%s%v", o.Width, optSizeDelimiter, o.Height)}
	if o.Fit {
		opts = append(opts, optFit)
	}
	if o.Rotate != 0 {
		opts = append(opts, fmt.Sprintf("%s%d", optRotatePrefix, o.Rotate))
	}
	if o.FlipVertical {
		opts = append(opts, optFlipVertical)
	}
	if o.FlipHorizontal {
		opts = append(opts, optFlipHorizontal)
	}
	if o.Quality != 0 {
		opts = append(opts, fmt.Sprintf("%s%d", optQualityPrefix, o.Quality))
	}
	if o.Signature != "" {
		opts = append(opts, fmt.Sprintf("%s%s", optSignaturePrefix, o.Signature))
	}
	if o.ScaleUp {
		opts = append(opts, optScaleUp)
	}
	if o.Format != "" {
		opts = append(opts, o.Format)
	}
	if o.CropX != 0 {
		opts = append(opts, fmt.Sprintf("%s%v", optCropX, o.CropX))
	}
	if o.CropY != 0 {
		opts = append(opts, fmt.Sprintf("%s%v", optCropY, o.CropY))
	}
	if o.CropWidth != 0 {
		opts = append(opts, fmt.Sprintf("%s%v", optCropWidth, o.CropWidth))
	}
	if o.CropHeight != 0 {
		opts = append(opts, fmt.Sprintf("%s%v", optCropHeight, o.CropHeight))
	}
	if o.SmartCrop {
		opts = append(opts, optSmartCrop)
	}
	sort.Strings(opts)
	return strings.Join(opts, ",")
}

// transform returns whether o includes transformation options.  Some fields
// are not transform related at all (like Signature), and others only apply in
// the presence of other fields (like Fit).  A non-empty Format value is
// assumed to involve a transformation.
func (o Options) transform() bool {
	return o.Width != 0 || o.Height != 0 || o.Rotate != 0 || o.FlipHorizontal || o.FlipVertical || o.Quality != 0 || o.Format != "" || o.CropX != 0 || o.CropY != 0 || o.CropWidth != 0 || o.CropHeight != 0
}

// ParseOptions parses str as a list of comma separated transformation options.
// The options can be specified in in order, with duplicate options overwriting
// previous values.
//
// Rectangle Crop
//
// There are four options controlling rectangle crop:
//
// 	cx{x}      - X coordinate of top left rectangle corner (default: 0)
// 	cy{y}      - Y coordinate of top left rectangle corner (default: 0)
// 	cw{width}  - rectangle width (default: image width)
// 	ch{height} - rectangle height (default: image height)
//
// For all options, integer values are interpreted as exact pixel values and
// floats between 0 and 1 are interpreted as percentages of the original image
// size. Negative values for cx and cy are measured from the right and bottom
// edges of the image, respectively.
//
// If the crop width or height exceed the width or height of the image, the
// crop width or height will be adjusted, preserving the specified cx and cy
// values.  Rectangular crop is applied before any other transformations.
//
// Smart Crop
//
// The "sc" option will perform a content-aware smart crop to fit the
// requested image width and height dimensions (see Size and Cropping below).
// The smart crop option will override any requested rectangular crop.
//
// Size and Cropping
//
// The size option takes the general form "{width}x{height}", where width and
// height are numbers. Integer values greater than 1 are interpreted as exact
// pixel values. Floats between 0 and 1 are interpreted as percentages of the
// original image size. If either value is omitted or set to 0, it will be
// automatically set to preserve the aspect ratio based on the other dimension.
// If a single number is provided (with no "x" separator), it will be used for
// both height and width.
//
// Depending on the size options specified, an image may be cropped to fit the
// requested size. In all cases, the original aspect ratio of the image will be
// preserved; imageproxy will never stretch the original image.
//
// When no explicit crop mode is specified, the following rules are followed:
//
// - If both width and height values are specified, the image will be scaled to
// fill the space, cropping if necessary to fit the exact dimension.
//
// - If only one of the width or height values is specified, the image will be
// resized to fit the specified dimension, scaling the other dimension as
// needed to maintain the aspect ratio.
//
// If the "fit" option is specified together with a width and height value, the
// image will be resized to fit within a containing box of the specified size.
// As always, the original aspect ratio will be preserved. Specifying the "fit"
// option with only one of either width or height does the same thing as if
// "fit" had not been specified.
//
// Rotation and Flips
//
// The "r{degrees}" option will rotate the image the specified number of
// degrees, counter-clockwise. Valid degrees values are 90, 180, and 270.
//
// The "fv" option will flip the image vertically. The "fh" option will flip
// the image horizontally. Images are flipped after being rotated.
//
// Quality
//
// The "q{qualityPercentage}" option can be used to specify the quality of the
// output file (JPEG only). If not specified, the default value of "95" is used.
//
// Format
//
// The "jpeg", "png", and "tiff"  options can be used to specify the desired
// image format of the proxied image.
//
// Signature
//
// The "s{signature}" option specifies an optional base64 encoded HMAC used to
// sign the remote URL in the request.  The HMAC key used to verify signatures is
// provided to the imageproxy server on startup.
//
// See https://github.com/willnorris/imageproxy/blob/master/docs/url-signing.md
// for examples of generating signatures.
//
// Examples
//
// 	0x0         - no resizing
// 	200x        - 200 pixels wide, proportional height
// 	x0.15       - 15% original height, proportional width
// 	100x150     - 100 by 150 pixels, cropping as needed
// 	100         - 100 pixels square, cropping as needed
// 	150,fit     - scale to fit 150 pixels square, no cropping
// 	100,r90     - 100 pixels square, rotated 90 degrees
// 	100,fv,fh   - 100 pixels square, flipped horizontal and vertical
// 	200x,q60    - 200 pixels wide, proportional height, 60% quality
// 	200x,png    - 200 pixels wide, converted to PNG format
// 	cw100,ch100 - crop image to 100px square, starting at (0,0)
// 	cx10,cy20,cw100,ch200 - crop image starting at (10,20) is 100px wide and 200px tall
func ParseOptions(str string) Options {
	var options Options

	for _, opt := range strings.Split(str, ",") {
		switch {
		case len(opt) == 0: // do nothing
		case opt == optFit:
			options.Fit = true
		case opt == optFlipVertical:
			options.FlipVertical = true
		case opt == optFlipHorizontal:
			options.FlipHorizontal = true
		case opt == optScaleUp: // this option is intentionally not documented above
			options.ScaleUp = true
		case opt == optFormatJPEG, opt == optFormatPNG, opt == optFormatTIFF:
			options.Format = opt
		case opt == optSmartCrop:
			options.SmartCrop = true
		case strings.HasPrefix(opt, optRotatePrefix):
			value := strings.TrimPrefix(opt, optRotatePrefix)
			options.Rotate, _ = strconv.Atoi(value)
		case strings.HasPrefix(opt, optQualityPrefix):
			value := strings.TrimPrefix(opt, optQualityPrefix)
			options.Quality, _ = strconv.Atoi(value)
		case strings.HasPrefix(opt, optSignaturePrefix):
			options.Signature = strings.TrimPrefix(opt, optSignaturePrefix)
		case strings.HasPrefix(opt, optCropX):
			value := strings.TrimPrefix(opt, optCropX)
			options.CropX, _ = strconv.ParseFloat(value, 64)
		case strings.HasPrefix(opt, optCropY):
			value := strings.TrimPrefix(opt, optCropY)
			options.CropY, _ = strconv.ParseFloat(value, 64)
		case strings.HasPrefix(opt, optCropWidth):
			value := strings.TrimPrefix(opt, optCropWidth)
			options.CropWidth, _ = strconv.ParseFloat(value, 64)
		case strings.HasPrefix(opt, optCropHeight):
			value := strings.TrimPrefix(opt, optCropHeight)
			options.CropHeight, _ = strconv.ParseFloat(value, 64)
		case strings.Contains(opt, optSizeDelimiter):
			size := strings.SplitN(opt, optSizeDelimiter, 2)
			if w := size[0]; w != "" {
				options.Width, _ = strconv.ParseFloat(w, 64)
			}
			if h := size[1]; h != "" {
				options.Height, _ = strconv.ParseFloat(h, 64)
			}
		default:
			if size, err := strconv.ParseFloat(opt, 64); err == nil {
				options.Width = size
				options.Height = size
			}
		}
	}

	return options
}

// Request is an imageproxy request which includes a remote URL of an image to
// proxy, and an optional set of transformations to perform.
type Request struct {
	URL      *url.URL      // URL of the image to proxy
	Options  Options       // Image transformation to perform
	Original *http.Request // The original HTTP request
}

// String returns the request URL as a string, with r.Options encoded in the
// URL fragment.
func (r Request) String() string {
	u := *r.URL
	u.Fragment = r.Options.String()
	return u.String()
}

// NewRequest parses an http.Request into an imageproxy Request.  Options and
// the remote image URL are specified in the request path, formatted as:
// /{options}/{remote_url}.  Options may be omitted, so a request path may
// simply contain /{remote_url}.  The remote URL must be an absolute "http" or
// "https" URL, should not be URL encoded, and may contain a query string.
//
// Assuming an imageproxy server running on localhost, the following are all
// valid imageproxy requests:
//
// 	http://localhost/100x200/http://example.com/image.jpg
// 	http://localhost/100x200,r90/http://example.com/image.jpg?foo=bar
// 	http://localhost//http://example.com/image.jpg
// 	http://localhost/http://example.com/image.jpg
func NewRequest(r *http.Request, baseURL *url.URL) (*Request, error) {
	var err error
	req := &Request{Original: r}

	path := r.URL.EscapedPath()[1:] // strip leading slash
	req.URL, err = parseURL(path)
	if err != nil || !req.URL.IsAbs() {
		// first segment should be options
		parts := strings.SplitN(path, "/", 2)
		if len(parts) != 2 {
			return nil, URLError{"too few path segments", r.URL}
		}

		var err error
		req.URL, err = parseURL(parts[1])
		if err != nil {
			return nil, URLError{fmt.Sprintf("unable to parse remote URL: %v", err), r.URL}
		}

		req.Options = ParseOptions(parts[0])
	}

	if baseURL != nil {
		req.URL = baseURL.ResolveReference(req.URL)
	}

	if !req.URL.IsAbs() {
		return nil, URLError{"must provide absolute remote URL", r.URL}
	}

	if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
		return nil, URLError{"remote URL must have http or https scheme", r.URL}
	}

	// query string is always part of the remote URL
	req.URL.RawQuery = r.URL.RawQuery
	return req, nil
}

var reCleanedURL = regexp.MustCompile(`^(https?):/+([^/])`)

// parseURL parses s as a URL, handling URLs that have been munged by
// path.Clean or a webserver that collapses multiple slashes.
func parseURL(s string) (*url.URL, error) {
	s = reCleanedURL.ReplaceAllString(s, "$1://$2")
	return url.Parse(s)
}
<commit_msg>godoc: fix godoc formatting<commit_after>// Copyright 2013 The imageproxy authors.
// SPDX-License-Identifier: Apache-2.0

package imageproxy

import (
	"fmt"
	"net/http"
	"net/url"
	"regexp"
	"sort"
	"strconv"
	"strings"
)

const (
	optFit             = "fit"
	optFlipVertical    = "fv"
	optFlipHorizontal  = "fh"
	optFormatJPEG      = "jpeg"
	optFormatPNG       = "png"
	optFormatTIFF      = "tiff"
	optRotatePrefix    = "r"
	optQualityPrefix   = "q"
	optSignaturePrefix = "s"
	optSizeDelimiter   = "x"
	optScaleUp         = "scaleUp"
	optCropX           = "cx"
	optCropY           = "cy"
	optCropWidth       = "cw"
	optCropHeight      = "ch"
	optSmartCrop       = "sc"
)

// URLError reports a malformed URL error.
type URLError struct {
	Message string
	URL     *url.URL
}

func (e URLError) Error() string {
	return fmt.Sprintf("malformed URL %q: %s", e.URL, e.Message)
}

// Options specifies transformations to be performed on the requested image.
type Options struct {
	// See ParseOptions for interpretation of Width and Height values
	Width  float64
	Height float64

	// If true, resize the image to fit in the specified dimensions.  Image
	// will not be cropped, and aspect ratio will be maintained.
	Fit bool

	// Rotate image the specified degrees counter-clockwise.  Valid values
	// are 90, 180, 270.
	Rotate int

	FlipVertical   bool
	FlipHorizontal bool

	// Quality of output image
	Quality int

	// HMAC Signature for signed requests.
	Signature string

	// Allow image to scale beyond its original dimensions.  This value
	// will always be overwritten by the value of Proxy.ScaleUp.
	ScaleUp bool

	// Desired image format. Valid values are "jpeg", "png", "tiff".
	Format string

	// Crop rectangle params
	CropX      float64
	CropY      float64
	CropWidth  float64
	CropHeight float64

	// Automatically find good crop points based on image content.
	SmartCrop bool
}

func (o Options) String() string {
	opts := []string{fmt.Sprintf("%v%s%v", o.Width, optSizeDelimiter, o.Height)}
	if o.Fit {
		opts = append(opts, optFit)
	}
	if o.Rotate != 0 {
		opts = append(opts, fmt.Sprintf("%s%d", optRotatePrefix, o.Rotate))
	}
	if o.FlipVertical {
		opts = append(opts, optFlipVertical)
	}
	if o.FlipHorizontal {
		opts = append(opts, optFlipHorizontal)
	}
	if o.Quality != 0 {
		opts = append(opts, fmt.Sprintf("%s%d", optQualityPrefix, o.Quality))
	}
	if o.Signature != "" {
		opts = append(opts, fmt.Sprintf("%s%s", optSignaturePrefix, o.Signature))
	}
	if o.ScaleUp {
		opts = append(opts, optScaleUp)
	}
	if o.Format != "" {
		opts = append(opts, o.Format)
	}
	if o.CropX != 0 {
		opts = append(opts, fmt.Sprintf("%s%v", optCropX, o.CropX))
	}
	if o.CropY != 0 {
		opts = append(opts, fmt.Sprintf("%s%v", optCropY, o.CropY))
	}
	if o.CropWidth != 0 {
		opts = append(opts, fmt.Sprintf("%s%v", optCropWidth, o.CropWidth))
	}
	if o.CropHeight != 0 {
		opts = append(opts, fmt.Sprintf("%s%v", optCropHeight, o.CropHeight))
	}
	if o.SmartCrop {
		opts = append(opts, optSmartCrop)
	}
	sort.Strings(opts)
	return strings.Join(opts, ",")
}

// transform returns whether o includes transformation options.  Some fields
// are not transform related at all (like Signature), and others only apply in
// the presence of other fields (like Fit).  A non-empty Format value is
// assumed to involve a transformation.
func (o Options) transform() bool {
	return o.Width != 0 || o.Height != 0 || o.Rotate != 0 || o.FlipHorizontal || o.FlipVertical || o.Quality != 0 || o.Format != "" || o.CropX != 0 || o.CropY != 0 || o.CropWidth != 0 || o.CropHeight != 0
}

// ParseOptions parses str as a list of comma separated transformation options.
// The options can be specified in in order, with duplicate options overwriting
// previous values.
//
// # Rectangle Crop
//
// There are four options controlling rectangle crop:
//
//	cx{x}      - X coordinate of top left rectangle corner (default: 0)
//	cy{y}      - Y coordinate of top left rectangle corner (default: 0)
//	cw{width}  - rectangle width (default: image width)
//	ch{height} - rectangle height (default: image height)
//
// For all options, integer values are interpreted as exact pixel values and
// floats between 0 and 1 are interpreted as percentages of the original image
// size. Negative values for cx and cy are measured from the right and bottom
// edges of the image, respectively.
//
// If the crop width or height exceed the width or height of the image, the
// crop width or height will be adjusted, preserving the specified cx and cy
// values.  Rectangular crop is applied before any other transformations.
//
// # Smart Crop
//
// The "sc" option will perform a content-aware smart crop to fit the
// requested image width and height dimensions (see Size and Cropping below).
// The smart crop option will override any requested rectangular crop.
//
// # Size and Cropping
//
// The size option takes the general form "{width}x{height}", where width and
// height are numbers. Integer values greater than 1 are interpreted as exact
// pixel values. Floats between 0 and 1 are interpreted as percentages of the
// original image size. If either value is omitted or set to 0, it will be
// automatically set to preserve the aspect ratio based on the other dimension.
// If a single number is provided (with no "x" separator), it will be used for
// both height and width.
//
// Depending on the size options specified, an image may be cropped to fit the
// requested size. In all cases, the original aspect ratio of the image will be
// preserved; imageproxy will never stretch the original image.
//
// When no explicit crop mode is specified, the following rules are followed:
//
// - If both width and height values are specified, the image will be scaled to
// fill the space, cropping if necessary to fit the exact dimension.
//
// - If only one of the width or height values is specified, the image will be
// resized to fit the specified dimension, scaling the other dimension as
// needed to maintain the aspect ratio.
//
// If the "fit" option is specified together with a width and height value, the
// image will be resized to fit within a containing box of the specified size.
// As always, the original aspect ratio will be preserved. Specifying the "fit"
// option with only one of either width or height does the same thing as if
// "fit" had not been specified.
//
// # Rotation and Flips
//
// The "r{degrees}" option will rotate the image the specified number of
// degrees, counter-clockwise. Valid degrees values are 90, 180, and 270.
//
// The "fv" option will flip the image vertically. The "fh" option will flip
// the image horizontally. Images are flipped after being rotated.
//
// # Quality
//
// The "q{qualityPercentage}" option can be used to specify the quality of the
// output file (JPEG only). If not specified, the default value of "95" is used.
//
// # Format
//
// The "jpeg", "png", and "tiff"  options can be used to specify the desired
// image format of the proxied image.
//
// # Signature
//
// The "s{signature}" option specifies an optional base64 encoded HMAC used to
// sign the remote URL in the request.  The HMAC key used to verify signatures is
// provided to the imageproxy server on startup.
//
// See https://github.com/willnorris/imageproxy/blob/master/docs/url-signing.md
// for examples of generating signatures.
//
// Examples
//
//	0x0         - no resizing
//	200x        - 200 pixels wide, proportional height
//	x0.15       - 15% original height, proportional width
//	100x150     - 100 by 150 pixels, cropping as needed
//	100         - 100 pixels square, cropping as needed
//	150,fit     - scale to fit 150 pixels square, no cropping
//	100,r90     - 100 pixels square, rotated 90 degrees
//	100,fv,fh   - 100 pixels square, flipped horizontal and vertical
//	200x,q60    - 200 pixels wide, proportional height, 60% quality
//	200x,png    - 200 pixels wide, converted to PNG format
//	cw100,ch100 - crop image to 100px square, starting at (0,0)
//	cx10,cy20,cw100,ch200 - crop image starting at (10,20) is 100px wide and 200px tall
func ParseOptions(str string) Options {
	var options Options

	for _, opt := range strings.Split(str, ",") {
		switch {
		case len(opt) == 0: // do nothing
		case opt == optFit:
			options.Fit = true
		case opt == optFlipVertical:
			options.FlipVertical = true
		case opt == optFlipHorizontal:
			options.FlipHorizontal = true
		case opt == optScaleUp: // this option is intentionally not documented above
			options.ScaleUp = true
		case opt == optFormatJPEG, opt == optFormatPNG, opt == optFormatTIFF:
			options.Format = opt
		case opt == optSmartCrop:
			options.SmartCrop = true
		case strings.HasPrefix(opt, optRotatePrefix):
			value := strings.TrimPrefix(opt, optRotatePrefix)
			options.Rotate, _ = strconv.Atoi(value)
		case strings.HasPrefix(opt, optQualityPrefix):
			value := strings.TrimPrefix(opt, optQualityPrefix)
			options.Quality, _ = strconv.Atoi(value)
		case strings.HasPrefix(opt, optSignaturePrefix):
			options.Signature = strings.TrimPrefix(opt, optSignaturePrefix)
		case strings.HasPrefix(opt, optCropX):
			value := strings.TrimPrefix(opt, optCropX)
			options.CropX, _ = strconv.ParseFloat(value, 64)
		case strings.HasPrefix(opt, optCropY):
			value := strings.TrimPrefix(opt, optCropY)
			options.CropY, _ = strconv.ParseFloat(value, 64)
		case strings.HasPrefix(opt, optCropWidth):
			value := strings.TrimPrefix(opt, optCropWidth)
			options.CropWidth, _ = strconv.ParseFloat(value, 64)
		case strings.HasPrefix(opt, optCropHeight):
			value := strings.TrimPrefix(opt, optCropHeight)
			options.CropHeight, _ = strconv.ParseFloat(value, 64)
		case strings.Contains(opt, optSizeDelimiter):
			size := strings.SplitN(opt, optSizeDelimiter, 2)
			if w := size[0]; w != "" {
				options.Width, _ = strconv.ParseFloat(w, 64)
			}
			if h := size[1]; h != "" {
				options.Height, _ = strconv.ParseFloat(h, 64)
			}
		default:
			if size, err := strconv.ParseFloat(opt, 64); err == nil {
				options.Width = size
				options.Height = size
			}
		}
	}

	return options
}

// Request is an imageproxy request which includes a remote URL of an image to
// proxy, and an optional set of transformations to perform.
type Request struct {
	URL      *url.URL      // URL of the image to proxy
	Options  Options       // Image transformation to perform
	Original *http.Request // The original HTTP request
}

// String returns the request URL as a string, with r.Options encoded in the
// URL fragment.
func (r Request) String() string {
	u := *r.URL
	u.Fragment = r.Options.String()
	return u.String()
}

// NewRequest parses an http.Request into an imageproxy Request.  Options and
// the remote image URL are specified in the request path, formatted as:
// /{options}/{remote_url}.  Options may be omitted, so a request path may
// simply contain /{remote_url}.  The remote URL must be an absolute "http" or
// "https" URL, should not be URL encoded, and may contain a query string.
//
// Assuming an imageproxy server running on localhost, the following are all
// valid imageproxy requests:
//
//	http://localhost/100x200/http://example.com/image.jpg
//	http://localhost/100x200,r90/http://example.com/image.jpg?foo=bar
//	http://localhost//http://example.com/image.jpg
//	http://localhost/http://example.com/image.jpg
func NewRequest(r *http.Request, baseURL *url.URL) (*Request, error) {
	var err error
	req := &Request{Original: r}

	path := r.URL.EscapedPath()[1:] // strip leading slash
	req.URL, err = parseURL(path)
	if err != nil || !req.URL.IsAbs() {
		// first segment should be options
		parts := strings.SplitN(path, "/", 2)
		if len(parts) != 2 {
			return nil, URLError{"too few path segments", r.URL}
		}

		var err error
		req.URL, err = parseURL(parts[1])
		if err != nil {
			return nil, URLError{fmt.Sprintf("unable to parse remote URL: %v", err), r.URL}
		}

		req.Options = ParseOptions(parts[0])
	}

	if baseURL != nil {
		req.URL = baseURL.ResolveReference(req.URL)
	}

	if !req.URL.IsAbs() {
		return nil, URLError{"must provide absolute remote URL", r.URL}
	}

	if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
		return nil, URLError{"remote URL must have http or https scheme", r.URL}
	}

	// query string is always part of the remote URL
	req.URL.RawQuery = r.URL.RawQuery
	return req, nil
}

var reCleanedURL = regexp.MustCompile(`^(https?):/+([^/])`)

// parseURL parses s as a URL, handling URLs that have been munged by
// path.Clean or a webserver that collapses multiple slashes.
func parseURL(s string) (*url.URL, error) {
	s = reCleanedURL.ReplaceAllString(s, "$1://$2")
	return url.Parse(s)
}
<|endoftext|>