<commit_before>package gotoc

import (
	"bytes"
	"fmt"
	"go/ast"
	"go/token"
	"strconv"
	"strings"

	"code.google.com/p/go.tools/go/types"
)

func (gtc *GTC) FuncDecl(d *ast.FuncDecl, il int) (cdds []*CDD) {
	f := gtc.object(d.Name).(*types.Func)

	cdd := gtc.newCDD(f, FuncDecl, il)
	w := new(bytes.Buffer)
	fname := cdd.NameStr(f, true)

	sig := f.Type().(*types.Signature)
	res, params := cdd.signature(sig, true)

	w.WriteString(res.typ)
	w.WriteByte(' ')
	w.WriteString(dimFuncPtr(fname+params, res.dim))

	cdds = append(cdds, res.acds...)
	cdds = append(cdds, cdd)

	cdd.init = (f.Name() == "init" && sig.Recv() == nil && !cdd.gtc.isLocal(f))

	if !cdd.init {
		cdd.copyDecl(w, ";\n")
	}

	if d.Body == nil {
		return
	}

	cdd.body = true

	w.WriteByte(' ')

	all := true
	if res.hasNames {
		cdd.indent(w)
		w.WriteString("{\n")
		cdd.il++
		for i, v := range res.fields {
			name := res.names[i]
			if name == "_" && len(res.fields) > 1 {
				all = false
				continue
			}
			cdd.indent(w)
			dim, acds := cdd.Type(w, v.Type())
			cdds = append(cdds, acds...)
			w.WriteByte(' ')
			w.WriteString(dimFuncPtr(name, dim))
			w.WriteString(" = {0};\n")
		}
		cdd.indent(w)
	}

	end, acds := cdd.BlockStmt(w, d.Body, res.typ, sig.Results())
	cdds = append(cdds, acds...)
	w.WriteByte('\n')

	if res.hasNames {
		if end {
			cdd.il--
			cdd.indent(w)
			w.WriteString("end:\n")
			cdd.il++

			cdd.indent(w)
			w.WriteString("return ")
			if len(res.fields) == 1 {
				w.WriteString(res.names[0])
			} else {
				w.WriteString("(" + res.typ + "){")
				comma := false
				for i, name := range res.names {
					if name == "_" {
						continue
					}
					if comma {
						w.WriteString(", ")
					} else {
						comma = true
					}
					if !all {
						w.WriteString("._" + strconv.Itoa(i) + "=")
					}
					w.WriteString(name)
				}
				w.WriteByte('}')
			}
			w.WriteString(";\n")
		}
		cdd.il--
		w.WriteString("}\n")
	}
	cdd.copyDef(w)

	if cdd.init {
		cdd.Init = []byte("\t" + fname + "();\n")
	}
	return
}

func (gtc *GTC) GenDecl(d *ast.GenDecl, il int) (cdds []*CDD) {
	w := new(bytes.Buffer)

	switch d.Tok {
	case token.IMPORT:
		// Only for unrefferenced imports
		for _, s := range d.Specs {
			is := s.(*ast.ImportSpec)
			if is.Name != nil && is.Name.Name == "_" {
				cdd := gtc.newCDD(gtc.object(is.Name), ImportDecl, il)
				cdds = append(cdds, cdd)
			}
		}

	case token.CONST:
		for _, s := range d.Specs {
			vs := s.(*ast.ValueSpec)

			for _, n := range vs.Names {
				c := gtc.object(n).(*types.Const)

				// All constants in expressions are evaluated so
				// only exported constants need be translated to C
				if !c.Exported() {
					continue
				}

				cdd := gtc.newCDD(c, ConstDecl, il)

				w.WriteString("#define ")
				cdd.Name(w, c, true)
				w.WriteByte(' ')
				cdd.Value(w, c.Val(), c.Type())
				cdd.copyDecl(w, "\n")
				w.Reset()

				cdds = append(cdds, cdd)
			}
		}

	case token.VAR:
		for _, s := range d.Specs {
			vs := s.(*ast.ValueSpec)
			vals := vs.Values

			for i, n := range vs.Names {
				v := gtc.object(n).(*types.Var)
				cdd := gtc.newCDD(v, VarDecl, il)
				name := cdd.NameStr(v, true)

				var val ast.Expr
				if i < len(vals) {
					val = vals[i]
				}
				if i > 0 {
					cdd.indent(w)
				}
				acds := cdd.varDecl(w, v.Type(), cdd.gtc.isGlobal(v), name, val)

				w.Reset()

				cdds = append(cdds, cdd)
				cdds = append(cdds, acds...)
			}
		}

	case token.TYPE:
		for i, s := range d.Specs {
			ts := s.(*ast.TypeSpec)
			to := gtc.object(ts.Name)
			tt := gtc.exprType(ts.Type)
			cdd := gtc.newCDD(to, TypeDecl, il)
			name := cdd.NameStr(to, true)

			if i > 0 {
				cdd.indent(w)
			}

			switch typ := tt.(type) {
			case *types.Struct:
				cdd.structDecl(w, name, typ)

			default:
				w.WriteString("typedef ")
				dim, acds := cdd.Type(w, typ)
				cdds = append(cdds, acds...)
				w.WriteByte(' ')
				w.WriteString(dimFuncPtr(name, dim))
				cdd.copyDecl(w, ";\n")
			}
			w.Reset()

			cdds = append(cdds, cdd)
		}

	default:
		// Return fake CDD for unknown declaration
		cdds = []*CDD{{
			Decl: []byte(fmt.Sprintf("@%v (%T)@\n", d.Tok, d)),
		}}
	}
	return
}

func (cdd *CDD) varDecl(w *bytes.Buffer, typ types.Type, global bool, name string, val ast.Expr) (acds []*CDD) {

	dim, acds := cdd.Type(w, typ)
	w.WriteByte(' ')
	w.WriteString(dimFuncPtr(name, dim))

	constInit := true // true if C declaration can init value

	if global {
		cdd.copyDecl(w, ";\n") // Global variables may need declaration
		if val != nil {
			constInit = cdd.exprValue(val) != nil
		}
	}
	if constInit {
		w.WriteString(" = ")
		if val != nil {
			cdd.Expr(w, val, typ)
		} else {
			w.WriteString("{0}")
		}
	}
	w.WriteString(";\n")
	cdd.copyDef(w)

	if !constInit {
		// Runtime initialisation
		w.Reset()

		assign := false

		switch t := typ.(type) {
		case *types.Slice:
			switch vt := val.(type) {
			case *ast.CompositeLit:
				aname := "array" + cdd.gtc.uniqueId()
				at := types.NewArray(t.Elem(), int64(len(vt.Elts)))
				o := types.NewVar(vt.Lbrace, cdd.gtc.pkg, aname, at)
				cdd.gtc.pkg.Scope().Insert(o)
				acd := cdd.gtc.newCDD(o, VarDecl, cdd.il)
				av := *vt
				cdd.gtc.ti.Types[&av] = types.TypeAndValue{Type: at} // BUG: thread-unsafe
				n := w.Len()
				acd.varDecl(w, o.Type(), cdd.gtc.isGlobal(o), aname, &av)
				w.Truncate(n)
				acds = append(acds, acd)

				w.WriteByte('\t')
				w.WriteString(name)
				w.WriteString(" = ASLICE(")
				w.WriteString(aname)
				w.WriteString(");\n")

			default:
				assign = true
			}

		case *types.Array:
			w.WriteByte('\t')
			w.WriteString("ACPY(")
			w.WriteString(name)
			w.WriteString(", ")

			switch val.(type) {
			case *ast.CompositeLit:
				w.WriteString("((")
				dim, _ := cdd.Type(w, t.Elem())
				dim = append([]string{"[]"}, dim...)
				w.WriteString("(" + dimFuncPtr("", dim) + "))")
				cdd.Expr(w, val, typ)

			default:
				cdd.Expr(w, val, typ)
			}

			w.WriteString("));\n")

		case *types.Pointer:
			u, ok := val.(*ast.UnaryExpr)
			if !ok {
				assign = true
				break
			}
			c, ok := u.X.(*ast.CompositeLit)
			if !ok {
				assign = true
				break
			}
			cname := "cl" + cdd.gtc.uniqueId()
			ct := cdd.exprType(c)
			o := types.NewVar(c.Lbrace, cdd.gtc.pkg, cname, ct)
			cdd.gtc.pkg.Scope().Insert(o)
			acd := cdd.gtc.newCDD(o, VarDecl, cdd.il)
			n := w.Len()
			acd.varDecl(w, o.Type(), cdd.gtc.isGlobal(o), cname, c)
			w.Truncate(n)
			acds = append(acds, acd)

			w.WriteByte('\t')
			w.WriteString(name)
			w.WriteString(" = &")
			w.WriteString(cname)
			w.WriteString(";\n")

		default:
			assign = true
		}

		if assign {
			// Ordinary assignment gos to the init() function
			cdd.init = true
			w.WriteByte('\t')
			w.WriteString(name)
			w.WriteString(" = ")
			cdd.Expr(w, val, typ)
			w.WriteString(";\n")
		}
		cdd.copyInit(w)
	}
	return
}

func (cdd *CDD) structDecl(w *bytes.Buffer, name string, typ *types.Struct) {
	n := w.Len()

	w.WriteString("struct ")
	w.WriteString(name)
	w.WriteString("_struct;\n")
	cdd.indent(w)
	w.WriteString("typedef struct ")
	w.WriteString(name)
	w.WriteString("_struct ")
	w.WriteString(name)

	cdd.copyDecl(w, ";\n")
	w.Truncate(n)

	tuple := strings.Contains(name, "$$")

	if tuple {
		cdd.indent(w)
		w.WriteString("#ifndef " + name + "$\n")
		cdd.indent(w)
		w.WriteString("#define " + name + "$\n")
	}
	cdd.indent(w)
	w.WriteString("struct ")
	w.WriteString(name)
	w.WriteByte('_')
	cdd.Type(w, typ)
	w.WriteString(";\n")
	if tuple {
		cdd.indent(w)
		w.WriteString("#endif\n")
	}

	cdd.copyDef(w)
	w.Truncate(n)
}

func (cc *GTC) Decl(decl ast.Decl, il int) []*CDD {
	switch d := decl.(type) {
	case *ast.FuncDecl:
		return cc.FuncDecl(d, il)

	case *ast.GenDecl:
		return cc.GenDecl(d, il)
	}

	panic(fmt.Sprint("Unknown declaration: ", decl))
}
<commit_msg>Supprot for empty structs and arrays<commit_after>package gotoc

import (
	"bytes"
	"fmt"
	"go/ast"
	"go/token"
	"strconv"
	"strings"

	"code.google.com/p/go.tools/go/types"
)

func (gtc *GTC) FuncDecl(d *ast.FuncDecl, il int) (cdds []*CDD) {
	f := gtc.object(d.Name).(*types.Func)

	cdd := gtc.newCDD(f, FuncDecl, il)
	w := new(bytes.Buffer)
	fname := cdd.NameStr(f, true)

	sig := f.Type().(*types.Signature)
	res, params := cdd.signature(sig, true)

	w.WriteString(res.typ)
	w.WriteByte(' ')
	w.WriteString(dimFuncPtr(fname+params, res.dim))

	cdds = append(cdds, res.acds...)
	cdds = append(cdds, cdd)

	cdd.init = (f.Name() == "init" && sig.Recv() == nil && !cdd.gtc.isLocal(f))

	if !cdd.init {
		cdd.copyDecl(w, ";\n")
	}

	if d.Body == nil {
		return
	}

	cdd.body = true

	w.WriteByte(' ')

	all := true
	if res.hasNames {
		cdd.indent(w)
		w.WriteString("{\n")
		cdd.il++
		for i, v := range res.fields {
			name := res.names[i]
			if name == "_" && len(res.fields) > 1 {
				all = false
				continue
			}
			cdd.indent(w)
			dim, acds := cdd.Type(w, v.Type())
			cdds = append(cdds, acds...)
			w.WriteByte(' ')
			w.WriteString(dimFuncPtr(name, dim))
			w.WriteString(" = {0};\n")
		}
		cdd.indent(w)
	}

	end, acds := cdd.BlockStmt(w, d.Body, res.typ, sig.Results())
	cdds = append(cdds, acds...)
	w.WriteByte('\n')

	if res.hasNames {
		if end {
			cdd.il--
			cdd.indent(w)
			w.WriteString("end:\n")
			cdd.il++

			cdd.indent(w)
			w.WriteString("return ")
			if len(res.fields) == 1 {
				w.WriteString(res.names[0])
			} else {
				w.WriteString("(" + res.typ + "){")
				comma := false
				for i, name := range res.names {
					if name == "_" {
						continue
					}
					if comma {
						w.WriteString(", ")
					} else {
						comma = true
					}
					if !all {
						w.WriteString("._" + strconv.Itoa(i) + "=")
					}
					w.WriteString(name)
				}
				w.WriteByte('}')
			}
			w.WriteString(";\n")
		}
		cdd.il--
		w.WriteString("}\n")
	}
	cdd.copyDef(w)

	if cdd.init {
		cdd.Init = []byte("\t" + fname + "();\n")
	}
	return
}

func (gtc *GTC) GenDecl(d *ast.GenDecl, il int) (cdds []*CDD) {
	w := new(bytes.Buffer)

	switch d.Tok {
	case token.IMPORT:
		// Only for unrefferenced imports
		for _, s := range d.Specs {
			is := s.(*ast.ImportSpec)
			if is.Name != nil && is.Name.Name == "_" {
				cdd := gtc.newCDD(gtc.object(is.Name), ImportDecl, il)
				cdds = append(cdds, cdd)
			}
		}

	case token.CONST:
		for _, s := range d.Specs {
			vs := s.(*ast.ValueSpec)

			for _, n := range vs.Names {
				c := gtc.object(n).(*types.Const)

				// All constants in expressions are evaluated so
				// only exported constants need be translated to C
				if !c.Exported() {
					continue
				}

				cdd := gtc.newCDD(c, ConstDecl, il)

				w.WriteString("#define ")
				cdd.Name(w, c, true)
				w.WriteByte(' ')
				cdd.Value(w, c.Val(), c.Type())
				cdd.copyDecl(w, "\n")
				w.Reset()

				cdds = append(cdds, cdd)
			}
		}

	case token.VAR:
		for _, s := range d.Specs {
			vs := s.(*ast.ValueSpec)
			vals := vs.Values

			for i, n := range vs.Names {
				v := gtc.object(n).(*types.Var)
				cdd := gtc.newCDD(v, VarDecl, il)
				name := cdd.NameStr(v, true)

				var val ast.Expr
				if i < len(vals) {
					val = vals[i]
				}
				if i > 0 {
					cdd.indent(w)
				}
				acds := cdd.varDecl(w, v.Type(), cdd.gtc.isGlobal(v), name, val)

				w.Reset()

				cdds = append(cdds, cdd)
				cdds = append(cdds, acds...)
			}
		}

	case token.TYPE:
		for i, s := range d.Specs {
			ts := s.(*ast.TypeSpec)
			to := gtc.object(ts.Name)
			tt := gtc.exprType(ts.Type)
			cdd := gtc.newCDD(to, TypeDecl, il)
			name := cdd.NameStr(to, true)

			if i > 0 {
				cdd.indent(w)
			}

			switch typ := tt.(type) {
			case *types.Struct:
				cdd.structDecl(w, name, typ)

			default:
				w.WriteString("typedef ")
				dim, acds := cdd.Type(w, typ)
				cdds = append(cdds, acds...)
				w.WriteByte(' ')
				w.WriteString(dimFuncPtr(name, dim))
				cdd.copyDecl(w, ";\n")
			}
			w.Reset()

			cdds = append(cdds, cdd)
		}

	default:
		// Return fake CDD for unknown declaration
		cdds = []*CDD{{
			Decl: []byte(fmt.Sprintf("@%v (%T)@\n", d.Tok, d)),
		}}
	}
	return
}

func (cdd *CDD) varDecl(w *bytes.Buffer, typ types.Type, global bool, name string, val ast.Expr) (acds []*CDD) {

	dim, acds := cdd.Type(w, typ)
	w.WriteByte(' ')
	w.WriteString(dimFuncPtr(name, dim))

	if t, ok := typ.(*types.Named); ok {
		typ = t.Underlying()
	}

	constInit := true // true if C declaration can init value

	if global {
		cdd.copyDecl(w, ";\n") // Global variables may need declaration
		if val != nil {
			constInit = cdd.exprValue(val) != nil
		}
	}

	if constInit {
		if val != nil {
			w.WriteString(" = ")
			cdd.Expr(w, val, typ)
		} else {
			if t, ok := typ.(*types.Struct); ok && t.NumFields() == 0 {
			} else if t, ok := typ.(*types.Array); ok && t.Len() == 0 {
			} else {
				w.WriteString(" = {0}")
			}
		}
	}
	w.WriteString(";\n")
	cdd.copyDef(w)

	if !constInit {
		// Runtime initialisation
		w.Reset()

		assign := false

		switch t := typ.(type) {
		case *types.Slice:
			switch vt := val.(type) {
			case *ast.CompositeLit:
				aname := "array" + cdd.gtc.uniqueId()
				at := types.NewArray(t.Elem(), int64(len(vt.Elts)))
				o := types.NewVar(vt.Lbrace, cdd.gtc.pkg, aname, at)
				cdd.gtc.pkg.Scope().Insert(o)
				acd := cdd.gtc.newCDD(o, VarDecl, cdd.il)
				av := *vt
				cdd.gtc.ti.Types[&av] = types.TypeAndValue{Type: at} // BUG: thread-unsafe
				n := w.Len()
				acd.varDecl(w, o.Type(), cdd.gtc.isGlobal(o), aname, &av)
				w.Truncate(n)
				acds = append(acds, acd)

				w.WriteByte('\t')
				w.WriteString(name)
				w.WriteString(" = ASLICE(")
				w.WriteString(aname)
				w.WriteString(");\n")

			default:
				assign = true
			}

		case *types.Array:
			w.WriteByte('\t')
			w.WriteString("ACPY(")
			w.WriteString(name)
			w.WriteString(", ")

			switch val.(type) {
			case *ast.CompositeLit:
				w.WriteString("((")
				dim, _ := cdd.Type(w, t.Elem())
				dim = append([]string{"[]"}, dim...)
				w.WriteString("(" + dimFuncPtr("", dim) + "))")
				cdd.Expr(w, val, typ)

			default:
				cdd.Expr(w, val, typ)
			}

			w.WriteString("));\n")

		case *types.Pointer:
			u, ok := val.(*ast.UnaryExpr)
			if !ok {
				assign = true
				break
			}
			c, ok := u.X.(*ast.CompositeLit)
			if !ok {
				assign = true
				break
			}
			cname := "cl" + cdd.gtc.uniqueId()
			ct := cdd.exprType(c)
			o := types.NewVar(c.Lbrace, cdd.gtc.pkg, cname, ct)
			cdd.gtc.pkg.Scope().Insert(o)
			acd := cdd.gtc.newCDD(o, VarDecl, cdd.il)
			n := w.Len()
			acd.varDecl(w, o.Type(), cdd.gtc.isGlobal(o), cname, c)
			w.Truncate(n)
			acds = append(acds, acd)

			w.WriteByte('\t')
			w.WriteString(name)
			w.WriteString(" = &")
			w.WriteString(cname)
			w.WriteString(";\n")

		default:
			assign = true
		}

		if assign {
			// Ordinary assignment gos to the init() function
			cdd.init = true
			w.WriteByte('\t')
			w.WriteString(name)
			w.WriteString(" = ")
			cdd.Expr(w, val, typ)
			w.WriteString(";\n")
		}
		cdd.copyInit(w)
	}
	return
}

func (cdd *CDD) structDecl(w *bytes.Buffer, name string, typ *types.Struct) {
	n := w.Len()

	w.WriteString("struct ")
	w.WriteString(name)
	w.WriteString("_struct;\n")
	cdd.indent(w)
	w.WriteString("typedef struct ")
	w.WriteString(name)
	w.WriteString("_struct ")
	w.WriteString(name)

	cdd.copyDecl(w, ";\n")
	w.Truncate(n)

	tuple := strings.Contains(name, "$$")

	if tuple {
		cdd.indent(w)
		w.WriteString("#ifndef " + name + "$\n")
		cdd.indent(w)
		w.WriteString("#define " + name + "$\n")
	}
	cdd.indent(w)
	w.WriteString("struct ")
	w.WriteString(name)
	w.WriteByte('_')
	cdd.Type(w, typ)
	w.WriteString(";\n")
	if tuple {
		cdd.indent(w)
		w.WriteString("#endif\n")
	}

	cdd.copyDef(w)
	w.Truncate(n)
}

func (cc *GTC) Decl(decl ast.Decl, il int) []*CDD {
	switch d := decl.(type) {
	case *ast.FuncDecl:
		return cc.FuncDecl(d, il)

	case *ast.GenDecl:
		return cc.GenDecl(d, il)
	}

	panic(fmt.Sprint("Unknown declaration: ", decl))
}
<|endoftext|>


<commit_before>package bitband

import (
	"mmio"
	"unsafe"
)

//c:volatile
type Bit struct {
	a *mmio.U32
}

func (b Bit) Load() int {
	return int(b.a.Load())
}

func (b Bit) Store(v int) {
	b.a.Store(uint32(v))
}

func (b Bit) Set() {
	b.Store(1)
}

func (b Bit) Clear() {
	b.Store(0)
}

// 0x20000000 - 0x200FFFFF: SRAM bit-band region.
// 0x22000000 - 0x23FFFFFF: SRAM bit-band alias.
//
// 0x40000000 - 0x400FFFFF: peripheral bit-band region.
// 0x42000000 - 0x43FFFFFF: peripheral bit-band alias.
func bitAlias(addr unsafe.Pointer) unsafe.Pointer {
	a := uintptr(addr)
	base := a &^ 0xfffff
	if base != 0x40000000 && base != 0x20000000 {
		panic("bitband: not in region")
	}
	base += 0x2000000
	offset := a & 0xfffff
	return unsafe.Pointer(base + offset*32)
}

type Bits8 struct {
	a *[8]mmio.U32
}

func (b Bits8) Bit(n int) Bit {
	return Bit{&b.a[n]}
}

func Alias8(r *mmio.U8) Bits8 {
	return Bits8{(*[8]mmio.U32)(bitAlias(unsafe.Pointer(r)))}
}

type Bits16 struct {
	a *[16]mmio.U32
}

func (b Bits16) Bit(n int) Bit {
	return Bit{&b.a[n]}
}

func Alias16(r *mmio.U16) Bits16 {
	return Bits16{(*[16]mmio.U32)(bitAlias(unsafe.Pointer(r)))}
}

type Bits32 struct {
	a *[32]mmio.U32
}

func (b Bits32) Bit(n int) Bit {
	return Bit{&b.a[n]}
}

func Alias32(r *mmio.U32) Bits32 {
	return Bits32{(*[32]mmio.U32)(bitAlias(unsafe.Pointer(r)))}
}
<commit_msg>arch/cortexm/bitband: Remove unnecessary volatile pragma.<commit_after>package bitband

import (
	"mmio"
	"unsafe"
)

type Bit struct {
	a *mmio.U32
}

func (b Bit) Load() int {
	return int(b.a.Load())
}

func (b Bit) Store(v int) {
	b.a.Store(uint32(v))
}

func (b Bit) Set() {
	b.Store(1)
}

func (b Bit) Clear() {
	b.Store(0)
}

// 0x20000000 - 0x200FFFFF: SRAM bit-band region.
// 0x22000000 - 0x23FFFFFF: SRAM bit-band alias.
//
// 0x40000000 - 0x400FFFFF: peripheral bit-band region.
// 0x42000000 - 0x43FFFFFF: peripheral bit-band alias.
func bitAlias(addr unsafe.Pointer) unsafe.Pointer {
	a := uintptr(addr)
	base := a &^ 0xfffff
	if base != 0x40000000 && base != 0x20000000 {
		panic("bitband: not in region")
	}
	base += 0x2000000
	offset := a & 0xfffff
	return unsafe.Pointer(base + offset*32)
}

type Bits8 struct {
	a *[8]mmio.U32
}

func (b Bits8) Bit(n int) Bit {
	return Bit{&b.a[n]}
}

func Alias8(r *mmio.U8) Bits8 {
	return Bits8{(*[8]mmio.U32)(bitAlias(unsafe.Pointer(r)))}
}

type Bits16 struct {
	a *[16]mmio.U32
}

func (b Bits16) Bit(n int) Bit {
	return Bit{&b.a[n]}
}

func Alias16(r *mmio.U16) Bits16 {
	return Bits16{(*[16]mmio.U32)(bitAlias(unsafe.Pointer(r)))}
}

type Bits32 struct {
	a *[32]mmio.U32
}

func (b Bits32) Bit(n int) Bit {
	return Bit{&b.a[n]}
}

func Alias32(r *mmio.U32) Bits32 {
	return Bits32{(*[32]mmio.U32)(bitAlias(unsafe.Pointer(r)))}
}
<|endoftext|>


<commit_before>package gridt

import (
	"testing"
)

const (
	checkMark = "\u2713"
	ballotX   = "\u2717"

	logMsg   = "\n%s: %s"
	fatalMsg = logMsg + "\nwidths = %v\nfit = %v"
)

func end(passed bool, msg string, ws []uint, f bool, t *testing.T) {
	if !passed {
		t.Fatalf(fatalMsg, msg, ballotX, ws, f)
	}
	t.Logf(logMsg, msg, checkMark)
}

func TestFromBidimensional(t *testing.T) {
	var msg string
	var passed, f bool
	var l uint
	var ws []uint

	t.Run("EmptyList", func(t *testing.T) {
		msg = "Should return an empty list of widths that fits"
		ws, l, f = FromBidimensional([]string{}, 10, TopToBottom, " ")
		passed = len(ws) == 0 && l == 0 && f
		end(passed, msg, ws, f, t)
	})

	t.Run("OneItem", func(t *testing.T) {
		t.Run("SufficientSize", func(t *testing.T) {
			msg = "Should return a list with one width that fits"
			ws, l, f = FromBidimensional([]string{"1234567890"}, 20, TopToBottom, " ")
			passed = len(ws) == 1 && l == 1 && f
			end(passed, msg, ws, f, t)
		})
		t.Run("UnsufficientSize", func(t *testing.T) {
			msg = "Should return an empty list that does not fit"
			ws, l, f = FromBidimensional([]string{"1234567890"}, 5, TopToBottom, " ")
			passed = len(ws) == 0 && l == 0 && !f
			end(passed, msg, ws, f, t)
		})
	})

	t.Run("TwoItems", func(t *testing.T) {
		t.Run("SufficientSizeForTwoColumns", func(t *testing.T) {
			msg = "Should return a list with two widths that fits"
			ws, l, f = FromBidimensional([]string{"1234567890", "1234567890"}, 50, TopToBottom, " ")
			passed = len(ws) == 2 && l == 1 && f
			end(passed, msg, ws, f, t)
		})
		t.Run("SufficientSizeForOneColumn", func(t *testing.T) {
			msg = "Should return a list with one width that fits"
			ws, l, f = FromBidimensional([]string{"1234567890", "1234567890"}, 15, TopToBottom, " ")
			passed = len(ws) == 1 && l == 2 && f
			end(passed, msg, ws, f, t)
		})
		t.Run("UnsufficientSizeFor", func(t *testing.T) {
			msg = "Should return an empty list that does not fit"
			ws, l, f = FromBidimensional([]string{"1234567890", "1234567890"}, 5, TopToBottom, " ")
			passed = len(ws) == 0 && l == 0 && !f
			end(passed, msg, ws, f, t)
		})
	})
}
<commit_msg>Details.<commit_after>package gridt

import (
	"testing"
)

const (
	checkMark = "\u2713"
	ballotX   = "\u2717"

	logMsgf   = "\n%s: %s"
	fatalMsgf = logMsgf + "\nwidths = %v\nfit = %v"
)

func end(passed bool, msg string, ws []uint, f bool, t *testing.T) {
	if !passed {
		t.Fatalf(fatalMsgf, msg, ballotX, ws, f)
	}
	t.Logf(logMsgf, msg, checkMark)
}

func TestFromBidimensional(t *testing.T) {
	var msg string
	var passed, f bool
	var l uint
	var ws []uint

	t.Run("EmptyList", func(t *testing.T) {
		msg = "Should return an empty list of widths that fits"
		ws, l, f = FromBidimensional([]string{}, 10, TopToBottom, " ")
		passed = len(ws) == 0 && l == 0 && f
		end(passed, msg, ws, f, t)
	})

	t.Run("OneItem", func(t *testing.T) {
		t.Run("SufficientSize", func(t *testing.T) {
			msg = "Should return a list with one width that fits"
			ws, l, f = FromBidimensional([]string{"1234567890"}, 20, TopToBottom, " ")
			passed = len(ws) == 1 && l == 1 && f
			end(passed, msg, ws, f, t)
		})
		t.Run("UnsufficientSize", func(t *testing.T) {
			msg = "Should return an empty list that does not fit"
			ws, l, f = FromBidimensional([]string{"1234567890"}, 5, TopToBottom, " ")
			passed = len(ws) == 0 && l == 0 && !f
			end(passed, msg, ws, f, t)
		})
	})

	t.Run("TwoItems", func(t *testing.T) {
		t.Run("SufficientSizeForTwoColumns", func(t *testing.T) {
			msg = "Should return a list with two widths that fits"
			ws, l, f = FromBidimensional([]string{"1234567890", "1234567890"}, 50, TopToBottom, " ")
			passed = len(ws) == 2 && l == 1 && f
			end(passed, msg, ws, f, t)
		})
		t.Run("SufficientSizeForOneColumn", func(t *testing.T) {
			msg = "Should return a list with one width that fits"
			ws, l, f = FromBidimensional([]string{"1234567890", "1234567890"}, 15, TopToBottom, " ")
			passed = len(ws) == 1 && l == 2 && f
			end(passed, msg, ws, f, t)
		})
		t.Run("UnsufficientSizeFor", func(t *testing.T) {
			msg = "Should return an empty list that does not fit"
			ws, l, f = FromBidimensional([]string{"1234567890", "1234567890"}, 5, TopToBottom, " ")
			passed = len(ws) == 0 && l == 0 && !f
			end(passed, msg, ws, f, t)
		})
	})
}
<|endoftext|>


<commit_before>package scanner

import (
	"bufio"
	"io"

	"github.com/brettlangdon/gython/errorcode"
	"github.com/brettlangdon/gython/token"
)

var EOF rune = 0
var MAXINDENT int = 100

type Scanner struct {
	state           errorcode.ErrorCode
	reader          *bufio.Reader
	currentPosition *Position
	positionBuffer  []*Position

	currentLine   int
	currentColumn int
}

func NewScanner(r io.Reader) *Scanner {
	return &Scanner{
		state:          errorcode.E_OK,
		reader:         bufio.NewReader(r),
		positionBuffer: make([]*Position, 0),
		currentLine:    1,
		currentColumn:  0,
	}
}

func (scanner *Scanner) nextPosition() *Position {
	if len(scanner.positionBuffer) > 0 {
		last := len(scanner.positionBuffer) - 1
		scanner.currentPosition = scanner.positionBuffer[last]
		scanner.positionBuffer = scanner.positionBuffer[0:last]
		return scanner.currentPosition
	}

	next, _, err := scanner.reader.ReadRune()
	if err != nil {
		scanner.state = errorcode.E_EOF
		next = EOF
		scanner.currentLine++
		scanner.currentColumn = 0
	}

	pos := &Position{
		Char:   next,
		Line:   scanner.currentLine,
		Column: scanner.currentColumn,
	}

	scanner.currentColumn++
	if next == '\n' || next == EOF {
		scanner.currentLine++
		scanner.currentColumn = 0
	}

	return pos
}

func (scanner *Scanner) unreadPosition(pos *Position) {
	scanner.positionBuffer = append(scanner.positionBuffer, pos)
}

func (scanner *Scanner) parseQuoted(positions *Positions, quote rune) *token.Token {
	// Determine quote size, 1 or 3 (e.g. 'string',  '''string''')
	quoteSize := 1
	endQuoteSize := 0
	pos := scanner.nextPosition()
	if pos.Char == quote {
		pos2 := scanner.nextPosition()
		if pos2.Char == quote {
			positions.Append(pos)
			positions.Append(pos2)
			quoteSize = 3
		} else {
			scanner.unreadPosition(pos2)
			endQuoteSize = 1
		}
	} else {
		scanner.unreadPosition(pos)
	}

	for {
		if endQuoteSize == quoteSize {
			break
		}
		pos = scanner.nextPosition()
		positions.Append(pos)
		if pos.Char == EOF {
			return positions.AsToken(token.ERRORTOKEN)
		}
		if quoteSize == 1 && pos.Char == '\n' {
			return positions.AsToken(token.ERRORTOKEN)
		}
		if pos.Char == quote {
			endQuoteSize += 1
		} else {
			endQuoteSize = 0
			if pos.Char == '\\' {
				pos = scanner.nextPosition()
			}
		}
	}
	return positions.AsToken(token.STRING)
}

func (scanner *Scanner) NextToken() *token.Token {
	positions := NewPositions()

	pos := scanner.nextPosition()
	// skip spaces
	for {
		if pos.Char != ' ' && pos.Char != '\t' {
			break
		}
		pos = scanner.nextPosition()
	}

	// skip comments
	if pos.Char == '#' {
		for {
			pos = scanner.nextPosition()
			if pos.Char == EOF || pos.Char == '\n' {
				break
			}
		}
	}

	positions.Append(pos)
	switch ch := pos.Char; {
	case ch == EOF:
		id := token.ENDMARKER
		if scanner.state != errorcode.E_EOF {
			id = token.ERRORTOKEN
		}
		return positions.AsToken(id)
	case IsIdentifierStart(ch):
		// Parse Identifier
		saw_b, saw_r, saw_u := false, false, false
		for {
			if !(saw_b || saw_u) && (ch == 'b' || ch == 'B') {
				saw_b = true
			} else if !(saw_b || saw_u || saw_r) && (ch == 'u' || ch == 'U') {
				saw_u = true
			} else if !(saw_r || saw_u) && (ch == 'r' || ch == 'R') {
				saw_r = true
			} else {
				break
			}
			pos = scanner.nextPosition()
			if IsQuote(pos.Char) {
				positions.Append(pos)
				return scanner.parseQuoted(positions, pos.Char)
			}
		}
		pos = scanner.nextPosition()
		for IsIdentifierChar(pos.Char) {
			positions.Append(pos)
			pos = scanner.nextPosition()
		}
		scanner.unreadPosition(pos)
		return positions.AsToken(token.NAME)
	case ch == '\n':
		return positions.AsToken(token.NEWLINE)
	case ch == '.':
		pos2 := scanner.nextPosition()
		if IsDigit(pos2.Char) {
			// Parse Number
		} else if pos2.Char == '.' {
			positions.Append(pos2)
			pos3 := scanner.nextPosition()
			if pos3.Char == '.' {
				positions.Append(pos3)
				return positions.AsToken(token.ELLIPSIS)
			}
			scanner.unreadPosition(pos3)
		}
		scanner.unreadPosition(pos2)

		return positions.AsToken(token.DOT)
	case IsDigit(ch):
		// Parse Number
	case IsQuote(ch):
		// Parse String
		return scanner.parseQuoted(positions, ch)
	case ch == '\\':
		// Parse Continuation
	default:
		// Two and Three character operators
		pos2 := scanner.nextPosition()
		op2Id := GetTwoCharTokenID(pos.Char, pos2.Char)
		if op2Id != token.OP {
			positions.Append(pos2)
			pos3 := scanner.nextPosition()
			op3Id := GetThreeCharTokenID(pos.Char, pos2.Char, pos3.Char)
			if op3Id != token.OP {
				positions.Append(pos3)
				return positions.AsToken(op3Id)
			}
			scanner.unreadPosition(pos3)
			return positions.AsToken(op2Id)
		}
		scanner.unreadPosition(pos2)
	}
	switch pos.Char {
	case '(', '[', '{':
		// Increment indentation level
		// scanner.indentationLevel++
		break
	case ')', ']', '}':
		// Decrement indentation level
		// scanner.indentationLevel--
		break
	}

	opId := GetOneCharTokenID(pos.Char)
	return positions.AsToken(opId)
}
<commit_msg>shuffle line/column incrementing around<commit_after>package scanner

import (
	"bufio"
	"io"

	"github.com/brettlangdon/gython/errorcode"
	"github.com/brettlangdon/gython/token"
)

var EOF rune = 0
var MAXINDENT int = 100

type Scanner struct {
	state           errorcode.ErrorCode
	reader          *bufio.Reader
	currentPosition *Position
	positionBuffer  []*Position

	currentLine   int
	currentColumn int
}

func NewScanner(r io.Reader) *Scanner {
	return &Scanner{
		state:          errorcode.E_OK,
		reader:         bufio.NewReader(r),
		positionBuffer: make([]*Position, 0),
		currentLine:    1,
		currentColumn:  0,
	}
}

func (scanner *Scanner) nextPosition() *Position {
	if len(scanner.positionBuffer) > 0 {
		last := len(scanner.positionBuffer) - 1
		scanner.currentPosition = scanner.positionBuffer[last]
		scanner.positionBuffer = scanner.positionBuffer[0:last]
		return scanner.currentPosition
	}

	next, _, err := scanner.reader.ReadRune()
	if err != nil {
		scanner.state = errorcode.E_EOF
		next = EOF
		scanner.currentLine++
		scanner.currentColumn = 0
	}

	if next == '\n' || next == EOF {
		scanner.currentLine++
		scanner.currentColumn = 0
	}

	pos := &Position{
		Char:   next,
		Line:   scanner.currentLine,
		Column: scanner.currentColumn,
	}
	scanner.currentColumn++
	return pos
}

func (scanner *Scanner) unreadPosition(pos *Position) {
	scanner.positionBuffer = append(scanner.positionBuffer, pos)
}

func (scanner *Scanner) parseQuoted(positions *Positions, quote rune) *token.Token {
	// Determine quote size, 1 or 3 (e.g. 'string',  '''string''')
	quoteSize := 1
	endQuoteSize := 0
	pos := scanner.nextPosition()
	if pos.Char == quote {
		pos2 := scanner.nextPosition()
		if pos2.Char == quote {
			positions.Append(pos)
			positions.Append(pos2)
			quoteSize = 3
		} else {
			scanner.unreadPosition(pos2)
			endQuoteSize = 1
		}
	} else {
		scanner.unreadPosition(pos)
	}

	for {
		if endQuoteSize == quoteSize {
			break
		}
		pos = scanner.nextPosition()
		positions.Append(pos)
		if pos.Char == EOF {
			return positions.AsToken(token.ERRORTOKEN)
		}
		if quoteSize == 1 && pos.Char == '\n' {
			return positions.AsToken(token.ERRORTOKEN)
		}
		if pos.Char == quote {
			endQuoteSize += 1
		} else {
			endQuoteSize = 0
			if pos.Char == '\\' {
				pos = scanner.nextPosition()
			}
		}
	}
	return positions.AsToken(token.STRING)
}

func (scanner *Scanner) NextToken() *token.Token {
	positions := NewPositions()

	pos := scanner.nextPosition()
	// skip spaces
	for {
		if pos.Char != ' ' && pos.Char != '\t' {
			break
		}
		pos = scanner.nextPosition()
	}

	// skip comments
	if pos.Char == '#' {
		for {
			pos = scanner.nextPosition()
			if pos.Char == EOF || pos.Char == '\n' {
				break
			}
		}
	}

	positions.Append(pos)
	switch ch := pos.Char; {
	case ch == EOF:
		id := token.ENDMARKER
		if scanner.state != errorcode.E_EOF {
			id = token.ERRORTOKEN
		}
		return positions.AsToken(id)
	case IsIdentifierStart(ch):
		// Parse Identifier
		saw_b, saw_r, saw_u := false, false, false
		for {
			if !(saw_b || saw_u) && (ch == 'b' || ch == 'B') {
				saw_b = true
			} else if !(saw_b || saw_u || saw_r) && (ch == 'u' || ch == 'U') {
				saw_u = true
			} else if !(saw_r || saw_u) && (ch == 'r' || ch == 'R') {
				saw_r = true
			} else {
				break
			}
			pos = scanner.nextPosition()
			if IsQuote(pos.Char) {
				positions.Append(pos)
				return scanner.parseQuoted(positions, pos.Char)
			}
		}
		pos = scanner.nextPosition()
		for IsIdentifierChar(pos.Char) {
			positions.Append(pos)
			pos = scanner.nextPosition()
		}
		scanner.unreadPosition(pos)
		return positions.AsToken(token.NAME)
	case ch == '\n':
		return positions.AsToken(token.NEWLINE)
	case ch == '.':
		pos2 := scanner.nextPosition()
		if IsDigit(pos2.Char) {
			// Parse Number
		} else if pos2.Char == '.' {
			positions.Append(pos2)
			pos3 := scanner.nextPosition()
			if pos3.Char == '.' {
				positions.Append(pos3)
				return positions.AsToken(token.ELLIPSIS)
			}
			scanner.unreadPosition(pos3)
		}
		scanner.unreadPosition(pos2)

		return positions.AsToken(token.DOT)
	case IsDigit(ch):
		// Parse Number
	case IsQuote(ch):
		// Parse String
		return scanner.parseQuoted(positions, ch)
	case ch == '\\':
		// Parse Continuation
	default:
		// Two and Three character operators
		pos2 := scanner.nextPosition()
		op2Id := GetTwoCharTokenID(pos.Char, pos2.Char)
		if op2Id != token.OP {
			positions.Append(pos2)
			pos3 := scanner.nextPosition()
			op3Id := GetThreeCharTokenID(pos.Char, pos2.Char, pos3.Char)
			if op3Id != token.OP {
				positions.Append(pos3)
				return positions.AsToken(op3Id)
			}
			scanner.unreadPosition(pos3)
			return positions.AsToken(op2Id)
		}
		scanner.unreadPosition(pos2)
	}
	switch pos.Char {
	case '(', '[', '{':
		// Increment indentation level
		// scanner.indentationLevel++
		break
	case ')', ']', '}':
		// Decrement indentation level
		// scanner.indentationLevel--
		break
	}

	opId := GetOneCharTokenID(pos.Char)
	return positions.AsToken(opId)
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/xml"
	"flag"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"io"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
)

var dbFile = flag.String("db", "xrguide.db", "Database file.")
var textDir = flag.String("t", ".", "Directory with text files.")
var verbose = flag.Bool("v", true, "Verbose output.")

var insert string = `
INSERT INTO text_entries
(language_id, page_id, text_id, text)
VALUES
(?, ?, ?, ?)
`

func main() {
	flag.Parse()
	err := backupDb(*dbFile)
	if err != nil {
		log.Fatal(err)
	}
	db, err := sql.Open("sqlite3", *dbFile)
	if err != nil {
		log.Fatalf("Error opening db: %v", err)
	}
	err = prepareDb(db)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()
	err = read(db, *textDir, *verbose)
	if err != nil {
		log.Fatal(err)
	}
}

type Text struct {
	Id    int64  `xml:"id,attr"`
	Entry string `xml:",innerxml"`
}

type Page struct {
	Id      int64  `xml:"id,attr"`
	Entries []Text `xml:"t"`
}

type LangFile struct {
	XMLName xml.Name `xml:"language"`
	LangId  int64    `xml:"id,attr"`
	Pages   []Page   `xml:"page"`
}

func read(db *sql.DB, directory string, verbose bool) error {
	info, err := os.Stat(directory)
	if err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("Could not stat text directory: %v", err)
	}
	if os.IsNotExist(err) {
		return fmt.Errorf("Text directory not found: %s", directory)
	}
	if !info.IsDir() {
		return fmt.Errorf("%s is not a directory.", directory)
	}
	dir, err := os.Open(directory)
	if err != nil {
		return fmt.Errorf("Error opening text directory: %v", err)
	}
	defer dir.Close()
	pattern := regexp.MustCompile("0001-L(\\d{3})\\.xml")
	stmt, err := db.Prepare(insert)
	if err != nil {
		return fmt.Errorf("Error preparing statement: %v", err)
	}
	var langId int64
	var fileName string
	var lang LangFile
	for {
		f, err := dir.Readdir(1)
		if err == io.EOF {
			break
		}
		if err != nil {
			return fmt.Errorf("Error on reading text directory: %v")
		}
		fileName = filepath.Join(dir.Name(), f[0].Name())
		if !pattern.MatchString(f[0].Name()) {
			log.Printf("Skipping %s", fileName)
			continue
		}
		matches := pattern.FindStringSubmatch(f[0].Name())
		langId, err = strconv.ParseInt(matches[1], 10, 64)
		if err != nil {
			log.Printf("Error on parsing filename: %s (%v)", fileName, err)
			continue
		}
		if langId == 0 {
			log.Printf("Error on filename %s. Invalid lang id.", fileName)
			continue
		}
		log.Printf("Text file %s. Language Id %d", fileName, langId)

		file, err := os.Open(fileName)
		if err != nil {
			return fmt.Errorf("Error opening file %s.", fileName)
		}
		decoder := xml.NewDecoder(file)
		err = decoder.Decode(&lang)
		if err != nil {
			file.Close()
			return fmt.Errorf("Error decoding file %s: %v", fileName, err)
		}
		for _, page := range lang.Pages {
			if lang.LangId != langId {
				fmt.Printf("Language Id does not match in %s. Id %d.", fileName, lang.LangId)
				break
			}
			if verbose {
				log.Printf("Lang %d Page %d.", lang.LangId, page.Id)
			}
			for _, t := range page.Entries {
				if verbose {
					log.Printf("Lang %d Page %d Text %d.", lang.LangId, page.Id, t.Id)
				}
				_, err = stmt.Exec(lang.LangId, page.Id, t.Id, t.Entry)
				if err != nil {
					return fmt.Errorf("Error on insert. Aborting: %v", err)
				}
			}
		}
		file.Close()
	}
	return nil
}

func backupDb(fileName string) error {
	info, err := os.Stat(fileName)
	if err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("Error backing up db stat: %v", err)
	}
	if os.IsNotExist(err) {
		return nil
	}
	if info.IsDir() {
		return fmt.Errorf("DB file is a directory. Cannot continue.")
	}
	orig, err := os.Open(fileName)
	if err != nil {
		return fmt.Errorf("Could not open db: %v", err)
	}
	defer orig.Close()
	bak, err := os.Create(fileName + ".bak")
	if err != nil {
		return fmt.Errorf("Error creating backup file: %v", err)
	}
	defer bak.Close()
	_, err = io.Copy(bak, orig)
	return nil
}

func prepareDb(db *sql.DB) error {
	var err error
	stmts := []string{
		`
DROP TABLE IF EXISTS languages;
		`,
		`
CREATE TABLE languages (
	id INTEGER PRIMARY KEY ASC,
	name TEXT UNIQUE
)
		`,
		`
DROP TABLE IF EXISTS text_entries;
		`,
		`
CREATE TABLE text_entries (
	language_id INTEGER,
	page_id INTEGER,
	text_id INTEGER,
	text TEXT,
	PRIMARY KEY (language_id, page_id, text_id ASC),
	FOREIGN KEY (language_id) REFERENCES languages(id) ON DELETE RESTRICT ON UPDATE CASCADE
)
		`,
		`
INSERT INTO languages
(id, name)
VALUES
(7, 'Russian'),
(33, 'French'),
(34, 'Spanish'),
(39, 'Italian'),
(44, 'English'),
(49, 'German'),
(86, 'Chinese (traditional)'),
(88, 'Chinese (simplified)')
		`,
	}
	for _, sql := range stmts {
		_, err = db.Exec(sql)
		if err != nil {
			return fmt.Errorf("Error preparing db: %v", err)
		}
	}
	return nil
}
<commit_msg>support selective reset of db, selective languages and pages. minor internal fixes<commit_after>package main

import (
	"database/sql"
	"encoding/xml"
	"flag"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"io"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
)

var dbFile = flag.String("db", "xrguide.db", "Database file.")
var rebuild = flag.Bool("r", false, "Whether to reinitialize db.")
var textDir = flag.String("t", ".", "Directory with text files.")
var verbose = flag.Bool("v", true, "Verbose output.")
var lang = flag.Int64("l", 0, "Language Id. If not specified all.")
var page = flag.Int64("p", 0, "Page Id. If not specified all.")

var reset string = `
DELETE FROM text_entries
WHERE
language_id = ?
AND
page_id = ?
`
var insert string = `
INSERT INTO text_entries
(language_id, page_id, text_id, text)
VALUES
(?, ?, ?, ?)
`

func main() {
	flag.Parse()
	err := backupDb(*dbFile)
	if err != nil {
		log.Fatal(err)
	}
	db, err := sql.Open("sqlite3", *dbFile)
	if err != nil {
		log.Fatalf("Error opening db: %v", err)
	}
	defer db.Close()
	if *rebuild {
		err = prepareDb(db)
		if err != nil {
			log.Fatal(err)
		}
	}
	err = read(db, *textDir, *verbose, *lang, *page)
	if err != nil {
		log.Fatal(err)
	}
}

type Text struct {
	Id    int64  `xml:"id,attr"`
	Entry string `xml:",innerxml"`
}

type Page struct {
	Id      int64  `xml:"id,attr"`
	Entries []Text `xml:"t"`
}

type LangFile struct {
	XMLName xml.Name `xml:"language"`
	LangId  int64    `xml:"id,attr"`
	Pages   []Page   `xml:"page"`
}

func read(db *sql.DB, directory string, verbose bool, useLang, usePage int64) error {
	info, err := os.Stat(directory)
	if err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("Could not stat text directory: %v", err)
	}
	if os.IsNotExist(err) {
		return fmt.Errorf("Text directory not found: %s", directory)
	}
	if !info.IsDir() {
		return fmt.Errorf("%s is not a directory.", directory)
	}
	dir, err := os.Open(directory)
	if err != nil {
		return fmt.Errorf("Error opening text directory: %v", err)
	}
	defer dir.Close()
	pattern := regexp.MustCompile("0001-L(\\d{3})\\.xml")
	stmt, err := db.Prepare(insert)
	if err != nil {
		return fmt.Errorf("Error preparing statement: %v", err)
	}
	reset, err := db.Prepare(reset)
	if err != nil {
		return fmt.Errorf("Error preparing statement: %v", err)
	}
	var langId int64
	var fileName string
	var lang LangFile
	for {
		f, err := dir.Readdir(1)
		if err == io.EOF {
			break
		}
		if err != nil {
			return fmt.Errorf("Error on reading text directory: %v")
		}
		fileName = filepath.Join(dir.Name(), f[0].Name())
		if !pattern.MatchString(f[0].Name()) {
			log.Printf("Skipping %s", fileName)
			continue
		}
		matches := pattern.FindStringSubmatch(f[0].Name())
		langId, err = strconv.ParseInt(matches[1], 10, 64)
		if err != nil {
			log.Printf("Error on parsing filename: %s (%v)", fileName, err)
			continue
		}
		if langId == 0 {
			log.Printf("Error on filename %s. Invalid lang id.", fileName)
			continue
		}
		log.Printf("Text file %s. Language Id %d", fileName, langId)

		if useLang != 0 && useLang != langId {
			log.Printf("Skipping %s.", fileName)
			continue
		}

		file, err := os.Open(fileName)
		if err != nil {
			return fmt.Errorf("Error opening file %s.", fileName)
		}
		decoder := xml.NewDecoder(file)
		err = decoder.Decode(&lang)
		if err != nil {
			file.Close()
			return fmt.Errorf("Error decoding file %s: %v", fileName, err)
		}
		for _, page := range lang.Pages {
			if lang.LangId != langId {
				log.Printf("Language Id does not match in %s. Id %d.", fileName, lang.LangId)
				break
			}
			if usePage != 0 && usePage != page.Id {
				if verbose {
					log.Printf("Skipping page %d.", page.Id)
				}
				continue
			}
			if verbose {
				log.Printf("Lang %d Page %d.", lang.LangId, page.Id)
			}
			_, err = reset.Exec(lang.LangId, page.Id)
			if err != nil {
				return fmt.Errorf("Error on reset. Aborting: %v", err)
			}
			for _, t := range page.Entries {
				if verbose {
					log.Printf("Lang %d Page %d Text %d.", lang.LangId, page.Id, t.Id)
				}
				_, err = stmt.Exec(lang.LangId, page.Id, t.Id, t.Entry)
				if err != nil {
					return fmt.Errorf("Error on insert. Aborting: %v", err)
				}
			}
		}
		file.Close()
	}
	return nil
}

func backupDb(fileName string) error {
	info, err := os.Stat(fileName)
	if err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("Error backing up db stat: %v", err)
	}
	if os.IsNotExist(err) {
		return nil
	}
	if info.IsDir() {
		return fmt.Errorf("DB file is a directory. Cannot continue.")
	}
	orig, err := os.Open(fileName)
	if err != nil {
		return fmt.Errorf("Could not open db: %v", err)
	}
	defer orig.Close()
	bak, err := os.Create(fileName + ".bak")
	if err != nil {
		return fmt.Errorf("Error creating backup file: %v", err)
	}
	defer bak.Close()
	_, err = io.Copy(bak, orig)
	return nil
}

func prepareDb(db *sql.DB) error {
	var err error
	stmts := []string{
		`
DROP TABLE IF EXISTS languages;
		`,
		`
CREATE TABLE languages (
	id INTEGER PRIMARY KEY ASC,
	name TEXT UNIQUE
)
		`,
		`
DROP TABLE IF EXISTS text_entries;
		`,
		`
CREATE TABLE text_entries (
	language_id INTEGER,
	page_id INTEGER,
	text_id INTEGER,
	text TEXT,
	PRIMARY KEY (language_id, page_id, text_id ASC),
	FOREIGN KEY (language_id) REFERENCES languages(id) ON DELETE RESTRICT ON UPDATE CASCADE
)
		`,
		`
INSERT INTO languages
(id, name)
VALUES
(7, 'Russian'),
(33, 'French'),
(34, 'Spanish'),
(39, 'Italian'),
(44, 'English'),
(49, 'German'),
(86, 'Chinese (traditional)'),
(88, 'Chinese (simplified)')
		`,
	}
	for _, sql := range stmts {
		_, err = db.Exec(sql)
		if err != nil {
			return fmt.Errorf("Error preparing db: %v", err)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2012 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"errors"
	"github.com/fsouza/gogit/git"
	"io"
	. "launchpad.net/gocheck"
	"os"
	"path"
	"syscall"
)

func writeConfig(sourceFile string, c *C) string {
	srcConfig, err := os.Open(sourceFile)
	c.Assert(err, IsNil)
	defer srcConfig.Close()
	p := path.Join(os.TempDir(), "guesser-tests")
	err = os.MkdirAll(p, 0700)
	c.Assert(err, IsNil)
	repo, err := git.InitRepository(p, false)
	c.Assert(err, IsNil)
	defer repo.Free()
	dstConfig, err := os.OpenFile(path.Join(p, ".git", "config"), syscall.O_WRONLY|syscall.O_TRUNC|syscall.O_CREAT|syscall.O_CLOEXEC, 0644)
	c.Assert(err, IsNil)
	defer dstConfig.Close()
	_, err = io.Copy(dstConfig, srcConfig)
	c.Assert(err, IsNil)
	return p
}

func (s *S) TestGitGuesser(c *C) {
	p := writeConfig("testdata/gitconfig-ok", c)
	defer os.RemoveAll(p)
	dirPath := path.Join(p, "somepath")
	err := os.MkdirAll(dirPath, 0700) // Will be removed when p is removed.
	c.Assert(err, IsNil)
	g := GitGuesser{}
	name, err := g.GuessName(p) // repository root
	c.Assert(err, IsNil)
	c.Assert(name, Equals, "gopher")
	name, err = g.GuessName(dirPath) // subdirectory
	c.Assert(err, IsNil)
	c.Assert(name, Equals, "gopher")
}

// This test may fail if you have a git repository in /tmp. By the way, if you
// do have a repository in the temporary file hierarchy, please kill yourself.
func (s *S) TestGitGuesserWhenTheDirectoryIsNotAGitRepository(c *C) {
	p := path.Join(os.TempDir(), "guesser-tests")
	err := os.MkdirAll(p, 0700)
	c.Assert(err, IsNil)
	defer os.RemoveAll(p)
	name, err := GitGuesser{}.GuessName(p)
	c.Assert(name, Equals, "")
	c.Assert(err, NotNil)
	c.Assert(err, ErrorMatches, "^Git repository not found:.*")
}

func (s *S) TestGitGuesserWithoutTsuruRemote(c *C) {
	p := writeConfig("testdata/gitconfig-without-tsuru-remote", c)
	defer os.RemoveAll(p)
	name, err := GitGuesser{}.GuessName(p)
	c.Assert(name, Equals, "")
	c.Assert(err, NotNil)
	c.Assert(err.Error(), Equals, "tsuru remote not declared.")
}

func (s *S) TestGitGuesserWithTsuruRemoteNotMatchingTsuruPattern(c *C) {
	p := writeConfig("testdata/gitconfig-not-matching", c)
	defer os.RemoveAll(p)
	name, err := GitGuesser{}.GuessName(p)
	c.Assert(name, Equals, "")
	c.Assert(err, NotNil)
	c.Assert(err.Error(), Equals, `"tsuru" remote did not match the pattern. Want something like git@<host>:<app-name>.git, got me@myhost.com:gopher.git`)
}

func (s *S) TestGuessingCommandGuesserNil(c *C) {
	g := GuessingCommand{g: nil}
	c.Assert(g.guesser(), FitsTypeOf, GitGuesser{})
}

func (s *S) TestGuessingCommandGuesserNonNil(c *C) {
	fake := &FakeGuesser{}
	g := GuessingCommand{g: fake}
	c.Assert(g.guesser(), DeepEquals, fake)
}

type FakeGuesser struct {
	name string
}

func (f *FakeGuesser) GuessName(path string) (string, error) {
	return f.name, nil
}

type FailingFakeGuesser struct {
	message string
}

func (f *FailingFakeGuesser) GuessName(path string) (string, error) {
	return "", errors.New(f.message)
}
<commit_msg>cmd/tsuru: logging guesses in fake guessers<commit_after>// Copyright 2012 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"errors"
	"github.com/fsouza/gogit/git"
	"io"
	. "launchpad.net/gocheck"
	"os"
	"path"
	"syscall"
)

func writeConfig(sourceFile string, c *C) string {
	srcConfig, err := os.Open(sourceFile)
	c.Assert(err, IsNil)
	defer srcConfig.Close()
	p := path.Join(os.TempDir(), "guesser-tests")
	err = os.MkdirAll(p, 0700)
	c.Assert(err, IsNil)
	repo, err := git.InitRepository(p, false)
	c.Assert(err, IsNil)
	defer repo.Free()
	dstConfig, err := os.OpenFile(path.Join(p, ".git", "config"), syscall.O_WRONLY|syscall.O_TRUNC|syscall.O_CREAT|syscall.O_CLOEXEC, 0644)
	c.Assert(err, IsNil)
	defer dstConfig.Close()
	_, err = io.Copy(dstConfig, srcConfig)
	c.Assert(err, IsNil)
	return p
}

func (s *S) TestGitGuesser(c *C) {
	p := writeConfig("testdata/gitconfig-ok", c)
	defer os.RemoveAll(p)
	dirPath := path.Join(p, "somepath")
	err := os.MkdirAll(dirPath, 0700) // Will be removed when p is removed.
	c.Assert(err, IsNil)
	g := GitGuesser{}
	name, err := g.GuessName(p) // repository root
	c.Assert(err, IsNil)
	c.Assert(name, Equals, "gopher")
	name, err = g.GuessName(dirPath) // subdirectory
	c.Assert(err, IsNil)
	c.Assert(name, Equals, "gopher")
}

// This test may fail if you have a git repository in /tmp. By the way, if you
// do have a repository in the temporary file hierarchy, please kill yourself.
func (s *S) TestGitGuesserWhenTheDirectoryIsNotAGitRepository(c *C) {
	p := path.Join(os.TempDir(), "guesser-tests")
	err := os.MkdirAll(p, 0700)
	c.Assert(err, IsNil)
	defer os.RemoveAll(p)
	name, err := GitGuesser{}.GuessName(p)
	c.Assert(name, Equals, "")
	c.Assert(err, NotNil)
	c.Assert(err, ErrorMatches, "^Git repository not found:.*")
}

func (s *S) TestGitGuesserWithoutTsuruRemote(c *C) {
	p := writeConfig("testdata/gitconfig-without-tsuru-remote", c)
	defer os.RemoveAll(p)
	name, err := GitGuesser{}.GuessName(p)
	c.Assert(name, Equals, "")
	c.Assert(err, NotNil)
	c.Assert(err.Error(), Equals, "tsuru remote not declared.")
}

func (s *S) TestGitGuesserWithTsuruRemoteNotMatchingTsuruPattern(c *C) {
	p := writeConfig("testdata/gitconfig-not-matching", c)
	defer os.RemoveAll(p)
	name, err := GitGuesser{}.GuessName(p)
	c.Assert(name, Equals, "")
	c.Assert(err, NotNil)
	c.Assert(err.Error(), Equals, `"tsuru" remote did not match the pattern. Want something like git@<host>:<app-name>.git, got me@myhost.com:gopher.git`)
}

func (s *S) TestGuessingCommandGuesserNil(c *C) {
	g := GuessingCommand{g: nil}
	c.Assert(g.guesser(), FitsTypeOf, GitGuesser{})
}

func (s *S) TestGuessingCommandGuesserNonNil(c *C) {
	fake := &FakeGuesser{}
	g := GuessingCommand{g: fake}
	c.Assert(g.guesser(), DeepEquals, fake)
}

type FakeGuesser struct {
	log  []string
	name string
}

func (f *FakeGuesser) GuessName(path string) (string, error) {
	f.log = append(f.log, "Guessing "+path)
	return f.name, nil
}

type FailingFakeGuesser struct {
	log     []string
	message string
}

func (f *FailingFakeGuesser) GuessName(path string) (string, error) {
	f.log = append(f.log, "Guessing "+path)
	return "", errors.New(f.message)
}
<|endoftext|>


<commit_before>package torrent

import (
	"bytes"
	"compress/gzip"
	"errors"
	"io"
	"net/http"
	"time"

	"github.com/boltdb/bolt"
	"github.com/cenkalti/backoff"
)

func (s *Session) startBlocklistReloader() error {
	if s.config.BlocklistURL == "" {
		return nil
	}
	blocklistTimestamp, err := s.getBlocklistTimestamp()
	if err != nil {
		return err
	}

	s.mBlocklist.Lock()
	s.blocklistTimestamp = blocklistTimestamp
	s.mBlocklist.Unlock()

	deadline := blocklistTimestamp.Add(s.config.BlocklistUpdateInterval)
	now := time.Now().UTC()
	delta := now.Sub(deadline)
	if blocklistTimestamp.IsZero() {
		s.log.Infof("Blocklist is empty. Loading blacklist...")
		s.retryReloadBlocklist()
	} else if delta > 0 {
		s.log.Infof("Last blocklist reload was %s ago. Reloading blacklist...", delta.String())
		s.retryReloadBlocklist()
	}
	go s.blocklistReloader(delta)
	return nil
}

func (s *Session) getBlocklistTimestamp() (time.Time, error) {
	var t time.Time
	err := s.db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(sessionBucket)
		val := b.Get(blocklistTimestampKey)
		if val == nil {
			return nil
		}
		var err2 error
		t, err2 = time.ParseInLocation(time.RFC3339, string(val), time.UTC)
		return err2
	})
	return t, err
}

func (s *Session) retryReloadBlocklist() {
	bo := backoff.NewExponentialBackOff()
	bo.MaxElapsedTime = 0

	ticker := backoff.NewTicker(bo)
	defer ticker.Stop()

	for {
		select {
		case <-ticker.C:
			err := s.reloadBlocklist()
			if err != nil {
				s.log.Errorln("cannot load blocklist:", err.Error())
				continue
			}
			return
		case <-s.closeC:
			return
		}
	}
}

func (s *Session) reloadBlocklist() error {
	resp, err := http.Get(s.config.BlocklistURL)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return errors.New("invalid blocklist status code")
	}

	var r io.Reader = resp.Body
	if resp.Header.Get("content-type") == "application/x-gzip" {
		gr, gerr := gzip.NewReader(r)
		if gerr != nil {
			return gerr
		}
		defer gr.Close()
		r = gr
	}

	buf := bytes.NewBuffer(make([]byte, 0, resp.ContentLength))
	r = io.TeeReader(r, buf)

	n, err := s.blocklist.Reload(r)
	if err != nil {
		return err
	}
	s.log.Infof("Loaded %d rules from blocklist.", n)

	now := time.Now()

	s.mBlocklist.Lock()
	s.blocklistTimestamp = now
	s.mBlocklist.Unlock()

	return s.db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket(sessionBucket)
		err2 := b.Put(blocklistKey, buf.Bytes())
		if err2 != nil {
			return err2
		}
		return b.Put(blocklistTimestampKey, []byte(now.UTC().Format(time.RFC3339)))
	})
}

func (s *Session) blocklistReloader(d time.Duration) {
	for {
		select {
		case <-time.After(d):
		case <-s.closeC:
			return
		}

		s.retryReloadBlocklist()
		d = s.config.BlocklistUpdateInterval
	}
}
<commit_msg>more log in blocklist reload<commit_after>package torrent

import (
	"bytes"
	"compress/gzip"
	"errors"
	"io"
	"net/http"
	"time"

	"github.com/boltdb/bolt"
	"github.com/cenkalti/backoff"
)

func (s *Session) startBlocklistReloader() error {
	if s.config.BlocklistURL == "" {
		return nil
	}
	blocklistTimestamp, err := s.getBlocklistTimestamp()
	if err != nil {
		return err
	}

	s.mBlocklist.Lock()
	s.blocklistTimestamp = blocklistTimestamp
	s.mBlocklist.Unlock()

	deadline := blocklistTimestamp.Add(s.config.BlocklistUpdateInterval)
	now := time.Now().UTC()
	delta := now.Sub(deadline)
	if blocklistTimestamp.IsZero() {
		s.log.Infof("Blocklist is empty. Loading blocklist...")
		s.retryReloadBlocklist()
	} else if delta > 0 {
		s.log.Infof("Last blocklist reload was %s ago. Reloading blocklist...", delta.String())
		s.retryReloadBlocklist()
	}
	go s.blocklistReloader(delta)
	return nil
}

func (s *Session) getBlocklistTimestamp() (time.Time, error) {
	var t time.Time
	err := s.db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(sessionBucket)
		val := b.Get(blocklistTimestampKey)
		if val == nil {
			return nil
		}
		var err2 error
		t, err2 = time.ParseInLocation(time.RFC3339, string(val), time.UTC)
		return err2
	})
	return t, err
}

func (s *Session) retryReloadBlocklist() {
	bo := backoff.NewExponentialBackOff()
	bo.MaxElapsedTime = 0

	ticker := backoff.NewTicker(bo)
	defer ticker.Stop()

	for {
		select {
		case <-ticker.C:
			err := s.reloadBlocklist()
			if err != nil {
				s.log.Errorln("cannot load blocklist:", err.Error())
				continue
			}
			return
		case <-s.closeC:
			return
		}
	}
}

func (s *Session) reloadBlocklist() error {
	resp, err := http.Get(s.config.BlocklistURL)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return errors.New("invalid blocklist status code")
	}

	var r io.Reader = resp.Body
	if resp.Header.Get("content-type") == "application/x-gzip" {
		gr, gerr := gzip.NewReader(r)
		if gerr != nil {
			return gerr
		}
		defer gr.Close()
		r = gr
	}

	buf := bytes.NewBuffer(make([]byte, 0, resp.ContentLength))
	r = io.TeeReader(r, buf)

	n, err := s.blocklist.Reload(r)
	if err != nil {
		return err
	}
	s.log.Infof("Loaded %d rules from blocklist.", n)

	now := time.Now()

	s.mBlocklist.Lock()
	s.blocklistTimestamp = now
	s.mBlocklist.Unlock()

	return s.db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket(sessionBucket)
		err2 := b.Put(blocklistKey, buf.Bytes())
		if err2 != nil {
			return err2
		}
		return b.Put(blocklistTimestampKey, []byte(now.UTC().Format(time.RFC3339)))
	})
}

func (s *Session) blocklistReloader(d time.Duration) {
	for {
		select {
		case <-time.After(d):
		case <-s.closeC:
			return
		}

		s.log.Info("Reloading blocklist...")
		s.retryReloadBlocklist()
		d = s.config.BlocklistUpdateInterval
	}
}
<|endoftext|>


<commit_before>package tq

import (
	"fmt"
	"hash"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"strconv"

	"github.com/git-lfs/git-lfs/errors"
	"github.com/git-lfs/git-lfs/tools"
	"github.com/rubyist/tracerx"
)

// Adapter for basic HTTP downloads, includes resuming via HTTP Range
type basicDownloadAdapter struct {
	*adapterBase
}

func (a *basicDownloadAdapter) ClearTempStorage() error {
	return os.RemoveAll(a.tempDir())
}

func (a *basicDownloadAdapter) tempDir() string {
	// Must be dedicated to this adapter as deleted by ClearTempStorage
	// Also make local to this repo not global, and separate to localstorage temp,
	// which gets cleared at the end of every invocation
	d := filepath.Join(a.fs.LFSStorageDir, "incomplete")
	if err := tools.MkdirAll(d, a.fs); err != nil {
		return os.TempDir()
	}
	return d
}

func (a *basicDownloadAdapter) WorkerStarting(workerNum int) (interface{}, error) {
	return nil, nil
}
func (a *basicDownloadAdapter) WorkerEnding(workerNum int, ctx interface{}) {
}

func (a *basicDownloadAdapter) DoTransfer(ctx interface{}, t *Transfer, cb ProgressCallback, authOkFunc func()) error {
	f, fromByte, hashSoFar, err := a.checkResumeDownload(t)
	if err != nil {
		return err
	}
	return a.download(t, cb, authOkFunc, f, fromByte, hashSoFar)
}

// Checks to see if a download can be resumed, and if so returns a non-nil locked file, byte start and hash
func (a *basicDownloadAdapter) checkResumeDownload(t *Transfer) (outFile *os.File, fromByte int64, hashSoFar hash.Hash, e error) {
	// lock the file by opening it for read/write, rather than checking Stat() etc
	// which could be subject to race conditions by other processes
	f, err := os.OpenFile(a.downloadFilename(t), os.O_RDWR, 0644)

	if err != nil {
		// Create a new file instead, must not already exist or error (permissions / race condition)
		newfile, err := os.OpenFile(a.downloadFilename(t), os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0644)
		return newfile, 0, nil, err
	}

	// Successfully opened an existing file at this point
	// Read any existing data into hash then return file handle at end
	hash := tools.NewLfsContentHash()
	n, err := io.Copy(hash, f)
	if err != nil {
		f.Close()
		return nil, 0, nil, err
	}
	tracerx.Printf("xfer: Attempting to resume download of %q from byte %d", t.Oid, n)
	return f, n, hash, nil

}

// Create or open a download file for resuming
func (a *basicDownloadAdapter) downloadFilename(t *Transfer) string {
	// Not a temp file since we will be resuming it
	return filepath.Join(a.tempDir(), t.Oid+".tmp")
}

// download starts or resumes and download. Always closes dlFile if non-nil
func (a *basicDownloadAdapter) download(t *Transfer, cb ProgressCallback, authOkFunc func(), dlFile *os.File, fromByte int64, hash hash.Hash) error {
	if dlFile != nil {
		// ensure we always close dlFile. Note that this does not conflict with the
		// early close below, as close is idempotent.
		defer dlFile.Close()
	}

	rel, err := t.Rel("download")
	if err != nil {
		return err
	}
	if rel == nil {
		return errors.Errorf("Object %s not found on the server.", t.Oid)
	}

	req, err := a.newHTTPRequest("GET", rel)
	if err != nil {
		return err
	}

	if fromByte > 0 {
		if dlFile == nil || hash == nil {
			return fmt.Errorf("Cannot restart %v from %d without a file & hash", t.Oid, fromByte)
		}

		if fromByte < t.Size-1 {
			// We could just use a start byte, but since we know the length be specific
			req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", fromByte, t.Size-1))
		} else {
			// Somehow we have more data than expected. Let's retry
			// from the top.
			dlFile.Close()
			os.Remove(dlFile.Name())

			dlFile = nil
			fromByte = 0
			hash = nil
		}
	}

	req = a.apiClient.LogRequest(req, "lfs.data.download")
	res, err := a.makeRequest(t, req)
	if err != nil {
		if res == nil {
			// We encountered a network or similar error which caused us
			// to not receive a response at all.
			return errors.NewRetriableError(err)
		}

		// Special-case status code 416 () - fall back
		if fromByte > 0 && dlFile != nil && res.StatusCode == 416 {
			tracerx.Printf("xfer: server rejected resume download request for %q from byte %d; re-downloading from start", t.Oid, fromByte)
			dlFile.Close()
			os.Remove(dlFile.Name())
			return a.download(t, cb, authOkFunc, nil, 0, nil)
		}

		// Special-cae status code 429 - retry after certain time
		if res.StatusCode == 429 {
			retLaterErr := errors.NewRetriableLaterError(err, res.Header["Retry-After"][0])
			if retLaterErr != nil {
				return retLaterErr
			}
		}

		return errors.NewRetriableError(err)
	}

	defer res.Body.Close()

	// Range request must return 206 & content range to confirm
	if fromByte > 0 {
		rangeRequestOk := false
		var failReason string
		// check 206 and Content-Range, fall back if either not as expected
		if res.StatusCode == 206 {
			// Probably a successful range request, check Content-Range
			if rangeHdr := res.Header.Get("Content-Range"); rangeHdr != "" {
				regex := regexp.MustCompile(`bytes (\d+)\-.*`)
				match := regex.FindStringSubmatch(rangeHdr)
				if match != nil && len(match) > 1 {
					contentStart, _ := strconv.ParseInt(match[1], 10, 64)
					if contentStart == fromByte {
						rangeRequestOk = true
					} else {
						failReason = fmt.Sprintf("Content-Range start byte incorrect: %s expected %d", match[1], fromByte)
					}
				} else {
					failReason = fmt.Sprintf("badly formatted Content-Range header: %q", rangeHdr)
				}
			} else {
				failReason = "missing Content-Range header in response"
			}
		} else {
			failReason = fmt.Sprintf("expected status code 206, received %d", res.StatusCode)
		}
		if rangeRequestOk {
			tracerx.Printf("xfer: server accepted resume download request: %q from byte %d", t.Oid, fromByte)
			advanceCallbackProgress(cb, t, fromByte)
		} else {
			// Abort resume, perform regular download
			tracerx.Printf("xfer: failed to resume download for %q from byte %d: %s. Re-downloading from start", t.Oid, fromByte, failReason)
			dlFile.Close()
			os.Remove(dlFile.Name())
			if res.StatusCode == 200 {
				// If status code was 200 then server just ignored Range header and
				// sent everything. Don't re-request, use this one from byte 0
				dlFile = nil
				fromByte = 0
				hash = nil
			} else {
				// re-request needed
				return a.download(t, cb, authOkFunc, nil, 0, nil)
			}
		}
	}

	// Signal auth OK on success response, before starting download to free up
	// other workers immediately
	if authOkFunc != nil {
		authOkFunc()
	}

	var hasher *tools.HashingReader
	httpReader := tools.NewRetriableReader(res.Body)

	if fromByte > 0 && hash != nil {
		// pre-load hashing reader with previous content
		hasher = tools.NewHashingReaderPreloadHash(httpReader, hash)
	} else {
		hasher = tools.NewHashingReader(httpReader)
	}

	if dlFile == nil {
		// New file start
		dlFile, err = os.OpenFile(a.downloadFilename(t), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
		if err != nil {
			return err
		}
		defer dlFile.Close()
	}
	dlfilename := dlFile.Name()
	// Wrap callback to give name context
	ccb := func(totalSize int64, readSoFar int64, readSinceLast int) error {
		if cb != nil {
			return cb(t.Name, totalSize, readSoFar+fromByte, readSinceLast)
		}
		return nil
	}
	written, err := tools.CopyWithCallback(dlFile, hasher, res.ContentLength, ccb)
	if err != nil {
		return errors.Wrapf(err, "cannot write data to tempfile %q", dlfilename)
	}
	if err := dlFile.Close(); err != nil {
		return fmt.Errorf("can't close tempfile %q: %v", dlfilename, err)
	}

	if actual := hasher.Hash(); actual != t.Oid {
		return fmt.Errorf("Expected OID %s, got %s after %d bytes written", t.Oid, actual, written)
	}

	err = tools.RenameFileCopyPermissions(dlfilename, t.Path)
	if _, err2 := os.Stat(t.Path); err2 == nil {
		// Target file already exists, possibly was downloaded by other git-lfs process
		return nil
	}
	return err
}

func configureBasicDownloadAdapter(m *Manifest) {
	m.RegisterNewAdapterFunc(BasicAdapterName, Download, func(name string, dir Direction) Adapter {
		switch dir {
		case Download:
			bd := &basicDownloadAdapter{newAdapterBase(m.fs, name, dir, nil)}
			// self implements impl
			bd.transferImpl = bd
			return bd
		case Upload:
			panic("Should never ask this func to upload")
		}
		return nil
	})
}

func (a *basicDownloadAdapter) makeRequest(t *Transfer, req *http.Request) (*http.Response, error) {
	res, err := a.doHTTP(t, req)
	if errors.IsAuthError(err) && len(req.Header.Get("Authorization")) == 0 {
		return a.makeRequest(t, req)
	}

	return res, err
}
<commit_msg>More robust handling of parallel attempts to download the same file<commit_after>package tq

import (
	"fmt"
	"hash"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"strconv"

	"github.com/git-lfs/git-lfs/errors"
	"github.com/git-lfs/git-lfs/tools"
	"github.com/rubyist/tracerx"
)

// Adapter for basic HTTP downloads, includes resuming via HTTP Range
type basicDownloadAdapter struct {
	*adapterBase
}

func (a *basicDownloadAdapter) ClearTempStorage() error {
	return os.RemoveAll(a.tempDir())
}

func (a *basicDownloadAdapter) tempDir() string {
	// Must be dedicated to this adapter as deleted by ClearTempStorage
	// Also make local to this repo not global, and separate to localstorage temp,
	// which gets cleared at the end of every invocation
	d := filepath.Join(a.fs.LFSStorageDir, "incomplete")
	if err := tools.MkdirAll(d, a.fs); err != nil {
		return os.TempDir()
	}
	return d
}

func (a *basicDownloadAdapter) WorkerStarting(workerNum int) (interface{}, error) {
	return nil, nil
}
func (a *basicDownloadAdapter) WorkerEnding(workerNum int, ctx interface{}) {
}

func (a *basicDownloadAdapter) DoTransfer(ctx interface{}, t *Transfer, cb ProgressCallback, authOkFunc func()) error {
	// Reserve a temporary filename. We need to make sure nobody operates on the file simultaneously with us.
	f, err := tools.TempFile(a.tempDir(), t.Oid, a.fs)
	if err != nil {
		return err
	}
	defer func() {
		f.Close()
		// This will delete temp file if:
		// - we failed to fully download file and move it to final location including the case when final location already
		//   exists because other parallel git-lfs processes downloaded file
		// - we also failed to move it to a partially-downloaded location
		os.Remove(f.Name())
	}()

	// Close file because we will attempt to move partially-downloaded one on top of it
	if err := f.Close(); err != nil {
		return err
	}

	// Attempt to resume download. No error checking here. If we fail, we'll simply download from the start
	os.Rename(a.downloadFilename(t), f.Name())

	// Open temp file. It is either empty or partially downloaded
	f, err = os.OpenFile(f.Name(), os.O_RDWR, 0644)
	if err != nil {
		return err
	}

	// Read any existing data into hash
	hash := tools.NewLfsContentHash()
	fromByte, err := io.Copy(hash, f)
	if err != nil {
		return err
	}

	// Ensure that partial file seems valid
	if fromByte > 0 {
		if fromByte < t.Size-1 {
			tracerx.Printf("xfer: Attempting to resume download of %q from byte %d", t.Oid, fromByte)
		} else {
			// Somehow we have more data than expected. Let's retry from the beginning.
			if _, err := f.Seek(0, io.SeekStart); err != nil {
				return err
			}
			if err := f.Truncate(0); err != nil {
				return err
			}
			fromByte = 0
			hash = nil
		}
	}

	err = a.download(t, cb, authOkFunc, f, fromByte, hash)

	if err != nil {
		f.Close()
		// Rename file so next download can resume from where we stopped.
		// No error checking here, if rename fails then file will be deleted and there just will be no download resuming
		os.Rename(f.Name(), a.downloadFilename(t))
	}

	return err
}

// Returns path where partially downloaded file should be stored for download resuming
func (a *basicDownloadAdapter) downloadFilename(t *Transfer) string {
	return filepath.Join(a.tempDir(), t.Oid+".part")
}

// download starts or resumes and download. dlFile is expected to be an existing file open in RW mode
func (a *basicDownloadAdapter) download(t *Transfer, cb ProgressCallback, authOkFunc func(), dlFile *os.File, fromByte int64, hash hash.Hash) error {
	rel, err := t.Rel("download")
	if err != nil {
		return err
	}
	if rel == nil {
		return errors.Errorf("Object %s not found on the server.", t.Oid)
	}

	req, err := a.newHTTPRequest("GET", rel)
	if err != nil {
		return err
	}

	if fromByte > 0 {
		// We could just use a start byte, but since we know the length be specific
		req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", fromByte, t.Size-1))
	}

	req = a.apiClient.LogRequest(req, "lfs.data.download")
	res, err := a.makeRequest(t, req)
	if err != nil {
		if res == nil {
			// We encountered a network or similar error which caused us
			// to not receive a response at all.
			return errors.NewRetriableError(err)
		}

		// Special-case status code 416 () - fall back
		if fromByte > 0 && dlFile != nil && res.StatusCode == 416 {
			tracerx.Printf("xfer: server rejected resume download request for %q from byte %d; re-downloading from start", t.Oid, fromByte)
			if _, err := dlFile.Seek(0, io.SeekStart); err != nil {
				return err
			}
			if err := dlFile.Truncate(0); err != nil {
				return err
			}
			return a.download(t, cb, authOkFunc, dlFile, 0, nil)
		}

		// Special-cae status code 429 - retry after certain time
		if res.StatusCode == 429 {
			retLaterErr := errors.NewRetriableLaterError(err, res.Header["Retry-After"][0])
			if retLaterErr != nil {
				return retLaterErr
			}
		}

		return errors.NewRetriableError(err)
	}

	defer res.Body.Close()

	// Range request must return 206 & content range to confirm
	if fromByte > 0 {
		rangeRequestOk := false
		var failReason string
		// check 206 and Content-Range, fall back if either not as expected
		if res.StatusCode == 206 {
			// Probably a successful range request, check Content-Range
			if rangeHdr := res.Header.Get("Content-Range"); rangeHdr != "" {
				regex := regexp.MustCompile(`bytes (\d+)\-.*`)
				match := regex.FindStringSubmatch(rangeHdr)
				if match != nil && len(match) > 1 {
					contentStart, _ := strconv.ParseInt(match[1], 10, 64)
					if contentStart == fromByte {
						rangeRequestOk = true
					} else {
						failReason = fmt.Sprintf("Content-Range start byte incorrect: %s expected %d", match[1], fromByte)
					}
				} else {
					failReason = fmt.Sprintf("badly formatted Content-Range header: %q", rangeHdr)
				}
			} else {
				failReason = "missing Content-Range header in response"
			}
		} else {
			failReason = fmt.Sprintf("expected status code 206, received %d", res.StatusCode)
		}
		if rangeRequestOk {
			tracerx.Printf("xfer: server accepted resume download request: %q from byte %d", t.Oid, fromByte)
			advanceCallbackProgress(cb, t, fromByte)
		} else {
			// Abort resume, perform regular download
			tracerx.Printf("xfer: failed to resume download for %q from byte %d: %s. Re-downloading from start", t.Oid, fromByte, failReason)

			if _, err := dlFile.Seek(0, io.SeekStart); err != nil {
				return err
			}
			if err := dlFile.Truncate(0); err != nil {
				return err
			}
			fromByte = 0
			hash = nil

			if res.StatusCode == 200 {
				// If status code was 200 then server just ignored Range header and
				// sent everything. Don't re-request, use this one from byte 0
			} else {
				// re-request needed
				return a.download(t, cb, authOkFunc, dlFile, fromByte, hash)
			}
		}
	}

	// Signal auth OK on success response, before starting download to free up
	// other workers immediately
	if authOkFunc != nil {
		authOkFunc()
	}

	var hasher *tools.HashingReader
	httpReader := tools.NewRetriableReader(res.Body)

	if fromByte > 0 && hash != nil {
		// pre-load hashing reader with previous content
		hasher = tools.NewHashingReaderPreloadHash(httpReader, hash)
	} else {
		hasher = tools.NewHashingReader(httpReader)
	}

	dlfilename := dlFile.Name()
	// Wrap callback to give name context
	ccb := func(totalSize int64, readSoFar int64, readSinceLast int) error {
		if cb != nil {
			return cb(t.Name, totalSize, readSoFar+fromByte, readSinceLast)
		}
		return nil
	}
	written, err := tools.CopyWithCallback(dlFile, hasher, res.ContentLength, ccb)
	if err != nil {
		return errors.Wrapf(err, "cannot write data to tempfile %q", dlfilename)
	}

	if actual := hasher.Hash(); actual != t.Oid {
		return fmt.Errorf("expected OID %s, got %s after %d bytes written", t.Oid, actual, written)
	}

	if err := dlFile.Close(); err != nil {
		return fmt.Errorf("can't close tempfile %q: %v", dlfilename, err)
	}

	err = tools.RenameFileCopyPermissions(dlfilename, t.Path)
	if _, err2 := os.Stat(t.Path); err2 == nil {
		// Target file already exists, possibly was downloaded by other git-lfs process
		return nil
	}
	return err
}

func configureBasicDownloadAdapter(m *Manifest) {
	m.RegisterNewAdapterFunc(BasicAdapterName, Download, func(name string, dir Direction) Adapter {
		switch dir {
		case Download:
			bd := &basicDownloadAdapter{newAdapterBase(m.fs, name, dir, nil)}
			// self implements impl
			bd.transferImpl = bd
			return bd
		case Upload:
			panic("Should never ask this func to upload")
		}
		return nil
	})
}

func (a *basicDownloadAdapter) makeRequest(t *Transfer, req *http.Request) (*http.Response, error) {
	res, err := a.doHTTP(t, req)
	if errors.IsAuthError(err) && len(req.Header.Get("Authorization")) == 0 {
		return a.makeRequest(t, req)
	}

	return res, err
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build darwin dragonfly freebsd linux netbsd openbsd

package syscall_test

import (
	"syscall"
	"testing"
)

func TestRlimit(t *testing.T) {
	var rlimit, zero syscall.Rlimit
	err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlimit)
	if err != nil {
		t.Fatalf("Getrlimit: save failed: %v", err)
	}
	if zero == rlimit {
		t.Fatalf("Getrlimit: save failed: got zero value %#v", rlimit)
	}
	set := rlimit
	set.Cur = set.Max - 1
	err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &set)
	if err != nil {
		t.Fatalf("Setrlimit: set failed: %#v %v", set, err)
	}
	var get syscall.Rlimit
	err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &get)
	if err != nil {
		t.Fatalf("Getrlimit: get failed: %v", err)
	}
	set = rlimit
	set.Cur = set.Max - 1
	if set != get {
		t.Fatalf("Rlimit: change failed: wanted %#v got %#v", set, get)
	}
	err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rlimit)
	if err != nil {
		t.Fatalf("Setrlimit: restore failed: %#v %v", rlimit, err)
	}
}
<commit_msg>syscall: fix build<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build darwin dragonfly freebsd linux netbsd openbsd

package syscall_test

import (
	"runtime"
	"syscall"
	"testing"
)

func TestRlimit(t *testing.T) {
	var rlimit, zero syscall.Rlimit
	err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlimit)
	if err != nil {
		t.Fatalf("Getrlimit: save failed: %v", err)
	}
	if zero == rlimit {
		t.Fatalf("Getrlimit: save failed: got zero value %#v", rlimit)
	}
	set := rlimit
	set.Cur = set.Max - 1
	err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &set)
	if err != nil {
		t.Fatalf("Setrlimit: set failed: %#v %v", set, err)
	}
	var get syscall.Rlimit
	err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &get)
	if err != nil {
		t.Fatalf("Getrlimit: get failed: %v", err)
	}
	set = rlimit
	set.Cur = set.Max - 1
	if set != get {
		// Seems like Darwin requires some privilege to
		// increse the soft limit of rlimit sandbox, though
		// Setrlimit never reports error.
		switch runtime.GOOS {
		case "darwin":
		default:
			t.Fatalf("Rlimit: change failed: wanted %#v got %#v", set, get)
		}
	}
	err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rlimit)
	if err != nil {
		t.Fatalf("Setrlimit: restore failed: %#v %v", rlimit, err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package websocket

import (
	"bufio"
	"bytes"
	"fmt"
	"http"
	"http/httptest"
	"io"
	"log"
	"net"
	"sync"
	"testing"
)

var serverAddr string
var once sync.Once

func echoServer(ws *Conn) { io.Copy(ws, ws) }

func startServer() {
	http.Handle("/echo", Handler(echoServer))
	http.Handle("/echoDraft75", Draft75Handler(echoServer))
	server := httptest.NewServer(nil)
	serverAddr = server.Listener.Addr().String()
	log.Print("Test WebSocket server listening on ", serverAddr)
}

// Test the getChallengeResponse function with values from section
// 5.1 of the specification steps 18, 26, and 43 from
// http://www.whatwg.org/specs/web-socket-protocol/
func TestChallenge(t *testing.T) {
	var part1 uint32 = 777007543
	var part2 uint32 = 114997259
	key3 := []byte{0x47, 0x30, 0x22, 0x2D, 0x5A, 0x3F, 0x47, 0x58}
	expected := []byte("0st3Rl&q-2ZU^weu")

	response, err := getChallengeResponse(part1, part2, key3)
	if err != nil {
		t.Errorf("getChallengeResponse: returned error %v", err)
		return
	}
	if !bytes.Equal(expected, response) {
		t.Errorf("getChallengeResponse: expected %q got %q", expected, response)
	}
}

func TestEcho(t *testing.T) {
	once.Do(startServer)

	// websocket.Dial()
	client, err := net.Dial("tcp", serverAddr)
	if err != nil {
		t.Fatal("dialing", err)
	}
	ws, err := newClient("/echo", "localhost", "http://localhost",
		"ws://localhost/echo", "", client, handshake)
	if err != nil {
		t.Errorf("WebSocket handshake error: %v", err)
		return
	}

	msg := []byte("hello, world\n")
	if _, err := ws.Write(msg); err != nil {
		t.Errorf("Write: %v", err)
	}
	var actual_msg = make([]byte, 512)
	n, err := ws.Read(actual_msg)
	if err != nil {
		t.Errorf("Read: %v", err)
	}
	actual_msg = actual_msg[0:n]
	if !bytes.Equal(msg, actual_msg) {
		t.Errorf("Echo: expected %q got %q", msg, actual_msg)
	}
	ws.Close()
}

func TestEchoDraft75(t *testing.T) {
	once.Do(startServer)

	// websocket.Dial()
	client, err := net.Dial("tcp", serverAddr)
	if err != nil {
		t.Fatal("dialing", err)
	}
	ws, err := newClient("/echoDraft75", "localhost", "http://localhost",
		"ws://localhost/echoDraft75", "", client, draft75handshake)
	if err != nil {
		t.Errorf("WebSocket handshake: %v", err)
		return
	}

	msg := []byte("hello, world\n")
	if _, err := ws.Write(msg); err != nil {
		t.Errorf("Write: error %v", err)
	}
	var actual_msg = make([]byte, 512)
	n, err := ws.Read(actual_msg)
	if err != nil {
		t.Errorf("Read: error %v", err)
	}
	actual_msg = actual_msg[0:n]
	if !bytes.Equal(msg, actual_msg) {
		t.Errorf("Echo: expected %q got %q", msg, actual_msg)
	}
	ws.Close()
}

func TestWithQuery(t *testing.T) {
	once.Do(startServer)

	client, err := net.Dial("tcp", serverAddr)
	if err != nil {
		t.Fatal("dialing", err)
	}

	ws, err := newClient("/echo?q=v", "localhost", "http://localhost",
		"ws://localhost/echo?q=v", "", client, handshake)
	if err != nil {
		t.Errorf("WebSocket handshake: %v", err)
		return
	}
	ws.Close()
}

func TestWithProtocol(t *testing.T) {
	once.Do(startServer)

	client, err := net.Dial("tcp", serverAddr)
	if err != nil {
		t.Fatal("dialing", err)
	}

	ws, err := newClient("/echo", "localhost", "http://localhost",
		"ws://localhost/echo", "test", client, handshake)
	if err != nil {
		t.Errorf("WebSocket handshake: %v", err)
		return
	}
	ws.Close()
}

func TestHTTP(t *testing.T) {
	once.Do(startServer)

	// If the client did not send a handshake that matches the protocol
	// specification, the server should abort the WebSocket connection.
	_, _, err := http.Get(fmt.Sprintf("http://%s/echo", serverAddr))
	if err == nil {
		t.Error("Get: unexpected success")
		return
	}
	urlerr, ok := err.(*http.URLError)
	if !ok {
		t.Errorf("Get: not URLError %#v", err)
		return
	}
	if urlerr.Error != io.ErrUnexpectedEOF {
		t.Errorf("Get: error %#v", err)
		return
	}
}

func TestHTTPDraft75(t *testing.T) {
	once.Do(startServer)

	r, _, err := http.Get(fmt.Sprintf("http://%s/echoDraft75", serverAddr))
	if err != nil {
		t.Errorf("Get: error %#v", err)
		return
	}
	if r.StatusCode != http.StatusBadRequest {
		t.Errorf("Get: got status %d", r.StatusCode)
	}
}

func TestTrailingSpaces(t *testing.T) {
	// http://code.google.com/p/go/issues/detail?id=955
	// The last runs of this create keys with trailing spaces that should not be
	// generated by the client.
	once.Do(startServer)
	for i := 0; i < 30; i++ {
		// body
		_, err := Dial(fmt.Sprintf("ws://%s/echo", serverAddr), "",
			"http://localhost/")
		if err != nil {
			panic("Dial failed: " + err.String())
		}
	}
}

func TestSmallBuffer(t *testing.T) {
	// http://code.google.com/p/go/issues/detail?id=1145
	// Read should be able to handle reading a fragment of a frame.
	once.Do(startServer)

	// websocket.Dial()
	client, err := net.Dial("tcp", serverAddr)
	if err != nil {
		t.Fatal("dialing", err)
	}
	ws, err := newClient("/echo", "localhost", "http://localhost",
		"ws://localhost/echo", "", client, handshake)
	if err != nil {
		t.Errorf("WebSocket handshake error: %v", err)
		return
	}

	msg := []byte("hello, world\n")
	if _, err := ws.Write(msg); err != nil {
		t.Errorf("Write: %v", err)
	}
	var small_msg = make([]byte, 8)
	n, err := ws.Read(small_msg)
	if err != nil {
		t.Errorf("Read: %v", err)
	}
	if !bytes.Equal(msg[:len(small_msg)], small_msg) {
		t.Errorf("Echo: expected %q got %q", msg[:len(small_msg)], small_msg)
	}
	var second_msg = make([]byte, len(msg))
	n, err = ws.Read(second_msg)
	if err != nil {
		t.Errorf("Read: %v", err)
	}
	second_msg = second_msg[0:n]
	if !bytes.Equal(msg[len(small_msg):], second_msg) {
		t.Errorf("Echo: expected %q got %q", msg[len(small_msg):], second_msg)
	}
	ws.Close()

}

func testSkipLengthFrame(t *testing.T) {
	b := []byte{'\x80', '\x01', 'x', 0, 'h', 'e', 'l', 'l', 'o', '\xff'}
	buf := bytes.NewBuffer(b)
	br := bufio.NewReader(buf)
	bw := bufio.NewWriter(buf)
	ws := newConn("http://127.0.0.1/", "ws://127.0.0.1/", "", bufio.NewReadWriter(br, bw), nil)
	msg := make([]byte, 5)
	n, err := ws.Read(msg)
	if err != nil {
		t.Errorf("Read: %v", err)
	}
	if !bytes.Equal(b[4:8], msg[0:n]) {
		t.Errorf("Read: expected %q got %q", msg[4:8], msg[0:n])
	}
}

func testSkipNoUTF8Frame(t *testing.T) {
	b := []byte{'\x01', 'n', '\xff', 0, 'h', 'e', 'l', 'l', 'o', '\xff'}
	buf := bytes.NewBuffer(b)
	br := bufio.NewReader(buf)
	bw := bufio.NewWriter(buf)
	ws := newConn("http://127.0.0.1/", "ws://127.0.0.1/", "", bufio.NewReadWriter(br, bw), nil)
	msg := make([]byte, 5)
	n, err := ws.Read(msg)
	if err != nil {
		t.Errorf("Read: %v", err)
	}
	if !bytes.Equal(b[4:8], msg[0:n]) {
		t.Errorf("Read: expected %q got %q", msg[4:8], msg[0:n])
	}
}
<commit_msg>websocket: fix socket leak in test<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package websocket

import (
	"bufio"
	"bytes"
	"fmt"
	"http"
	"http/httptest"
	"io"
	"log"
	"net"
	"sync"
	"testing"
)

var serverAddr string
var once sync.Once

func echoServer(ws *Conn) { io.Copy(ws, ws) }

func startServer() {
	http.Handle("/echo", Handler(echoServer))
	http.Handle("/echoDraft75", Draft75Handler(echoServer))
	server := httptest.NewServer(nil)
	serverAddr = server.Listener.Addr().String()
	log.Print("Test WebSocket server listening on ", serverAddr)
}

// Test the getChallengeResponse function with values from section
// 5.1 of the specification steps 18, 26, and 43 from
// http://www.whatwg.org/specs/web-socket-protocol/
func TestChallenge(t *testing.T) {
	var part1 uint32 = 777007543
	var part2 uint32 = 114997259
	key3 := []byte{0x47, 0x30, 0x22, 0x2D, 0x5A, 0x3F, 0x47, 0x58}
	expected := []byte("0st3Rl&q-2ZU^weu")

	response, err := getChallengeResponse(part1, part2, key3)
	if err != nil {
		t.Errorf("getChallengeResponse: returned error %v", err)
		return
	}
	if !bytes.Equal(expected, response) {
		t.Errorf("getChallengeResponse: expected %q got %q", expected, response)
	}
}

func TestEcho(t *testing.T) {
	once.Do(startServer)

	// websocket.Dial()
	client, err := net.Dial("tcp", serverAddr)
	if err != nil {
		t.Fatal("dialing", err)
	}
	ws, err := newClient("/echo", "localhost", "http://localhost",
		"ws://localhost/echo", "", client, handshake)
	if err != nil {
		t.Errorf("WebSocket handshake error: %v", err)
		return
	}

	msg := []byte("hello, world\n")
	if _, err := ws.Write(msg); err != nil {
		t.Errorf("Write: %v", err)
	}
	var actual_msg = make([]byte, 512)
	n, err := ws.Read(actual_msg)
	if err != nil {
		t.Errorf("Read: %v", err)
	}
	actual_msg = actual_msg[0:n]
	if !bytes.Equal(msg, actual_msg) {
		t.Errorf("Echo: expected %q got %q", msg, actual_msg)
	}
	ws.Close()
}

func TestEchoDraft75(t *testing.T) {
	once.Do(startServer)

	// websocket.Dial()
	client, err := net.Dial("tcp", serverAddr)
	if err != nil {
		t.Fatal("dialing", err)
	}
	ws, err := newClient("/echoDraft75", "localhost", "http://localhost",
		"ws://localhost/echoDraft75", "", client, draft75handshake)
	if err != nil {
		t.Errorf("WebSocket handshake: %v", err)
		return
	}

	msg := []byte("hello, world\n")
	if _, err := ws.Write(msg); err != nil {
		t.Errorf("Write: error %v", err)
	}
	var actual_msg = make([]byte, 512)
	n, err := ws.Read(actual_msg)
	if err != nil {
		t.Errorf("Read: error %v", err)
	}
	actual_msg = actual_msg[0:n]
	if !bytes.Equal(msg, actual_msg) {
		t.Errorf("Echo: expected %q got %q", msg, actual_msg)
	}
	ws.Close()
}

func TestWithQuery(t *testing.T) {
	once.Do(startServer)

	client, err := net.Dial("tcp", serverAddr)
	if err != nil {
		t.Fatal("dialing", err)
	}

	ws, err := newClient("/echo?q=v", "localhost", "http://localhost",
		"ws://localhost/echo?q=v", "", client, handshake)
	if err != nil {
		t.Errorf("WebSocket handshake: %v", err)
		return
	}
	ws.Close()
}

func TestWithProtocol(t *testing.T) {
	once.Do(startServer)

	client, err := net.Dial("tcp", serverAddr)
	if err != nil {
		t.Fatal("dialing", err)
	}

	ws, err := newClient("/echo", "localhost", "http://localhost",
		"ws://localhost/echo", "test", client, handshake)
	if err != nil {
		t.Errorf("WebSocket handshake: %v", err)
		return
	}
	ws.Close()
}

func TestHTTP(t *testing.T) {
	once.Do(startServer)

	// If the client did not send a handshake that matches the protocol
	// specification, the server should abort the WebSocket connection.
	_, _, err := http.Get(fmt.Sprintf("http://%s/echo", serverAddr))
	if err == nil {
		t.Error("Get: unexpected success")
		return
	}
	urlerr, ok := err.(*http.URLError)
	if !ok {
		t.Errorf("Get: not URLError %#v", err)
		return
	}
	if urlerr.Error != io.ErrUnexpectedEOF {
		t.Errorf("Get: error %#v", err)
		return
	}
}

func TestHTTPDraft75(t *testing.T) {
	once.Do(startServer)

	r, _, err := http.Get(fmt.Sprintf("http://%s/echoDraft75", serverAddr))
	if err != nil {
		t.Errorf("Get: error %#v", err)
		return
	}
	if r.StatusCode != http.StatusBadRequest {
		t.Errorf("Get: got status %d", r.StatusCode)
	}
}

func TestTrailingSpaces(t *testing.T) {
	// http://code.google.com/p/go/issues/detail?id=955
	// The last runs of this create keys with trailing spaces that should not be
	// generated by the client.
	once.Do(startServer)
	for i := 0; i < 30; i++ {
		// body
		ws, err := Dial(fmt.Sprintf("ws://%s/echo", serverAddr), "", "http://localhost/")
		if err != nil {
			t.Error("Dial failed:", err.String())
			break
		}
		ws.Close()
	}
}

func TestSmallBuffer(t *testing.T) {
	// http://code.google.com/p/go/issues/detail?id=1145
	// Read should be able to handle reading a fragment of a frame.
	once.Do(startServer)

	// websocket.Dial()
	client, err := net.Dial("tcp", serverAddr)
	if err != nil {
		t.Fatal("dialing", err)
	}
	ws, err := newClient("/echo", "localhost", "http://localhost",
		"ws://localhost/echo", "", client, handshake)
	if err != nil {
		t.Errorf("WebSocket handshake error: %v", err)
		return
	}

	msg := []byte("hello, world\n")
	if _, err := ws.Write(msg); err != nil {
		t.Errorf("Write: %v", err)
	}
	var small_msg = make([]byte, 8)
	n, err := ws.Read(small_msg)
	if err != nil {
		t.Errorf("Read: %v", err)
	}
	if !bytes.Equal(msg[:len(small_msg)], small_msg) {
		t.Errorf("Echo: expected %q got %q", msg[:len(small_msg)], small_msg)
	}
	var second_msg = make([]byte, len(msg))
	n, err = ws.Read(second_msg)
	if err != nil {
		t.Errorf("Read: %v", err)
	}
	second_msg = second_msg[0:n]
	if !bytes.Equal(msg[len(small_msg):], second_msg) {
		t.Errorf("Echo: expected %q got %q", msg[len(small_msg):], second_msg)
	}
	ws.Close()

}

func testSkipLengthFrame(t *testing.T) {
	b := []byte{'\x80', '\x01', 'x', 0, 'h', 'e', 'l', 'l', 'o', '\xff'}
	buf := bytes.NewBuffer(b)
	br := bufio.NewReader(buf)
	bw := bufio.NewWriter(buf)
	ws := newConn("http://127.0.0.1/", "ws://127.0.0.1/", "", bufio.NewReadWriter(br, bw), nil)
	msg := make([]byte, 5)
	n, err := ws.Read(msg)
	if err != nil {
		t.Errorf("Read: %v", err)
	}
	if !bytes.Equal(b[4:8], msg[0:n]) {
		t.Errorf("Read: expected %q got %q", msg[4:8], msg[0:n])
	}
}

func testSkipNoUTF8Frame(t *testing.T) {
	b := []byte{'\x01', 'n', '\xff', 0, 'h', 'e', 'l', 'l', 'o', '\xff'}
	buf := bytes.NewBuffer(b)
	br := bufio.NewReader(buf)
	bw := bufio.NewWriter(buf)
	ws := newConn("http://127.0.0.1/", "ws://127.0.0.1/", "", bufio.NewReadWriter(br, bw), nil)
	msg := make([]byte, 5)
	n, err := ws.Read(msg)
	if err != nil {
		t.Errorf("Read: %v", err)
	}
	if !bytes.Equal(b[4:8], msg[0:n]) {
		t.Errorf("Read: expected %q got %q", msg[4:8], msg[0:n])
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 Krister Svanlund
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

// The rest/utils package supplies some tools that are useful when processing
// REST requests and generating their responses.
package utils

import (
    "github.com/gorilla/mux"
    "net/http"
    "strconv"
    // "strings"
)

var (
    prevrequest = map[string]uint64{}
)

// Check if a certain request is a PUT and has a txnId in their request. If it
// does and the access token of the requests hasn't already made a request with
// the same txnId `true` is returned, otherwise `false`.
func CheckTxnId(r *http.Request) bool {
    vars := mux.Vars(r)
    token := r.URL.Query().Get("access_token")
    if stxnId := vars["txnId"]; r.Method == "PUT" && stxnId != "" {
        if txnId, err := strconv.ParseUint(stxnId, 10, 64); err == nil {
            // ip := strings.Split(r.RemoteAddr, ":")[0]
            // if prevrequest[ip] == txnId {
            if prevrequest[token] == txnId {
                return false
            } else {
                // prevrequest[ip] = txnId
                prevrequest[token] = txnId
            }
        }
    }
    return true
}

// By calling this in a request handler the http://matrix.org live test tool
// is able to make requests.
func AllowMatrixOrg(w http.ResponseWriter, r *http.Request) {
    w.Header().Set("Access-Control-Allow-Origin", "http://matrix.org")
    w.Header().Set("Access-Control-Allow-Headers", "Content-Type")
    w.WriteHeader(200)
}
<commit_msg>Ensure that the TxnID check doesn't panic<commit_after>// Copyright 2014 Krister Svanlund
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

// The rest/utils package supplies some tools that are useful when processing
// REST requests and generating their responses.
package utils

import (
    "github.com/gorilla/mux"
    "net/http"
    "strconv"
    // "strings"
)

var (
    prevrequest = map[string]uint64{}
)

// Check if a certain request is a PUT and has a txnId in their request. If it
// does and the access token of the requests hasn't already made a request with
// the same txnId `true` is returned, otherwise `false`.
func CheckTxnId(r *http.Request) bool {
    vars := mux.Vars(r)
    token := r.URL.Query().Get("access_token")
    if stxnId, ok := vars["txnId"]; ok && r.Method == "PUT" && stxnId != "" {
        if txnId, err := strconv.ParseUint(stxnId, 10, 64); err == nil {
            // ip := strings.Split(r.RemoteAddr, ":")[0]
            // if prevrequest[ip] == txnId {
            if prevrequest[token] == txnId {
                return false
            } else {
                // prevrequest[ip] = txnId
                prevrequest[token] = txnId
            }
        }
    }
    return true
}

// By calling this in a request handler the http://matrix.org live test tool
// is able to make requests.
func AllowMatrixOrg(w http.ResponseWriter, r *http.Request) {
    w.Header().Set("Access-Control-Allow-Origin", "http://matrix.org")
    w.Header().Set("Access-Control-Allow-Headers", "Content-Type")
    w.WriteHeader(200)
}
<|endoftext|>