<commit_before>package sexpconv

import (
	"go/ast"
	"go/token"
	"go/types"
	"lisp/function"
	"sexp"
)

func (conv *Converter) AssignStmt(node *ast.AssignStmt) sexp.Form {
	switch node.Tok {
	case token.ADD_ASSIGN:
		return conv.addAssign(node.Lhs[0], node.Rhs[0])
	case token.SUB_ASSIGN:
		return conv.subAssign(node.Lhs[0], node.Rhs[0])
	case token.MUL_ASSIGN:
		return conv.mulAssign(node.Lhs[0], node.Rhs[0])
	case token.QUO_ASSIGN:
		return conv.quoAssign(node.Lhs[0], node.Rhs[0])

	default:
		if len(node.Rhs) == len(node.Lhs) {
			return conv.singleValueAssign(node)
		}
		return conv.multiValueAssign(node)
	}
}

func (conv *Converter) addAssign(lhs ast.Expr, rhs ast.Expr) sexp.Form {
	typ := conv.basicTypeOf(rhs)
	args := [2]sexp.Form{conv.Expr(lhs), conv.Expr(rhs)}

	if typ.Info()&types.IsNumeric != 0 {
		return conv.assign(lhs, &sexp.NumAdd{Typ: typ, Args: args})
	}
	if typ.Kind() == types.String {
		return conv.assign(lhs, &sexp.Concat{Args: args})
	}

	panic("unimplemented")
}

func (conv *Converter) subAssign(lhs ast.Expr, rhs ast.Expr) sexp.Form {
	return conv.assign(lhs, &sexp.NumSub{
		Typ:  conv.basicTypeOf(rhs),
		Args: [2]sexp.Form{conv.Expr(lhs), conv.Expr(rhs)},
	})
}

func (conv *Converter) mulAssign(lhs ast.Expr, rhs ast.Expr) sexp.Form {
	return conv.assign(lhs, &sexp.NumMul{
		Typ:  conv.basicTypeOf(rhs),
		Args: [2]sexp.Form{conv.Expr(lhs), conv.Expr(rhs)},
	})
}

func (conv *Converter) quoAssign(lhs ast.Expr, rhs ast.Expr) sexp.Form {
	return conv.assign(lhs, &sexp.NumQuo{
		Typ:  conv.basicTypeOf(rhs),
		Args: [2]sexp.Form{conv.Expr(lhs), conv.Expr(rhs)},
	})
}

func (conv *Converter) multiValueAssign(node *ast.AssignStmt) *sexp.FormList {
	forms := make([]sexp.Form, len(node.Lhs))

	// First result is returned in a normal way.
	forms[0] = conv.assign(node.Lhs[0], conv.Expr(node.Rhs[0]))

	// Other results are assigned to a global variable.
	// Index uniquely identifies variable used for storage.
	for i := 1; i < len(node.Lhs); i++ {
		forms[i] = conv.assign(node.Lhs[i], &sexp.MultiValueRef{Index: i})
	}

	return &sexp.FormList{Forms: forms}
}

func (conv *Converter) singleValueAssign(node *ast.AssignStmt) *sexp.FormList {
	forms := make([]sexp.Form, 0, 1)

	for i, lhs := range node.Lhs {
		conv.ctxType = conv.typeOf(lhs)
		rhs := conv.Expr(node.Rhs[i])
		forms = append(forms, conv.assign(lhs, rhs))
	}

	return &sexp.FormList{Forms: forms}
}

func (conv *Converter) assign(lhs ast.Expr, expr sexp.Form) sexp.Form {
	switch lhs := lhs.(type) {
	case *ast.Ident:
		if def := conv.info.Defs[lhs]; def == nil {
			return &sexp.Rebind{Name: lhs.Name, Expr: expr}
		}
		return &sexp.Bind{Name: lhs.Name, Init: conv.valueCopy(expr)}

	case *ast.IndexExpr:
		switch typ := conv.typeOf(lhs.X); typ.(type) {
		case *types.Map:
			call := &sexp.Call{
				Fn: function.MapInsert,
				Args: conv.valueCopyList([]sexp.Form{
					conv.Expr(lhs.Index),
					expr,
					conv.Expr(lhs.X),
				}),
			}
			return sexp.CallStmt{Call: call}

		case *types.Array:
			return &sexp.ArrayUpdate{
				Array: conv.Expr(lhs.X),
				Index: conv.Expr(lhs.Index),
				Expr:  expr,
			}

		default:
			panic("unimplemented")
		}

	// #TODO: struct assign, indirect assign
	default:
		panic("unimplemented")
	}
}
<commit_msg>blank ident in single and multiple assignments<commit_after>package sexpconv

import (
	"go/ast"
	"go/token"
	"go/types"
	"lisp/function"
	"sexp"
)

// Go blank identifier which allows a value to be evaluated, but discarded.
const blankIdent = "_"

// This form produces no executable code. Used as NO-OP.
var emptyForm = &sexp.FormList{}

func isBlankIdent(node ast.Expr) bool {
	if node, ok := node.(*ast.Ident); ok {
		return node.Name == blankIdent
	}
	return false
}

func (conv *Converter) AssignStmt(node *ast.AssignStmt) sexp.Form {
	switch node.Tok {
	case token.ADD_ASSIGN:
		return conv.addAssign(node.Lhs[0], node.Rhs[0])
	case token.SUB_ASSIGN:
		return conv.subAssign(node.Lhs[0], node.Rhs[0])
	case token.MUL_ASSIGN:
		return conv.mulAssign(node.Lhs[0], node.Rhs[0])
	case token.QUO_ASSIGN:
		return conv.quoAssign(node.Lhs[0], node.Rhs[0])

	default:
		if len(node.Rhs) == len(node.Lhs) {
			return conv.singleValueAssign(node)
		}
		return conv.multiValueAssign(node)
	}
}

func (conv *Converter) addAssign(lhs ast.Expr, rhs ast.Expr) sexp.Form {
	typ := conv.basicTypeOf(rhs)
	args := [2]sexp.Form{conv.Expr(lhs), conv.Expr(rhs)}

	if typ.Info()&types.IsNumeric != 0 {
		return conv.assign(lhs, &sexp.NumAdd{Typ: typ, Args: args})
	}
	if typ.Kind() == types.String {
		return conv.assign(lhs, &sexp.Concat{Args: args})
	}

	panic("unimplemented")
}

func (conv *Converter) subAssign(lhs ast.Expr, rhs ast.Expr) sexp.Form {
	return conv.assign(lhs, &sexp.NumSub{
		Typ:  conv.basicTypeOf(rhs),
		Args: [2]sexp.Form{conv.Expr(lhs), conv.Expr(rhs)},
	})
}

func (conv *Converter) mulAssign(lhs ast.Expr, rhs ast.Expr) sexp.Form {
	return conv.assign(lhs, &sexp.NumMul{
		Typ:  conv.basicTypeOf(rhs),
		Args: [2]sexp.Form{conv.Expr(lhs), conv.Expr(rhs)},
	})
}

func (conv *Converter) quoAssign(lhs ast.Expr, rhs ast.Expr) sexp.Form {
	return conv.assign(lhs, &sexp.NumQuo{
		Typ:  conv.basicTypeOf(rhs),
		Args: [2]sexp.Form{conv.Expr(lhs), conv.Expr(rhs)},
	})
}

func (conv *Converter) multiValueAssign(node *ast.AssignStmt) *sexp.FormList {
	forms := make([]sexp.Form, len(node.Lhs))

	tuple := conv.typeOf(node.Rhs[0]).(*types.Tuple)

	// First result is returned in a normal way.
	if isBlankIdent(node.Lhs[0]) {
		// Do a call, but discard the return value.
		forms[0] = sexp.CallStmt{
			Call: conv.Expr(node.Rhs[0]).(*sexp.Call),
		}
	} else {
		forms[0] = conv.assign(node.Lhs[0], conv.Expr(node.Rhs[0]))
	}

	// Other results are assigned to a global variable.
	// Index uniquely identifies variable used for storage.
	for i := 1; i < len(node.Lhs); i++ {
		forms[i] = conv.assign(node.Lhs[i], &sexp.MultiValueRef{
			Index: i,
			Typ:   tuple.At(i).Type(),
		})
	}

	return &sexp.FormList{Forms: forms}
}

func (conv *Converter) singleValueAssign(node *ast.AssignStmt) *sexp.FormList {
	forms := make([]sexp.Form, 0, 1)

	for i, lhs := range node.Lhs {
		conv.ctxType = conv.typeOf(lhs)
		rhs := conv.Expr(node.Rhs[i])
		forms = append(forms, conv.assign(lhs, rhs))
	}

	return &sexp.FormList{Forms: forms}
}

func (conv *Converter) assign(lhs ast.Expr, expr sexp.Form) sexp.Form {
	switch lhs := lhs.(type) {
	case *ast.Ident:
		if lhs.Name == blankIdent {
			return conv.ignoredExpr(expr)
		}
		if def := conv.info.Defs[lhs]; def == nil {
			return &sexp.Rebind{Name: lhs.Name, Expr: expr}
		}
		return &sexp.Bind{Name: lhs.Name, Init: conv.valueCopy(expr)}

	case *ast.IndexExpr:
		switch typ := conv.typeOf(lhs.X); typ.(type) {
		case *types.Map:
			call := &sexp.Call{
				Fn: function.MapInsert,
				Args: conv.valueCopyList([]sexp.Form{
					conv.Expr(lhs.Index),
					expr,
					conv.Expr(lhs.X),
				}),
			}
			return sexp.CallStmt{Call: call}

		case *types.Array:
			return &sexp.ArrayUpdate{
				Array: conv.Expr(lhs.X),
				Index: conv.Expr(lhs.Index),
				Expr:  expr,
			}

		default:
			panic("unimplemented")
		}

	// #TODO: struct assign, indirect assign
	default:
		panic("unimplemented")
	}
}

func (conv *Converter) ignoredExpr(expr sexp.Form) sexp.Form {
	switch expr := expr.(type) {
	case *sexp.Call:
		// Function call can not be ignored because
		// it may have side effects.
		return sexp.CallStmt{Call: expr}

	default:
		// Ignored completely.
		return emptyForm
	}
}
<|endoftext|>


<commit_before>package zoom

import (
	"errors"
	"fmt"
	"github.com/stephenalexbrowne/zoom/redis"
	"reflect"
)

// File contains glue code that connects the Model
// abstraction to the database. In other words,
// this is where the magic happens. The most important
// public-facing methods are here.

// writes the interface to the redis database
// throws an error if the type has not yet been
// registered. If in.Id is nil, will mutate in
// by setting the Id.
func Save(in ModelInterface) error {

	// make sure we'll dealing with a pointer
	typ := reflect.TypeOf(in)
	if typ.Kind() != reflect.Ptr {
		msg := fmt.Sprintf("zoom: Save() requires a pointer as an argument. The type %T is not a pointer.", in)
		return errors.New(msg)
	}

	// get the value
	val := reflect.ValueOf(in)
	if val.IsNil() {
		return errors.New("zoom: attempted to call save on a nil pointer!")
	}

	// get a connection from the pool
	conn := pool.Get()
	defer conn.Close()

	// get the struct spec
	ss := structSpecForType(typ.Elem())

	// get the registered name
	name, err := getRegisteredNameFromInterface(in)
	if err != nil {
		return err
	}

	// prepare the arguments for redis driver
	// if no id was provided, we should generate one
	if in.GetId() == "" {
		in.SetId(generateRandomId())
	}
	key := name + ":" + in.GetId()

	// start a multi/exec command. i.e. create a command queue
	if err := conn.Send("multi"); err != nil {
		// cancel transaction and return err
		conn.Do("discard")
		return err
	}

	// add command to queue
	if err := conn.Send("hmset", Args{}.Add(key).AddFlat(in)...); err != nil {
		// cancel transaction and return err
		conn.Do("discard")
		return err
	}

	// add to the index for this model
	err = queueAddToIndex(name, in.GetId(), conn)
	if err != nil {
		return err
	}

	// save the relations
	err = saveRelations(in, val, ss, name, conn)
	if err != nil {
		return err
	}

	// finally, commit the transaction
	// they were all writes, so the return value isn't needed
	if _, err := conn.Do("exec"); err != nil {
		return err
	}

	// add to the cache
	modelCache.Set(key, newCacheValue(in))

	return nil
}

// Removes the record from the database
func Delete(in ModelInterface) error {

	// get the registered name
	name, err := getRegisteredNameFromInterface(in)
	if err != nil {
		return err
	}

	// TODO: make sure it has an id

	return DeleteById(name, in.GetId())
}

// Delete a record from the interface by its id only
func DeleteById(modelName, id string) error {

	key := modelName + ":" + id

	// get a connection
	conn := pool.Get()
	defer conn.Close()

	// start a transaction
	conn.Send("multi")

	// add a command to the queue which will
	// delete the main key
	if err := conn.Send("del", key); err != nil {
		return err
	}

	// add a command to the queue which will
	// remove it from the index
	key = modelName + ":index"
	if err := conn.Send("srem", key, id); err != nil {
		return err
	}

	// execute the commands
	_, err := conn.Do("exec")
	if err != nil {
		return err
	}

	// remove from the cache
	modelCache.Delete(key)

	return nil
}

// Find a model by modelName and id. modelName must be the
// same name that was used in the Register() call
func FindById(modelName, id string) (interface{}, error) {
	// get the registered type
	typ, err := getRegisteredTypeFromName(modelName)
	if err != nil {
		return nil, err
	}

	// create the key based on the modelName and id
	key := modelName + ":" + id

	// check if the model is in the cache
	val, found := modelCache.Get(key)
	if found {
		cv, ok := val.(*cacheValue)
		if !ok {
			return nil, errors.New("zoom: Got from cache but couldn't convert to cacheValue")
		}
		return cv.value, nil
	}

	// open a connection
	conn := pool.Get()
	defer conn.Close()

	// make sure the key exists
	exists, err := KeyExists(key, conn)
	if err != nil {
		return nil, err
	}
	if !exists {
		msg := fmt.Sprintf("Couldn't find %s with id = %s", modelName, id)
		return nil, NewKeyNotFoundError(msg)
	}

	// create a new struct and instantiate its Model attribute
	// this gives us the embedded methods and properties on Model
	modelVal := reflect.New(typ.Elem())
	modelVal.Elem().FieldByName("Model").Set(reflect.ValueOf(new(Model)))

	// type assert to ModelInterface so we can use SetId()
	model := modelVal.Interface().(ModelInterface)

	// get the field values from redis
	reply, err := conn.Do("hgetall", key)
	bulk, err := redis.MultiBulk(reply, err)
	if err != nil {
		return nil, err
	}

	// fill in the values of the struct
	err = ScanStruct(bulk, model)
	if err != nil {
		return nil, err
	}

	// set the id
	model.SetId(id)

	// scan relations and add them as attributes to model
	ss := structSpecForType(typ.Elem())
	if err := scanRelations(ss, modelName, id, modelVal, conn); err != nil {
		return nil, err
	}

	// return it
	return model, nil
}

// ScanById is like FindById, but it will scan the results from the database
// into model, avoiding the need for typecasting after the find.
func ScanById(model ModelInterface, id string) error {

	// get the type and name
	typ := reflect.TypeOf(model)
	modelName, found := typeToName[typ]
	if !found {
		return NewModelTypeNotRegisteredError(typ)
	}

	// create the key based on the modelName and id
	key := modelName + ":" + id

	// check if the model is in the cache
	val, found := modelCache.Get(key)
	if found {
		cv, ok := val.(*cacheValue)
		if !ok {
			return errors.New("zoom: Got from cache but couldn't convert to cacheValue")
		}
		modelVal := reflect.ValueOf(model).Elem()
		modelVal.Set(reflect.ValueOf(cv.value).Elem())
		return nil
	}

	// open a connection
	conn := pool.Get()
	defer conn.Close()

	// make sure the key exists
	exists, err := KeyExists(key, conn)
	if err != nil {
		return err
	}
	if !exists {
		msg := fmt.Sprintf("Couldn't find %s with id = %s", modelName, id)
		return NewKeyNotFoundError(msg)
	}

	// get the stuff from redis
	reply, err := conn.Do("hgetall", key)
	bulk, err := redis.MultiBulk(reply, err)
	if err != nil {
		return err
	}

	// create a new struct and instantiate its Model attribute
	// this gives us the embedded methods and properties on Model
	modelVal := reflect.ValueOf(model)
	modelVal.Elem().FieldByName("Model").Set(reflect.ValueOf(new(Model)))

	// invoke redis driver to fill in the values of the struct
	err = ScanStruct(bulk, model)
	if err != nil {
		return err
	}

	// set the id
	model.SetId(id)

	// scan relations and add them as attributes to model
	ss := structSpecForType(typ.Elem())
	if err := scanRelations(ss, modelName, id, modelVal, conn); err != nil {
		return err
	}

	return nil
}

func FindAll(modelName string) ([]interface{}, error) {

	// get a connection
	conn := pool.Get()
	defer conn.Close()

	// invoke redis driver to get indexed keys and convert to []interface{}
	key := modelName + ":index"
	ids, err := redis.Strings(conn.Do("smembers", key))
	if err != nil {
		return nil, err
	}

	// iterate through each id. find the corresponding model. append to results.
	results := make([]interface{}, len(ids), len(ids))
	for i, id := range ids {
		m, err := FindById(modelName, id)
		if err != nil {
			return nil, err
		}
		results[i] = m
	}

	return results, nil
}
<commit_msg>Add to cache on Find operations. (Leaving it out would mean technically not following LRU)<commit_after>package zoom

import (
	"errors"
	"fmt"
	"github.com/stephenalexbrowne/zoom/redis"
	"reflect"
)

// File contains glue code that connects the Model
// abstraction to the database. In other words,
// this is where the magic happens. The most important
// public-facing methods are here.

// writes the interface to the redis database
// throws an error if the type has not yet been
// registered. If in.Id is nil, will mutate in
// by setting the Id.
func Save(in ModelInterface) error {

	// make sure we'll dealing with a pointer
	typ := reflect.TypeOf(in)
	if typ.Kind() != reflect.Ptr {
		msg := fmt.Sprintf("zoom: Save() requires a pointer as an argument. The type %T is not a pointer.", in)
		return errors.New(msg)
	}

	// get the value
	val := reflect.ValueOf(in)
	if val.IsNil() {
		return errors.New("zoom: attempted to call save on a nil pointer!")
	}

	// get a connection from the pool
	conn := pool.Get()
	defer conn.Close()

	// get the struct spec
	ss := structSpecForType(typ.Elem())

	// get the registered name
	name, err := getRegisteredNameFromInterface(in)
	if err != nil {
		return err
	}

	// prepare the arguments for redis driver
	// if no id was provided, we should generate one
	if in.GetId() == "" {
		in.SetId(generateRandomId())
	}
	key := name + ":" + in.GetId()

	// start a multi/exec command. i.e. create a command queue
	if err := conn.Send("multi"); err != nil {
		// cancel transaction and return err
		conn.Do("discard")
		return err
	}

	// add command to queue
	if err := conn.Send("hmset", Args{}.Add(key).AddFlat(in)...); err != nil {
		// cancel transaction and return err
		conn.Do("discard")
		return err
	}

	// add to the index for this model
	err = queueAddToIndex(name, in.GetId(), conn)
	if err != nil {
		return err
	}

	// save the relations
	err = saveRelations(in, val, ss, name, conn)
	if err != nil {
		return err
	}

	// finally, commit the transaction
	// they were all writes, so the return value isn't needed
	if _, err := conn.Do("exec"); err != nil {
		return err
	}

	// add to the cache
	modelCache.Set(key, newCacheValue(in))

	return nil
}

// Removes the record from the database
func Delete(in ModelInterface) error {

	// get the registered name
	name, err := getRegisteredNameFromInterface(in)
	if err != nil {
		return err
	}

	// TODO: make sure it has an id

	return DeleteById(name, in.GetId())
}

// Delete a record from the interface by its id only
func DeleteById(modelName, id string) error {

	key := modelName + ":" + id

	// get a connection
	conn := pool.Get()
	defer conn.Close()

	// start a transaction
	conn.Send("multi")

	// add a command to the queue which will
	// delete the main key
	if err := conn.Send("del", key); err != nil {
		return err
	}

	// add a command to the queue which will
	// remove it from the index
	key = modelName + ":index"
	if err := conn.Send("srem", key, id); err != nil {
		return err
	}

	// execute the commands
	_, err := conn.Do("exec")
	if err != nil {
		return err
	}

	// remove from the cache
	modelCache.Delete(key)

	return nil
}

// Find a model by modelName and id. modelName must be the
// same name that was used in the Register() call
func FindById(modelName, id string) (interface{}, error) {
	// get the registered type
	typ, err := getRegisteredTypeFromName(modelName)
	if err != nil {
		return nil, err
	}

	// create the key based on the modelName and id
	key := modelName + ":" + id

	// check if the model is in the cache
	val, found := modelCache.Get(key)
	if found {
		cv, ok := val.(*cacheValue)
		if !ok {
			return nil, errors.New("zoom: Got from cache but couldn't convert to cacheValue")
		}
		return cv.value, nil
	}

	// open a connection
	conn := pool.Get()
	defer conn.Close()

	// make sure the key exists
	exists, err := KeyExists(key, conn)
	if err != nil {
		return nil, err
	}
	if !exists {
		msg := fmt.Sprintf("Couldn't find %s with id = %s", modelName, id)
		return nil, NewKeyNotFoundError(msg)
	}

	// create a new struct and instantiate its Model attribute
	// this gives us the embedded methods and properties on Model
	modelVal := reflect.New(typ.Elem())
	modelVal.Elem().FieldByName("Model").Set(reflect.ValueOf(new(Model)))

	// type assert to ModelInterface so we can use SetId()
	model := modelVal.Interface().(ModelInterface)

	// get the field values from redis
	reply, err := conn.Do("hgetall", key)
	bulk, err := redis.MultiBulk(reply, err)
	if err != nil {
		return nil, err
	}

	// fill in the values of the struct
	err = ScanStruct(bulk, model)
	if err != nil {
		return nil, err
	}

	// set the id
	model.SetId(id)

	// scan relations and add them as attributes to model
	ss := structSpecForType(typ.Elem())
	if err := scanRelations(ss, modelName, id, modelVal, conn); err != nil {
		return nil, err
	}

	// add to the cache
	modelCache.SetIfAbsent(key, newCacheValue(model))

	// return it
	return model, nil
}

// ScanById is like FindById, but it will scan the results from the database
// into model, avoiding the need for typecasting after the find.
func ScanById(model ModelInterface, id string) error {

	// get the type and name
	typ := reflect.TypeOf(model)
	modelName, found := typeToName[typ]
	if !found {
		return NewModelTypeNotRegisteredError(typ)
	}

	// create the key based on the modelName and id
	key := modelName + ":" + id

	// check if the model is in the cache
	val, found := modelCache.Get(key)
	if found {
		cv, ok := val.(*cacheValue)
		if !ok {
			return errors.New("zoom: Got from cache but couldn't convert to cacheValue")
		}
		modelVal := reflect.ValueOf(model).Elem()
		modelVal.Set(reflect.ValueOf(cv.value).Elem())
		return nil
	}

	// open a connection
	conn := pool.Get()
	defer conn.Close()

	// make sure the key exists
	exists, err := KeyExists(key, conn)
	if err != nil {
		return err
	}
	if !exists {
		msg := fmt.Sprintf("Couldn't find %s with id = %s", modelName, id)
		return NewKeyNotFoundError(msg)
	}

	// get the stuff from redis
	reply, err := conn.Do("hgetall", key)
	bulk, err := redis.MultiBulk(reply, err)
	if err != nil {
		return err
	}

	// create a new struct and instantiate its Model attribute
	// this gives us the embedded methods and properties on Model
	modelVal := reflect.ValueOf(model)
	modelVal.Elem().FieldByName("Model").Set(reflect.ValueOf(new(Model)))

	// invoke redis driver to fill in the values of the struct
	err = ScanStruct(bulk, model)
	if err != nil {
		return err
	}

	// set the id
	model.SetId(id)

	// scan relations and add them as attributes to model
	ss := structSpecForType(typ.Elem())
	if err := scanRelations(ss, modelName, id, modelVal, conn); err != nil {
		return err
	}

	// add to the cache
	modelCache.SetIfAbsent(key, newCacheValue(model))

	return nil
}

func FindAll(modelName string) ([]interface{}, error) {

	// get a connection
	conn := pool.Get()
	defer conn.Close()

	// invoke redis driver to get indexed keys and convert to []interface{}
	key := modelName + ":index"
	ids, err := redis.Strings(conn.Do("smembers", key))
	if err != nil {
		return nil, err
	}

	// iterate through each id. find the corresponding model. append to results.
	results := make([]interface{}, len(ids), len(ids))
	for i, id := range ids {
		m, err := FindById(modelName, id)
		if err != nil {
			return nil, err
		}
		results[i] = m
	}

	return results, nil
}
<|endoftext|>


<commit_before>package ssh

import (
	"fmt"
	"time"
)

// debug printing utilities

var verbose bool = true

func p(format string, a ...interface{}) {
	if verbose {
		tsPrintf(format, a...)
	}
}

// quiet: discard text
func q(format string, a ...interface{}) {}

func pp(format string, a ...interface{}) {
	tsPrintf(format, a...)
}

// time-stamped printf
func tsPrintf(format string, a ...interface{}) {
	fmt.Printf("\n%s ", ts())
	fmt.Printf(format+"\n", a...)
}

// get timestamp for logging purposes
func ts() string {
	return time.Now().Format("2006-01-02 15:04:05.999 -0700 MST")
}
<commit_msg>timeouts green<commit_after>package ssh

import (
	"fmt"
	"time"
)

// debug printing utilities

var verbose bool = false

func p(format string, a ...interface{}) {
	if verbose {
		tsPrintf(format, a...)
	}
}

// quiet: discard text
func q(format string, a ...interface{}) {}

func pp(format string, a ...interface{}) {
	tsPrintf(format, a...)
}

// time-stamped printf
func tsPrintf(format string, a ...interface{}) {
	fmt.Printf("\n%s ", ts())
	fmt.Printf(format+"\n", a...)
}

// get timestamp for logging purposes
func ts() string {
	return time.Now().Format("2006-01-02 15:04:05.999 -0700 MST")
}
<|endoftext|>


<commit_before>package srnd

import (
	//"io"
	"fmt"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"sync"
	"time"
)

type FileCache struct {
	database Database
	store    ArticleStore

	webroot_dir string
	name        string

	regen_threads int
	attachments   bool

	prefix          string
	regenThreadChan chan ArticleEntry
	regenGroupChan  chan groupRegenRequest
	regenBoardMap   map[string]groupRegenRequest
	regenThreadMap  map[string]ArticleEntry

	regenBoardTicker  *time.Ticker
	ukkoTicker        *time.Ticker
	longTermTicker    *time.Ticker
	regenThreadTicker *time.Ticker

	regenThreadLock sync.RWMutex
	regenBoardLock  sync.RWMutex
}

func (self *FileCache) DeleteBoardMarkup(group string) {
	pages, _ := self.database.GetPagesPerBoard(group)
	for page := 0; page < pages; page++ {
		fname := self.getFilenameForBoardPage(group, page, false)
		os.Remove(fname)
		fname = self.getFilenameForBoardPage(group, page, true)
		os.Remove(fname)
	}
}

// try to delete root post's page
func (self *FileCache) DeleteThreadMarkup(root_post_id string) {
	fname := self.getFilenameForThread(root_post_id, false)
	os.Remove(fname)
	fname = self.getFilenameForThread(root_post_id, true)
	os.Remove(fname)
}

func (self *FileCache) getFilenameForThread(root_post_id string, json bool) string {
	var ext string
	if json {
		ext = "json"
	} else {
		ext = "html"
	}
	fname := fmt.Sprintf("thread-%s.%s", HashMessageID(root_post_id), ext)
	return filepath.Join(self.webroot_dir, fname)
}

func (self *FileCache) getFilenameForBoardPage(boardname string, pageno int, json bool) string {
	var ext string
	if json {
		ext = "json"
	} else {
		ext = "html"
	}
	fname := fmt.Sprintf("%s-%d.%s", boardname, pageno, ext)
	return filepath.Join(self.webroot_dir, fname)
}

// regen every newsgroup
func (self *FileCache) RegenAll() {
	log.Println("regen all on http frontend")

	// get all groups
	groups := self.database.GetAllNewsgroups()
	if groups != nil {
		for _, group := range groups {
			// send every thread for this group down the regen thread channel
			go self.database.GetGroupThreads(group, self.regenThreadChan)
			pages := self.database.GetGroupPageCount(group)
			var pg int64
			for pg = 0; pg < pages; pg++ {
				self.regenGroupChan <- groupRegenRequest{group, int(pg)}
			}
		}
	}
}

func (self *FileCache) regenLongTerm() {
	wr, err := os.Create(filepath.Join(self.webroot_dir, "history.html"))
	defer wr.Close()
	if err != nil {
		log.Println("cannot render history graph", err)
		return
	}
	template.genGraphs(self.prefix, wr, self.database)
}

func (self *FileCache) pollLongTerm() {
	for {
		<-self.longTermTicker.C
		// regenerate long term stuff
		self.regenLongTerm()
	}
}

func (self *FileCache) pollRegen() {
	for {
		select {
		// listen for regen board requests
		case req := <-self.regenGroupChan:
			self.regenBoardLock.Lock()
			self.regenBoardMap[fmt.Sprintf("%s|%s", req.group, req.page)] = req
			self.regenBoardLock.Unlock()
			// listen for regen thread requests
		case entry := <-self.regenThreadChan:
			self.regenThreadLock.Lock()
			self.regenThreadMap[fmt.Sprintf("%s|%s", entry[0], entry[1])] = entry
			self.regenThreadLock.Unlock()
			// regen ukko
		case _ = <-self.ukkoTicker.C:
			self.regenUkko()
			self.RegenFrontPage()
		case _ = <-self.regenThreadTicker.C:
			self.regenThreadLock.Lock()
			for _, entry := range self.regenThreadMap {
				self.regenerateThread(entry, false)
				self.regenerateThread(entry, true)
			}
			self.regenThreadMap = make(map[string]ArticleEntry)
			self.regenThreadLock.Unlock()
		case _ = <-self.regenBoardTicker.C:
			self.regenBoardLock.Lock()
			for _, v := range self.regenBoardMap {
				self.regenerateBoardPage(v.group, v.page, false)
				self.regenerateBoardPage(v.group, v.page, true)
			}
			self.regenBoardMap = make(map[string]groupRegenRequest)
			self.regenBoardLock.Unlock()
		}
	}
}

// regen every page of the board
func (self *FileCache) RegenerateBoard(group string) {
	pages := template.prepareGenBoard(self.attachments, self.prefix, self.name, group, self.database)
	for page := 0; page < pages; page++ {
		self.regenerateBoardPage(group, page, false)
		self.regenerateBoardPage(group, page, true)
	}
}

// regenerate just a thread page
func (self *FileCache) regenerateThread(root ArticleEntry, json bool) {
	msgid := root.MessageID()
	if self.store.HasArticle(msgid) {
		fname := self.getFilenameForThread(msgid, json)
		wr, err := os.Create(fname)
		defer wr.Close()
		if err != nil {
			log.Println("did not write", fname, err)
			return
		}
		template.genThread(self.attachments, root, self.prefix, self.name, wr, self.database, json)
	} else {
		log.Println("don't have root post", msgid, "not regenerating thread")
	}
}

// regenerate just a page on a board
func (self *FileCache) regenerateBoardPage(board string, page int, json bool) {
	fname := self.getFilenameForBoardPage(board, page, json)
	wr, err := os.Create(fname)
	defer wr.Close()
	if err != nil {
		log.Println("error generating board page", page, "for", board, err)
		return
	}
	template.genBoardPage(self.attachments, self.prefix, self.name, board, page, wr, self.database, json)
}

// regenerate the front page
func (self *FileCache) RegenFrontPage() {
	indexwr, err1 := os.Create(filepath.Join(self.webroot_dir, "index.html"))
	defer indexwr.Close()
	if err1 != nil {
		log.Println("cannot render front page", err1)
		return
	}
	boardswr, err2 := os.Create(filepath.Join(self.webroot_dir, "boards.html"))
	defer boardswr.Close()
	if err2 != nil {
		log.Println("cannot render board list page", err2)
		return
	}

	template.genFrontPage(10, self.prefix, self.name, indexwr, boardswr, self.database)
}

// regenerate the overboard
func (self *FileCache) regenUkko() {

	// markup
	fname := filepath.Join(self.webroot_dir, "ukko.html")
	wr, err := os.Create(fname)
	defer wr.Close()
	if err != nil {
		log.Println("error generating ukko markup", err)
		return
	}
	template.genUkko(self.prefix, self.name, wr, self.database, false)

	// json
	fname = filepath.Join(self.webroot_dir, "ukko.json")
	wr, err = os.Create(fname)
	defer wr.Close()
	if err != nil {
		log.Println("error generating ukko json", err)
		return
	}
	template.genUkko(self.prefix, self.name, wr, self.database, true)
}

// regenerate pages after a mod event
func (self *FileCache) RegenOnModEvent(newsgroup, msgid, root string, page int) {
	if root == msgid {
		fname := self.getFilenameForThread(root, false)
		os.Remove(fname)
		fname = self.getFilenameForThread(root, true)
		os.Remove(fname)
	} else {
		self.regenThreadChan <- ArticleEntry{root, newsgroup}
	}
	self.regenGroupChan <- groupRegenRequest{newsgroup, int(page)}
}

func (self *FileCache) Start() {
	threads := self.regen_threads

	// check for invalid number of threads
	if threads <= 0 {
		threads = 1
	}

	// use N threads for regeneration
	for threads > 0 {
		go self.pollRegen()
		threads--
	}
	// run long term regen jobs
	go self.pollLongTerm()
}

func (self *FileCache) Regen(msg ArticleEntry) {
	self.regenThreadChan <- msg
	self.RegenerateBoard(msg.Newsgroup())
}

func (self *FileCache) GetThreadChan() chan ArticleEntry {
	return self.regenThreadChan
}

func (self *FileCache) GetGroupChan() chan groupRegenRequest {
	return self.regenGroupChan
}

func (self *FileCache) GetHandler() http.Handler {
	return http.FileServer(http.Dir(self.webroot_dir))
}

func (self *FileCache) Close() {
	//nothig to do
}

func NewFileCache(prefix, webroot, name string, threads int, attachments bool, db Database, store ArticleStore) CacheInterface {
	cache := new(FileCache)
	cache.regenBoardTicker = time.NewTicker(time.Second * 10)
	cache.longTermTicker = time.NewTicker(time.Hour)
	cache.ukkoTicker = time.NewTicker(time.Second * 30)
	cache.regenThreadTicker = time.NewTicker(time.Second)
	cache.regenBoardMap = make(map[string]groupRegenRequest)
	cache.regenThreadMap = make(map[string]ArticleEntry)
	cache.regenThreadChan = make(chan ArticleEntry, 16)
	cache.regenGroupChan = make(chan groupRegenRequest, 8)

	cache.prefix = prefix
	cache.webroot_dir = webroot
	cache.name = name
	cache.regen_threads = threads
	cache.attachments = attachments
	cache.database = db
	cache.store = store

	return cache
}
<commit_msg>Add catalog to file cache<commit_after>package srnd

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"sync"
	"time"
)

type FileCache struct {
	database Database
	store    ArticleStore

	webroot_dir string
	name        string

	regen_threads int
	attachments   bool

	prefix          string
	regenThreadChan chan ArticleEntry
	regenGroupChan  chan groupRegenRequest
	regenBoardMap   map[string]groupRegenRequest
	regenThreadMap  map[string]ArticleEntry
	regenCatalogMap map[string]bool

	regenBoardTicker   *time.Ticker
	ukkoTicker         *time.Ticker
	longTermTicker     *time.Ticker
	regenThreadTicker  *time.Ticker
	regenCatalogTicker *time.Ticker

	regenThreadLock  sync.RWMutex
	regenBoardLock   sync.RWMutex
	regenCatalogLock sync.RWMutex
}

func (self *FileCache) DeleteBoardMarkup(group string) {
	pages, _ := self.database.GetPagesPerBoard(group)
	for page := 0; page < pages; page++ {
		fname := self.getFilenameForBoardPage(group, page, false)
		os.Remove(fname)
		fname = self.getFilenameForBoardPage(group, page, true)
		os.Remove(fname)
	}
}

// try to delete root post's page
func (self *FileCache) DeleteThreadMarkup(root_post_id string) {
	fname := self.getFilenameForThread(root_post_id, false)
	os.Remove(fname)
	fname = self.getFilenameForThread(root_post_id, true)
	os.Remove(fname)
}

func (self *FileCache) getFilenameForThread(root_post_id string, json bool) string {
	var ext string
	if json {
		ext = "json"
	} else {
		ext = "html"
	}
	fname := fmt.Sprintf("thread-%s.%s", HashMessageID(root_post_id), ext)
	return filepath.Join(self.webroot_dir, fname)
}

func (self *FileCache) getFilenameForBoardPage(boardname string, pageno int, json bool) string {
	var ext string
	if json {
		ext = "json"
	} else {
		ext = "html"
	}
	fname := fmt.Sprintf("%s-%d.%s", boardname, pageno, ext)
	return filepath.Join(self.webroot_dir, fname)
}

func (self *FileCache) getFilenameForCatalog(boardname string) string {
	fname := fmt.Sprintf("catalog-%s.html", boardname)
	return filepath.Join(self.webroot_dir, fname)
}

// regen every newsgroup
func (self *FileCache) RegenAll() {
	log.Println("regen all on http frontend")

	// get all groups
	groups := self.database.GetAllNewsgroups()
	if groups != nil {
		for _, group := range groups {
			// send every thread for this group down the regen thread channel
			go self.database.GetGroupThreads(group, self.regenThreadChan)
			pages := self.database.GetGroupPageCount(group)
			var pg int64
			for pg = 0; pg < pages; pg++ {
				self.regenGroupChan <- groupRegenRequest{group, int(pg)}
			}
		}
	}
}

func (self *FileCache) regenLongTerm() {
	wr, err := os.Create(filepath.Join(self.webroot_dir, "history.html"))
	defer wr.Close()
	if err != nil {
		log.Println("cannot render history graph", err)
		return
	}
	template.genGraphs(self.prefix, wr, self.database)
}

func (self *FileCache) pollLongTerm() {
	for {
		<-self.longTermTicker.C
		// regenerate long term stuff
		self.regenLongTerm()
	}
}

func (self *FileCache) pollRegen() {
	for {
		select {
		// listen for regen board requests
		case req := <-self.regenGroupChan:
			self.regenBoardLock.Lock()
			self.regenBoardMap[fmt.Sprintf("%s|%s", req.group, req.page)] = req
			self.regenBoardLock.Unlock()

			self.regenCatalogLock.Lock()
			self.regenCatalogMap[req.group] = true
			self.regenCatalogLock.Unlock()
			// listen for regen thread requests
		case entry := <-self.regenThreadChan:
			self.regenThreadLock.Lock()
			self.regenThreadMap[fmt.Sprintf("%s|%s", entry[0], entry[1])] = entry
			self.regenThreadLock.Unlock()
			// regen ukko
		case _ = <-self.ukkoTicker.C:
			self.regenUkko()
			self.RegenFrontPage()
		case _ = <-self.regenThreadTicker.C:
			self.regenThreadLock.Lock()
			for _, entry := range self.regenThreadMap {
				self.regenerateThread(entry, false)
				self.regenerateThread(entry, true)
			}
			self.regenThreadMap = make(map[string]ArticleEntry)
			self.regenThreadLock.Unlock()
		case _ = <-self.regenBoardTicker.C:
			self.regenBoardLock.Lock()
			for _, v := range self.regenBoardMap {
				self.regenerateBoardPage(v.group, v.page, false)
				self.regenerateBoardPage(v.group, v.page, true)
			}
			self.regenBoardMap = make(map[string]groupRegenRequest)
			self.regenBoardLock.Unlock()
		case _ = <-self.regenCatalogTicker.C:
			self.regenCatalogLock.Lock()
			for board, _ := range self.regenCatalogMap {
				self.regenerateCatalog(board)
			}
			self.regenCatalogMap = make(map[string]bool)
			self.regenCatalogLock.Unlock()
		}
	}
}

// regen every page of the board
func (self *FileCache) RegenerateBoard(group string) {
	pages := template.prepareGenBoard(self.attachments, self.prefix, self.name, group, self.database)
	for page := 0; page < pages; page++ {
		self.regenerateBoardPage(group, page, false)
		self.regenerateBoardPage(group, page, true)
	}
}

// regenerate just a thread page
func (self *FileCache) regenerateThread(root ArticleEntry, json bool) {
	msgid := root.MessageID()
	if self.store.HasArticle(msgid) {
		fname := self.getFilenameForThread(msgid, json)
		wr, err := os.Create(fname)
		defer wr.Close()
		if err != nil {
			log.Println("did not write", fname, err)
			return
		}
		template.genThread(self.attachments, root, self.prefix, self.name, wr, self.database, json)
	} else {
		log.Println("don't have root post", msgid, "not regenerating thread")
	}
}

// regenerate just a page on a board
func (self *FileCache) regenerateBoardPage(board string, page int, json bool) {
	fname := self.getFilenameForBoardPage(board, page, json)
	wr, err := os.Create(fname)
	defer wr.Close()
	if err != nil {
		log.Println("error generating board page", page, "for", board, err)
		return
	}
	template.genBoardPage(self.attachments, self.prefix, self.name, board, page, wr, self.database, json)
}

// regenerate the catalog for a board
func (self *FileCache) regenerateCatalog(board string) {
	fname := self.getFilenameForCatalog(board)
	wr, err := os.Create(fname)
	defer wr.Close()
	if err != nil {
		log.Println("error generating catalog for", board, err)
		return
	}
	template.genCatalog(self.prefix, self.name, board, wr, self.database)
}

// regenerate the front page
func (self *FileCache) RegenFrontPage() {
	indexwr, err1 := os.Create(filepath.Join(self.webroot_dir, "index.html"))
	defer indexwr.Close()
	if err1 != nil {
		log.Println("cannot render front page", err1)
		return
	}
	boardswr, err2 := os.Create(filepath.Join(self.webroot_dir, "boards.html"))
	defer boardswr.Close()
	if err2 != nil {
		log.Println("cannot render board list page", err2)
		return
	}

	template.genFrontPage(10, self.prefix, self.name, indexwr, boardswr, self.database)
}

// regenerate the overboard
func (self *FileCache) regenUkko() {

	// markup
	fname := filepath.Join(self.webroot_dir, "ukko.html")
	wr, err := os.Create(fname)
	defer wr.Close()
	if err != nil {
		log.Println("error generating ukko markup", err)
		return
	}
	template.genUkko(self.prefix, self.name, wr, self.database, false)

	// json
	fname = filepath.Join(self.webroot_dir, "ukko.json")
	wr, err = os.Create(fname)
	defer wr.Close()
	if err != nil {
		log.Println("error generating ukko json", err)
		return
	}
	template.genUkko(self.prefix, self.name, wr, self.database, true)
}

// regenerate pages after a mod event
func (self *FileCache) RegenOnModEvent(newsgroup, msgid, root string, page int) {
	if root == msgid {
		fname := self.getFilenameForThread(root, false)
		os.Remove(fname)
		fname = self.getFilenameForThread(root, true)
		os.Remove(fname)
	} else {
		self.regenThreadChan <- ArticleEntry{root, newsgroup}
	}
	self.regenGroupChan <- groupRegenRequest{newsgroup, int(page)}
}

func (self *FileCache) Start() {
	threads := self.regen_threads

	// check for invalid number of threads
	if threads <= 0 {
		threads = 1
	}

	// use N threads for regeneration
	for threads > 0 {
		go self.pollRegen()
		threads--
	}
	// run long term regen jobs
	go self.pollLongTerm()
}

func (self *FileCache) Regen(msg ArticleEntry) {
	self.regenThreadChan <- msg
	self.RegenerateBoard(msg.Newsgroup())
}

func (self *FileCache) GetThreadChan() chan ArticleEntry {
	return self.regenThreadChan
}

func (self *FileCache) GetGroupChan() chan groupRegenRequest {
	return self.regenGroupChan
}

func (self *FileCache) GetHandler() http.Handler {
	return http.FileServer(http.Dir(self.webroot_dir))
}

func (self *FileCache) Close() {
	//nothig to do
}

func NewFileCache(prefix, webroot, name string, threads int, attachments bool, db Database, store ArticleStore) CacheInterface {
	cache := new(FileCache)

	cache.regenBoardTicker = time.NewTicker(time.Second * 10)
	cache.longTermTicker = time.NewTicker(time.Hour)
	cache.ukkoTicker = time.NewTicker(time.Second * 30)
	cache.regenThreadTicker = time.NewTicker(time.Second)
	cache.regenCatalogTicker = time.NewTicker(time.Second * 20)

	cache.regenBoardMap = make(map[string]groupRegenRequest)
	cache.regenThreadMap = make(map[string]ArticleEntry)
	cache.regenCatalogMap = make(map[string]bool)

	cache.regenThreadChan = make(chan ArticleEntry, 16)
	cache.regenGroupChan = make(chan groupRegenRequest, 8)

	cache.prefix = prefix
	cache.webroot_dir = webroot
	cache.name = name
	cache.regen_threads = threads
	cache.attachments = attachments
	cache.database = db
	cache.store = store

	return cache
}
<|endoftext|>


<commit_before>package compute

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"strings"
)

// PublicIPBlock represents an allocated block of public IPv4 addresses.
type PublicIPBlock struct {
	ID              string `json:"id"`
	NetworkDomainID string `json:"networkDomainId"`
	DataCenterID    string `json:"datacenterId"`
	BaseIP          string `json:"baseIp"`
	Size            int    `json:"size"`
	CreateTime      string `json:"createTime"`
	State           string `json:"state"`
}

// GetID returns the public IPv4 address block's Id.
func (block *PublicIPBlock) GetID() string {
	return block.ID
}

// GetResourceType returns the public IP block's resource type.
func (block *PublicIPBlock) GetResourceType() ResourceType {
	return ResourceTypePublicIPBlock
}

// GetName returns the public IPv4 address block's name.
func (block *PublicIPBlock) GetName() string {
	return fmt.Sprintf("%s+%d", block.BaseIP, block.Size)
}

// GetState returns the network block's current state.
func (block *PublicIPBlock) GetState() string {
	return block.State
}

// IsDeleted determines whether the public IPv4 address block has been deleted (is nil).
func (block *PublicIPBlock) IsDeleted() bool {
	return block == nil
}

// ToEntityReference creates an EntityReference representing the CustomerImage.
func (block *PublicIPBlock) ToEntityReference() EntityReference {
	return EntityReference{
		ID: block.ID,
		Name: fmt.Sprintf("%s+%d",
			block.BaseIP,
			block.Size,
		),
	}
}

var _ Resource = &PublicIPBlock{}

// PublicIPBlocks represents a page of PublicIPBlock results.
type PublicIPBlocks struct {
	Blocks []PublicIPBlock `json:"publicIpBlock"`

	PagedResult
}

// ReservedPublicIP represents a public IPv4 address reserved for NAT or a VIP.
type ReservedPublicIP struct {
	IPBlockID       string `json:"ipBlockId"`
	DataCenterID    string `json:"datacenterId"`
	NetworkDomainID string `json:"networkDomainId"`
	Address         string `json:"value"`
}

// ReservedPublicIPs represents a page of ReservedPublicIP results.
type ReservedPublicIPs struct {
	IPs []ReservedPublicIP `json:"ip"`

	PagedResult
}

// Request body for adding a public IPv4 address block.
type addPublicAddressBlock struct {
	NetworkDomainID string `json:"networkDomainId"`
}

// Request body for removing a public IPv4 address block.
type removePublicAddressBlock struct {
	IPBlockID string `json:"id"`
}

// GetPublicIPBlock retrieves the public IPv4 address block with the specified Id.
// Returns nil if no IPv4 address block is found with the specified Id.
func (client *Client) GetPublicIPBlock(id string) (block *PublicIPBlock, err error) {
	organizationID, err := client.getOrganizationID()
	if err != nil {
		return nil, err
	}

	requestURI := fmt.Sprintf("%s/network/publicIpBlock/%s",
		url.QueryEscape(organizationID),
		url.QueryEscape(id),
	)
	request, err := client.newRequestV22(requestURI, http.MethodGet, nil)
	if err != nil {
		return nil, err
	}
	responseBody, statusCode, err := client.executeRequest(request)
	if err != nil {
		return nil, err
	}

	if statusCode != http.StatusOK {
		var apiResponse *APIResponseV2

		apiResponse, err = readAPIResponseAsJSON(responseBody, statusCode)
		if err != nil {
			return nil, err
		}

		if apiResponse.ResponseCode == ResponseCodeResourceNotFound {
			return nil, nil // Not an error, but was not found.
		}

		return nil, apiResponse.ToError("Request to retrieve public IPv4 address block failed with status code %d (%s): %s", statusCode, apiResponse.ResponseCode, apiResponse.Message)
	}

	block = &PublicIPBlock{}
	err = json.Unmarshal(responseBody, block)
	if err != nil {
		return nil, err
	}

	return block, nil
}

// ListPublicIPBlocks retrieves all blocks of public IPv4 addresses that have been allocated to the specified network domain.
func (client *Client) ListPublicIPBlocks(networkDomainID string, paging *Paging) (blocks *PublicIPBlocks, err error) {
	organizationID, err := client.getOrganizationID()
	if err != nil {
		return nil, err
	}

	requestURI := fmt.Sprintf("%s/network/publicIpBlock?networkDomainId=%s&%s",
		url.QueryEscape(organizationID),
		url.QueryEscape(networkDomainID),
		paging.EnsurePaging().toQueryParameters(),
	)
	request, err := client.newRequestV22(requestURI, http.MethodGet, nil)
	if err != nil {
		return nil, err
	}

	responseBody, statusCode, err := client.executeRequest(request)
	if err != nil {
		return nil, err
	}

	if statusCode != http.StatusOK {
		var apiResponse *APIResponseV2

		apiResponse, err = readAPIResponseAsJSON(responseBody, statusCode)
		if err != nil {
			return nil, err
		}

		return nil, apiResponse.ToError("Request to list public IPv4 address blocks failed with status code %d (%s): %s", statusCode, apiResponse.ResponseCode, apiResponse.Message)
	}

	blocks = &PublicIPBlocks{}
	err = json.Unmarshal(responseBody, blocks)

	return blocks, err
}

// AddPublicIPBlock adds a new block of public IPv4 addresses to the specified network domain.
func (client *Client) AddPublicIPBlock(networkDomainID string) (blockID string, err error) {
	organizationID, err := client.getOrganizationID()
	if err != nil {
		return "", err
	}

	requestURI := fmt.Sprintf("%s/network/addPublicIpBlock",
		url.QueryEscape(organizationID),
	)
	request, err := client.newRequestV22(requestURI, http.MethodPost,
		&addPublicAddressBlock{networkDomainID},
	)
	responseBody, statusCode, err := client.executeRequest(request)
	if err != nil {
		return "", err
	}

	apiResponse, err := readAPIResponseAsJSON(responseBody, statusCode)
	if err != nil {
		return "", err
	}

	if apiResponse.ResponseCode != ResponseCodeOK {
		return "", apiResponse.ToError("Request to add IPv4 address block to network domain '%s' failed with unexpected status code %d (%s): %s", networkDomainID, statusCode, apiResponse.ResponseCode, apiResponse.Message)
	}

	// Expected: "info" { "name": "ipBlockId", "value": "the-Id-of-the-new-IP-block" }
	ipBlockIDMessage := apiResponse.GetFieldMessage("ipBlockId")
	if ipBlockIDMessage == nil {
		return "", apiResponse.ToError("Received an unexpected response (missing 'ipBlockId') with status code %d (%s): %s", statusCode, apiResponse.ResponseCode, apiResponse.Message)
	}

	return *ipBlockIDMessage, nil
}

// RemovePublicIPBlock removes the specified block of public IPv4 addresses from its network domain.
// This operation is synchronous.
func (client *Client) RemovePublicIPBlock(id string) error {
	organizationID, err := client.getOrganizationID()
	if err != nil {
		return err
	}

	requestURI := fmt.Sprintf("%s/network/removePublicIpBlock",
		url.QueryEscape(organizationID),
	)
	request, err := client.newRequestV22(requestURI, http.MethodPost,
		&removePublicAddressBlock{id},
	)
	responseBody, statusCode, err := client.executeRequest(request)
	if err != nil {
		return err
	}

	apiResponse, err := readAPIResponseAsJSON(responseBody, statusCode)
	if err != nil {
		return err
	}

	if apiResponse.ResponseCode != ResponseCodeOK {
		return apiResponse.ToError("Request to remove IPv4 address block '%s' failed with unexpected status code %d (%s): %s", id, statusCode, apiResponse.ResponseCode, apiResponse.Message)
	}

	return nil
}

// ListReservedPublicIPAddresses retrieves all public IPv4 addresses in the specified network domain that have been reserved.
func (client *Client) ListReservedPublicIPAddresses(networkDomainID string, paging *Paging) (reservedPublicIPs *ReservedPublicIPs, err error) {
	organizationID, err := client.getOrganizationID()
	if err != nil {
		return nil, err
	}

	requestURI := fmt.Sprintf("%s/network/reservedPublicIpv4Address?networkDomainId=%s&%s",
		url.QueryEscape(organizationID),
		url.QueryEscape(networkDomainID),
		paging.EnsurePaging().toQueryParameters(),
	)
	request, err := client.newRequestV22(requestURI, http.MethodGet, nil)
	if err != nil {
		return nil, err
	}

	responseBody, statusCode, err := client.executeRequest(request)
	if err != nil {
		return nil, err
	}

	if statusCode != http.StatusOK {
		var apiResponse *APIResponseV2

		apiResponse, err = readAPIResponseAsJSON(responseBody, statusCode)
		if err != nil {
			return nil, err
		}

		return nil, apiResponse.ToError("Request to list reserved public IPv4 addresses failed with status code %d (%s): %s", statusCode, apiResponse.ResponseCode, apiResponse.Message)
	}

	reservedPublicIPs = &ReservedPublicIPs{}
	err = json.Unmarshal(responseBody, reservedPublicIPs)

	return reservedPublicIPs, err
}

// GetAvailablePublicIPAddresses retrieves all public IPv4 addresses in the specified network domain that are available for use.
//
// Returns a map of IP block IDs, keyed by public IP address.
func (client *Client) GetAvailablePublicIPAddresses(networkDomainID string) (availableIPs map[string]string, err error) {
	availableIPs = make(map[string]string)

	// Public IPs are allocated in blocks.
	page := DefaultPaging()
	for {
		var publicIPBlocks *PublicIPBlocks
		publicIPBlocks, err = client.ListPublicIPBlocks(networkDomainID, page)
		if err != nil {
			return
		}
		if publicIPBlocks.IsEmpty() {
			break // We're done
		}

		var blockAddresses []string
		for _, block := range publicIPBlocks.Blocks {
			blockAddresses, err = calculateBlockAddresses(block)
			if err != nil {
				return
			}

			for _, address := range blockAddresses {
				availableIPs[address] = block.ID
			}
		}

		page.Next()
	}

	// Some of those IPs may be reserved for other NAT rules or VIPs.
	page.First()
	for {
		var reservedIPs *ReservedPublicIPs
		reservedIPs, err = client.ListReservedPublicIPAddresses(networkDomainID, page)
		if err != nil {
			return
		}
		if reservedIPs.IsEmpty() {
			break // We're done
		}

		for _, reservedIP := range reservedIPs.IPs {
			delete(availableIPs, reservedIP.Address)
		}

		page.Next()
	}

	return
}

func calculateBlockAddresses(block PublicIPBlock) ([]string, error) {
	addresses := make([]string, block.Size)

	baseAddressComponents := strings.Split(block.BaseIP, ".")
	if len(baseAddressComponents) != 4 {
		return addresses, fmt.Errorf("invalid base IP address '%s'", block.BaseIP)
	}
	baseOctet, err := strconv.Atoi(baseAddressComponents[3])
	if err != nil {
		return addresses, err
	}

	for index := range addresses {
		// Increment the last octet to determine the next address in the block.
		baseAddressComponents[3] = strconv.Itoa(baseOctet + index)
		addresses[index] = strings.Join(baseAddressComponents, ".")
	}

	return addresses, nil
}
<commit_msg>update api version for ip block<commit_after>package compute

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"strings"
)

// PublicIPBlock represents an allocated block of public IPv4 addresses.
type PublicIPBlock struct {
	ID              string `json:"id"`
	NetworkDomainID string `json:"networkDomainId"`
	DataCenterID    string `json:"datacenterId"`
	BaseIP          string `json:"baseIp"`
	Size            int    `json:"size"`
	CreateTime      string `json:"createTime"`
	State           string `json:"state"`
}

// GetID returns the public IPv4 address block's Id.
func (block *PublicIPBlock) GetID() string {
	return block.ID
}

// GetResourceType returns the public IP block's resource type.
func (block *PublicIPBlock) GetResourceType() ResourceType {
	return ResourceTypePublicIPBlock
}

// GetName returns the public IPv4 address block's name.
func (block *PublicIPBlock) GetName() string {
	return fmt.Sprintf("%s+%d", block.BaseIP, block.Size)
}

// GetState returns the network block's current state.
func (block *PublicIPBlock) GetState() string {
	return block.State
}

// IsDeleted determines whether the public IPv4 address block has been deleted (is nil).
func (block *PublicIPBlock) IsDeleted() bool {
	return block == nil
}

// ToEntityReference creates an EntityReference representing the CustomerImage.
func (block *PublicIPBlock) ToEntityReference() EntityReference {
	return EntityReference{
		ID: block.ID,
		Name: fmt.Sprintf("%s+%d",
			block.BaseIP,
			block.Size,
		),
	}
}

var _ Resource = &PublicIPBlock{}

// PublicIPBlocks represents a page of PublicIPBlock results.
type PublicIPBlocks struct {
	Blocks []PublicIPBlock `json:"publicIpBlock"`

	PagedResult
}

// ReservedPublicIP represents a public IPv4 address reserved for NAT or a VIP.
type ReservedPublicIP struct {
	IPBlockID       string `json:"ipBlockId"`
	DataCenterID    string `json:"datacenterId"`
	NetworkDomainID string `json:"networkDomainId"`
	Address         string `json:"value"`
}

// ReservedPublicIPs represents a page of ReservedPublicIP results.
type ReservedPublicIPs struct {
	IPs []ReservedPublicIP `json:"ip"`

	PagedResult
}

// Request body for adding a public IPv4 address block.
type addPublicAddressBlock struct {
	NetworkDomainID string `json:"networkDomainId"`
}

// Request body for removing a public IPv4 address block.
type removePublicAddressBlock struct {
	IPBlockID string `json:"id"`
}

// GetPublicIPBlock retrieves the public IPv4 address block with the specified Id.
// Returns nil if no IPv4 address block is found with the specified Id.
func (client *Client) GetPublicIPBlock(id string) (block *PublicIPBlock, err error) {
	organizationID, err := client.getOrganizationID()
	if err != nil {
		return nil, err
	}

	requestURI := fmt.Sprintf("%s/network/publicIpBlock/%s",
		url.QueryEscape(organizationID),
		url.QueryEscape(id),
	)
	request, err := client.newRequestV213(requestURI, http.MethodGet, nil)
	if err != nil {
		return nil, err
	}
	responseBody, statusCode, err := client.executeRequest(request)
	if err != nil {
		return nil, err
	}

	if statusCode != http.StatusOK {
		var apiResponse *APIResponseV2

		apiResponse, err = readAPIResponseAsJSON(responseBody, statusCode)
		if err != nil {
			return nil, err
		}

		if apiResponse.ResponseCode == ResponseCodeResourceNotFound {
			return nil, nil // Not an error, but was not found.
		}

		return nil, apiResponse.ToError("Request to retrieve public IPv4 address block failed with status code %d (%s): %s", statusCode, apiResponse.ResponseCode, apiResponse.Message)
	}

	block = &PublicIPBlock{}
	err = json.Unmarshal(responseBody, block)
	if err != nil {
		return nil, err
	}

	return block, nil
}

// ListPublicIPBlocks retrieves all blocks of public IPv4 addresses that have been allocated to the specified network domain.
func (client *Client) ListPublicIPBlocks(networkDomainID string, paging *Paging) (blocks *PublicIPBlocks, err error) {
	organizationID, err := client.getOrganizationID()
	if err != nil {
		return nil, err
	}

	requestURI := fmt.Sprintf("%s/network/publicIpBlock?networkDomainId=%s&%s",
		url.QueryEscape(organizationID),
		url.QueryEscape(networkDomainID),
		paging.EnsurePaging().toQueryParameters(),
	)
	request, err := client.newRequestV213(requestURI, http.MethodGet, nil)
	if err != nil {
		return nil, err
	}

	responseBody, statusCode, err := client.executeRequest(request)
	if err != nil {
		return nil, err
	}

	if statusCode != http.StatusOK {
		var apiResponse *APIResponseV2

		apiResponse, err = readAPIResponseAsJSON(responseBody, statusCode)
		if err != nil {
			return nil, err
		}

		return nil, apiResponse.ToError("Request to list public IPv4 address blocks failed with status code %d (%s): %s", statusCode, apiResponse.ResponseCode, apiResponse.Message)
	}

	blocks = &PublicIPBlocks{}
	err = json.Unmarshal(responseBody, blocks)

	return blocks, err
}

// AddPublicIPBlock adds a new block of public IPv4 addresses to the specified network domain.
func (client *Client) AddPublicIPBlock(networkDomainID string) (blockID string, err error) {
	organizationID, err := client.getOrganizationID()
	if err != nil {
		return "", err
	}

	requestURI := fmt.Sprintf("%s/network/addPublicIpBlock",
		url.QueryEscape(organizationID),
	)
	request, err := client.newRequestV213(requestURI, http.MethodPost,
		&addPublicAddressBlock{networkDomainID},
	)
	responseBody, statusCode, err := client.executeRequest(request)
	if err != nil {
		return "", err
	}

	apiResponse, err := readAPIResponseAsJSON(responseBody, statusCode)
	if err != nil {
		return "", err
	}

	if apiResponse.ResponseCode != ResponseCodeOK {
		return "", apiResponse.ToError("Request to add IPv4 address block to network domain '%s' failed with unexpected status code %d (%s): %s", networkDomainID, statusCode, apiResponse.ResponseCode, apiResponse.Message)
	}

	// Expected: "info" { "name": "ipBlockId", "value": "the-Id-of-the-new-IP-block" }
	ipBlockIDMessage := apiResponse.GetFieldMessage("ipBlockId")
	if ipBlockIDMessage == nil {
		return "", apiResponse.ToError("Received an unexpected response (missing 'ipBlockId') with status code %d (%s): %s", statusCode, apiResponse.ResponseCode, apiResponse.Message)
	}

	return *ipBlockIDMessage, nil
}

// RemovePublicIPBlock removes the specified block of public IPv4 addresses from its network domain.
// This operation is synchronous.
func (client *Client) RemovePublicIPBlock(id string) error {
	organizationID, err := client.getOrganizationID()
	if err != nil {
		return err
	}

	requestURI := fmt.Sprintf("%s/network/removePublicIpBlock",
		url.QueryEscape(organizationID),
	)
	request, err := client.newRequestV213(requestURI, http.MethodPost,
		&removePublicAddressBlock{id},
	)
	responseBody, statusCode, err := client.executeRequest(request)
	if err != nil {
		return err
	}

	apiResponse, err := readAPIResponseAsJSON(responseBody, statusCode)
	if err != nil {
		return err
	}

	if apiResponse.ResponseCode != ResponseCodeOK {
		return apiResponse.ToError("Request to remove IPv4 address block '%s' failed with unexpected status code %d (%s): %s", id, statusCode, apiResponse.ResponseCode, apiResponse.Message)
	}

	return nil
}

// ListReservedPublicIPAddresses retrieves all public IPv4 addresses in the specified network domain that have been reserved.
func (client *Client) ListReservedPublicIPAddresses(networkDomainID string, paging *Paging) (reservedPublicIPs *ReservedPublicIPs, err error) {
	organizationID, err := client.getOrganizationID()
	if err != nil {
		return nil, err
	}

	requestURI := fmt.Sprintf("%s/network/reservedPublicIpv4Address?networkDomainId=%s&%s",
		url.QueryEscape(organizationID),
		url.QueryEscape(networkDomainID),
		paging.EnsurePaging().toQueryParameters(),
	)
	request, err := client.newRequestV213(requestURI, http.MethodGet, nil)
	if err != nil {
		return nil, err
	}

	responseBody, statusCode, err := client.executeRequest(request)
	if err != nil {
		return nil, err
	}

	if statusCode != http.StatusOK {
		var apiResponse *APIResponseV2

		apiResponse, err = readAPIResponseAsJSON(responseBody, statusCode)
		if err != nil {
			return nil, err
		}

		return nil, apiResponse.ToError("Request to list reserved public IPv4 addresses failed with status code %d (%s): %s", statusCode, apiResponse.ResponseCode, apiResponse.Message)
	}

	reservedPublicIPs = &ReservedPublicIPs{}
	err = json.Unmarshal(responseBody, reservedPublicIPs)

	return reservedPublicIPs, err
}

// GetAvailablePublicIPAddresses retrieves all public IPv4 addresses in the specified network domain that are available for use.
//
// Returns a map of IP block IDs, keyed by public IP address.
func (client *Client) GetAvailablePublicIPAddresses(networkDomainID string) (availableIPs map[string]string, err error) {
	availableIPs = make(map[string]string)

	// Public IPs are allocated in blocks.
	page := DefaultPaging()
	for {
		var publicIPBlocks *PublicIPBlocks
		publicIPBlocks, err = client.ListPublicIPBlocks(networkDomainID, page)
		if err != nil {
			return
		}
		if publicIPBlocks.IsEmpty() {
			break // We're done
		}

		var blockAddresses []string
		for _, block := range publicIPBlocks.Blocks {
			blockAddresses, err = calculateBlockAddresses(block)
			if err != nil {
				return
			}

			for _, address := range blockAddresses {
				availableIPs[address] = block.ID
			}
		}

		page.Next()
	}

	// Some of those IPs may be reserved for other NAT rules or VIPs.
	page.First()
	for {
		var reservedIPs *ReservedPublicIPs
		reservedIPs, err = client.ListReservedPublicIPAddresses(networkDomainID, page)
		if err != nil {
			return
		}
		if reservedIPs.IsEmpty() {
			break // We're done
		}

		for _, reservedIP := range reservedIPs.IPs {
			delete(availableIPs, reservedIP.Address)
		}

		page.Next()
	}

	return
}

func calculateBlockAddresses(block PublicIPBlock) ([]string, error) {
	addresses := make([]string, block.Size)

	baseAddressComponents := strings.Split(block.BaseIP, ".")
	if len(baseAddressComponents) != 4 {
		return addresses, fmt.Errorf("invalid base IP address '%s'", block.BaseIP)
	}
	baseOctet, err := strconv.Atoi(baseAddressComponents[3])
	if err != nil {
		return addresses, err
	}

	for index := range addresses {
		// Increment the last octet to determine the next address in the block.
		baseAddressComponents[3] = strconv.Itoa(baseOctet + index)
		addresses[index] = strings.Join(baseAddressComponents, ".")
	}

	return addresses, nil
}
<|endoftext|>


<commit_before>

package time

import "errors"

type Time struct {

    sec int64

    nsec int32

    loc *Location
}

// After reports whether the time instant t is after u.
func (t Time) After(u Time) bool {
    return t.sec > u.sec || t.sec == u.sec && t.nsec > u.nsec
}

// Before reports whether the time instant t is before u.
func (t Time) Before(u Time) bool {
    return t.sec < u.sec || t.sec == u.sec && t.nsec < u.nsec
}

func (t Time) Equal(u Time) bool {
    return t.sec == u.sec && t.nsec == u.nsec
}

type Month int

const (
    January Month = 1 + iota
    February
    March
    April
    May
    June
    July
    August
    September
    October
    November
    December
)

var months = [...]string{
    "January",
    "February",
    "March",
    "April",
    "May",
    "June",
    "July",
    "August",
    "September",
    "October",
    "November",
    "December",
}

// String returns the English name of the month ("January", "February", ...).
func (m Month) String() string { return months[m-1] }

// A Weekday specifies a day of the week (Sunday = 0, ...).
type Weekday int

const (
    Sunday Weekday = iota
    Monday
    Tuesday
    Wednesday
    Thursday
    Friday
    Saturday
)

var days = [...]string{
    "Sunday",
    "Monday",
    "Tuesday",
    "Wednesday",
    "Thursday",
    "Friday",
    "Saturday",
}

// String returns the English name of the day ("Sunday", "Monday", ...).
func (d Weekday) String() string { return days[d] }

func (t Time) IsZero() bool {
    return t.sec == 0 && t.nsec == 0
}
<commit_msg>Update time.go<commit_after>package time

import "errors"

type Time struct {

    sec int64

    nsec int32

    loc *Location
}

// Before reports whether the time instant t is before u.
func (t Time) Before(u Time) bool {
    return t.sec < u.sec || t.sec == u.sec && t.nsec < u.nsec
}

func (t Time) Equal(u Time) bool {
    return t.sec == u.sec && t.nsec == u.nsec
}

type Month int

const (
    January Month = 1 + iota
    February
    March
    April
    May
    June
    July
    August
    September
    October
    November
    December
)

var months = [...]string{
    "January",
    "February",
    "March",
    "April",
    "May",
    "June",
    "July",
    "August",
    "September",
    "October",
    "November",
    "December",
}

// String returns the English name of the month ("January", "February", ...).
func (m Month) String() string { return months[m-1] }

// A Weekday specifies a day of the week (Sunday = 0, ...).
type Weekday int

const (
    Sunday Weekday = iota
    Monday
    Tuesday
    Wednesday
    Thursday
    Friday
    Saturday
)

var days = [...]string{
    "Sunday",
    "Monday",
    "Tuesday",
    "Wednesday",
    "Thursday",
    "Friday",
    "Saturday",
}

// String returns the English name of the day ("Sunday", "Monday", ...).
func (d Weekday) String() string { return days[d] }

func (t Time) IsZero() bool {
    return t.sec == 0 && t.nsec == 0
}
<|endoftext|>


<commit_before>package readon

import (
	"code.google.com/p/go-html-transform/h5"
	"code.google.com/p/go-html-transform/html/transform"
	"code.google.com/p/go.net/html"
	"io"
	"math"
	"strings"
)

func Readon(reader io.Reader) (string, error) {
	tree, _ := h5.New(reader)
	t := transform.New(tree)
	removeScripts(t)
	removeUnlikely(t)
	removeCss(t)
	// TODO(pascalj): replace double BR with P
	removeBr(t)
	removeTags(t, []string{"form", "h1", "object", "iframe"})
	topTag := topCancidate(tree)
	return innerText(topTag), nil
}

func removeScripts(t *transform.Transformer) {
	t.Apply(transform.Replace(), "script")
	t.Apply(transform.Replace(), "noscript")
}

func removeCss(t *transform.Transformer) {
	t.Apply(transform.Replace(), "link")
	t.Apply(transform.ModifyAttrib("style", ""), "[style]")
}

func removeUnlikely(t *transform.Transformer) {
	unlikelyClasses := ".combx, .comment, .community, .disqus, .extra, .foot, .header, .menu, .remark, .rss, .shoutbox, .sidebar, .sponsor, .ad-break, .agegate, .pagination, .pager, .popup, .tweet, .twitter"
	unlikelyIds := "#combx, #comment, #community, #disqus, #extra, #foot, #header, #menu, #remark, #rss, #shoutbox, #sidebar, #sponsor, #ad-break, #agegate, #pagination, #pager, #popup, #tweet, #twitter"
	applyGroup(unlikelyClasses, func(sel string) { t.Apply(transform.Replace(), sel) })
	applyGroup(unlikelyIds, func(sel string) { t.Apply(transform.Replace(), sel) })
}

func removeBr(t *transform.Transformer) {
	t.Apply(transform.Replace(), "br")
}

func removeTags(t *transform.Transformer, tags []string) {
	for _, tag := range tags {
		t.Apply(transform.Replace(), tag)
	}
}

func topCancidate(tree *h5.Tree) *html.Node {
	ratings := rateDocument(tree.Top())
	var topCancidate *html.Node

	// Weight the links by linkDensity (less links is better) and get the top candidate
	for node, score := range ratings {
		if topCancidate == nil || score > ratings[topCancidate] {
			topCancidate = node
		}
	}

	return topCancidate
}

func rateDocument(node *html.Node) map[*html.Node]int {
	ratings := make(map[*html.Node]int)
	h5.WalkNodes(node, func(node *html.Node) {

		// Only consider paragraphs
		if node.Type != html.ElementNode || node.Data != "p" {
			return
		}
		score := rateNode(node)
		if node.Parent != nil {
			if _, containsNode := ratings[node.Parent]; !containsNode {
				ratings[node.Parent] = 0
			}
			ratings[node.Parent] += int(float32(score) * (1.0 - linkDensity(node.Parent)))
		}
		if node.Parent != nil && node.Parent.Parent != nil {
			if _, containsNode := ratings[node.Parent.Parent]; !containsNode {
				ratings[node.Parent.Parent] = 0
			}
			ratings[node.Parent.Parent] += int(float32(score)*(1.0-linkDensity(node.Parent.Parent))) / 2
		}
	})
	return ratings
}

func rateNode(node *html.Node) int {
	score := 1
	innerText := innerText(node)

	// Don't even consider short paragraphs
	if len(innerText) <= 25 {
		return 0
	}

	// Add a point for ervery comma
	score += strings.Count(innerText, ",")

	// Add up to three points for each 100 chars
	if hChars := int(math.Floor(float64(len(innerText)) / 100)); hChars <= 3 {
		score += hChars
	} else {
		score += 3
	}
	return score
}

func linkDensity(node *html.Node) float32 {
	var textLength, linkLength int
	textLength = len(innerText(node))
	h5.WalkNodes(node, func(node *html.Node) {
		if node.Type == html.ElementNode && node.Data == "a" {
			linkLength = linkLength + len(innerText(node))
		}
	})
	return float32(linkLength) / float32(textLength)
}

func innerText(node *html.Node) string {
	var content string
	h5.WalkNodes(node, func(node *html.Node) {
		if node.Type == html.TextNode {
			content = content + node.Data
		}
	})
	return content
}

func count(node *html.Node, tag string) int {
	var total int
	h5.WalkNodes(node, func(node *html.Node) {
		if node.Type == html.ElementNode && node.Data == tag {
			total = total + 1
		}
	})
	return total
}

func applyGroup(group string, applyFunc func(string)) {
	for _, sel := range strings.Split(group, ",") {
		applyFunc(strings.Trim(sel, " \t"))
	}
}
<commit_msg>Rename rateDocument<commit_after>package readon

import (
	"code.google.com/p/go-html-transform/h5"
	"code.google.com/p/go-html-transform/html/transform"
	"code.google.com/p/go.net/html"
	"io"
	"math"
	"strings"
)

func Readon(reader io.Reader) (string, error) {
	tree, _ := h5.New(reader)
	t := transform.New(tree)
	removeScripts(t)
	removeUnlikely(t)
	removeCss(t)
	// TODO(pascalj): replace double BR with P
	removeBr(t)
	removeTags(t, []string{"form", "h1", "object", "iframe"})
	topTag := topCancidate(tree)
	return innerText(topTag), nil
}

func removeScripts(t *transform.Transformer) {
	t.Apply(transform.Replace(), "script")
	t.Apply(transform.Replace(), "noscript")
}

func removeCss(t *transform.Transformer) {
	t.Apply(transform.Replace(), "link")
	t.Apply(transform.ModifyAttrib("style", ""), "[style]")
}

func removeUnlikely(t *transform.Transformer) {
	unlikelyClasses := ".combx, .comment, .community, .disqus, .extra, .foot, .header, .menu, .remark, .rss, .shoutbox, .sidebar, .sponsor, .ad-break, .agegate, .pagination, .pager, .popup, .tweet, .twitter"
	unlikelyIds := "#combx, #comment, #community, #disqus, #extra, #foot, #header, #menu, #remark, #rss, #shoutbox, #sidebar, #sponsor, #ad-break, #agegate, #pagination, #pager, #popup, #tweet, #twitter"
	applyGroup(unlikelyClasses, func(sel string) { t.Apply(transform.Replace(), sel) })
	applyGroup(unlikelyIds, func(sel string) { t.Apply(transform.Replace(), sel) })
}

func removeBr(t *transform.Transformer) {
	t.Apply(transform.Replace(), "br")
}

func removeTags(t *transform.Transformer, tags []string) {
	for _, tag := range tags {
		t.Apply(transform.Replace(), tag)
	}
}

func topCancidate(tree *h5.Tree) *html.Node {
	ratings := rateCancidates(tree.Top())
	var topCancidate *html.Node

	// Weight the links by linkDensity (less links is better) and get the top candidate
	for node, score := range ratings {
		if topCancidate == nil || score > ratings[topCancidate] {
			topCancidate = node
		}
	}

	return topCancidate
}

func rateCancidates(node *html.Node) map[*html.Node]int {
	ratings := make(map[*html.Node]int)
	h5.WalkNodes(node, func(node *html.Node) {

		// Only consider paragraphs
		if node.Type != html.ElementNode || node.Data != "p" {
			return
		}
		score := rateNode(node)
		if node.Parent != nil {
			if _, containsNode := ratings[node.Parent]; !containsNode {
				ratings[node.Parent] = 0
			}
			ratings[node.Parent] += int(float32(score) * (1.0 - linkDensity(node.Parent)))
		}
		if node.Parent != nil && node.Parent.Parent != nil {
			if _, containsNode := ratings[node.Parent.Parent]; !containsNode {
				ratings[node.Parent.Parent] = 0
			}
			ratings[node.Parent.Parent] += int(float32(score)*(1.0-linkDensity(node.Parent.Parent))) / 2
		}
	})
	return ratings
}

func rateNode(node *html.Node) int {
	score := 1
	innerText := innerText(node)

	// Don't even consider short paragraphs
	if len(innerText) <= 25 {
		return 0
	}

	// Add a point for ervery comma
	score += strings.Count(innerText, ",")

	// Add up to three points for each 100 chars
	if hChars := int(math.Floor(float64(len(innerText)) / 100)); hChars <= 3 {
		score += hChars
	} else {
		score += 3
	}
	return score
}

func linkDensity(node *html.Node) float32 {
	var textLength, linkLength int
	textLength = len(innerText(node))
	h5.WalkNodes(node, func(node *html.Node) {
		if node.Type == html.ElementNode && node.Data == "a" {
			linkLength = linkLength + len(innerText(node))
		}
	})
	return float32(linkLength) / float32(textLength)
}

func innerText(node *html.Node) string {
	var content string
	h5.WalkNodes(node, func(node *html.Node) {
		if node.Type == html.TextNode {
			content = content + node.Data
		}
	})
	return content
}

func count(node *html.Node, tag string) int {
	var total int
	h5.WalkNodes(node, func(node *html.Node) {
		if node.Type == html.ElementNode && node.Data == tag {
			total = total + 1
		}
	})
	return total
}

func applyGroup(group string, applyFunc func(string)) {
	for _, sel := range strings.Split(group, ",") {
		applyFunc(strings.Trim(sel, " \t"))
	}
}
<|endoftext|>


<commit_before>/* Copyright (c) 2014-2015, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import "html/template"

var tmpl *template.Template

func loadTemplates() {
	for name, s := range templates {
		var t *template.Template
		if tmpl == nil {
			tmpl = template.New(name)
		}
		if name == tmpl.Name() {
			t = tmpl
		} else {
			t = tmpl.New(name)
		}
		if _, err := t.Parse(s); err != nil {
			panic("could not load templates")
		}
	}
}

var templates = map[string]string{
	"/": `

Set up an alias:

    $ alias pcat='curl -F "{{.FieldName}}=<-" {{.SiteURL}}'

Upload a new paste:

    $ echo foo | pcat
    {{.SiteURL}}/a63d03b9

Fetch it:

    $ curl {{.SiteURL}}/a63d03b9
    foo

You can also use the web form.
{{if gt .MaxSize 0.0}}
Maximum size per paste is {{.MaxSize}}
{{end}}{{if gt .LifeTime 0}}
Pastes will be deleted after {{.LifeTime}}
{{end}}
github.com/mvdan/pastecat

`,
	"/form": `

	<form action="{{.SiteURL}}" method="post" enctype="multipart/form-data">
		<textarea cols=80 rows=24 name="{{.FieldName}}"></textarea>

		<button type="submit">Paste text</button>
	</form>

	<form action="{{.SiteURL}}" method="post" enctype="multipart/form-data">
		<input type="file" name="{{.FieldName}}"></input>

		<button type="submit">Paste file</button>
	</form>

`,
}
<commit_msg>Remove
 between the text and file forms<commit_after>/* Copyright (c) 2014-2015, Daniel Martí <mvdan@mvdan.cc> */
/* See LICENSE for licensing information */

package main

import "html/template"

var tmpl *template.Template

func loadTemplates() {
	for name, s := range templates {
		var t *template.Template
		if tmpl == nil {
			tmpl = template.New(name)
		}
		if name == tmpl.Name() {
			t = tmpl
		} else {
			t = tmpl.New(name)
		}
		if _, err := t.Parse(s); err != nil {
			panic("could not load templates")
		}
	}
}

var templates = map[string]string{
	"/": `

Set up an alias:

    $ alias pcat='curl -F "{{.FieldName}}=<-" {{.SiteURL}}'

Upload a new paste:

    $ echo foo | pcat
    {{.SiteURL}}/a63d03b9

Fetch it:

    $ curl {{.SiteURL}}/a63d03b9
    foo

You can also use the web form.
{{if gt .MaxSize 0.0}}
Maximum size per paste is {{.MaxSize}}
{{end}}{{if gt .LifeTime 0}}
Pastes will be deleted after {{.LifeTime}}
{{end}}
github.com/mvdan/pastecat

`,
	"/form": `

	<form action="{{.SiteURL}}" method="post" enctype="multipart/form-data">
		<textarea cols=80 rows=24 name="{{.FieldName}}"></textarea>

		<button type="submit">Paste text</button>
	</form>
	<form action="{{.SiteURL}}" method="post" enctype="multipart/form-data">
		<input type="file" name="{{.FieldName}}"></input>

		<button type="submit">Paste file</button>
	</form>

`,
}
<|endoftext|>


<commit_before>// Copyright (C) 2017 Damon Revoe. All rights reserved.
// Use of this source code is governed by the MIT
// license, which can be found in the LICENSE file.

package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"text/template"
)

type templateParams map[string]interface{}

type fileParams struct {
	filename string
	params   templateParams
}

type pathnameTemplateMultiplier struct {
	paramName    string
	paramValues  []string
	continuation pathnameTemplateText
}

type pathnameTemplateText struct {
	text string
	next *pathnameTemplateMultiplier
}

// Subst updates the 'pathnameTemplateText' receiver by replacing all
// instances of 'name' surrounded by braces with 'value', which can be
// either a string or a slice of strings.  In the latter case, the text
// in the receiver structure gets truncated by the substitution and the
// receiver structure gets extended by a new pathnameTemplateMultiplier
// structure.  Subst returns the number of substitution values.
func (t *pathnameTemplateText) subst(name string, value interface{}) int {
	if textValue, ok := value.(string); ok {
		t.text = strings.Replace(t.text, "{"+name+"}",
			textValue, -1)
	} else if arrayValue, ok := value.([]string); !ok {
		t.text = strings.Replace(t.text, "{"+name+"}",
			fmt.Sprint(value), -1)
	} else if pos := strings.Index(t.text, "{"+name+"}"); pos >= 0 {
		t.next = &pathnameTemplateMultiplier{name, arrayValue,
			pathnameTemplateText{t.text[pos+len(name)+2:], t.next}}
		t.text = t.text[:pos]
		return len(arrayValue)
	}

	return 1
}

// ExpandPathnameTemplate takes a pathname template and substitutes
// template parameter names with their values. Parameter values can be
// either strings or slices of strings. Each template value that is a
// slice of strings multiplies the number of output strings by the number
// of strings in the slice.
func expandPathnameTemplate(pathname string,
	params templateParams) []fileParams {
	root := pathnameTemplateText{pathname, nil}

	resultSize := 1

	for name, value := range params {
		resultSize *= root.subst(name, value)

		for n := root.next; n != nil; n = n.continuation.next {
			resultSize *= n.continuation.subst(name, value)
		}
	}

	result := make([]fileParams, resultSize)

	for i := 0; i < resultSize; i++ {
		result[i].filename = root.text
		copyOfParams := templateParams{}
		for name, value := range params {
			copyOfParams[name] = value
		}
		result[i].params = copyOfParams
	}

	if resultSize == 0 {
		return result
	}

	sliceSize := 1

	for a := root.next; a != nil; a = a.continuation.next {
		numberOfValues := len(a.paramValues)

		continuationText := a.continuation.text

		for i := 0; i < resultSize; {
			for j := 0; j < numberOfValues; j++ {
				value := a.paramValues[j]
				filenameFragment := value + continuationText
				for k := 0; k < sliceSize; k++ {
					result[i].filename += filenameFragment
					result[i].params[a.paramName] = value
					i++
				}
			}
		}

		sliceSize *= numberOfValues
	}

	return result
}

func must(err error) {
	if err != nil {
		panic(err)
	}
}

func varName(arg string) string {
	return strings.Map(func(r rune) rune {
		if r >= 'A' && r <= 'Z' || r >= 'a' && r <= 'z' ||
			r >= '0' && r <= '9' {
			return r
		} else if r == '+' {
			return 'x'
		}
		return '_'
	}, arg)
}

func varNameUC(arg string) string {
	return strings.Map(func(r rune) rune {
		if r >= 'a' && r <= 'z' {
			return r - 'a' + 'A'
		} else if r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' {
			return r
		} else if r == '+' {
			return 'X'
		}
		return '_'
	}, arg)
}

func libName(arg string) string {
	return strings.Map(func(r rune) rune {
		if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' ||
			r >= '0' && r <= '9' ||
			r == '+' || r == '-' || r == '.' {
			return r
		}
		return '_'
	}, arg)
}

var funcMap = template.FuncMap{
	"VarName":   varName,
	"VarNameUC": varNameUC,
	"LibName":   libName,
}

func generateFileFromTemplate(projectDir, templatePathname string,
	templateContents []byte, templateFileMode os.FileMode,
	params templateParams) error {
	// Parse the template file. The parsed template will be
	// reused multiple times if expandPathnameTemplate()
	// returns more than one pathname expansion.
	t, err := template.New(filepath.Base(templatePathname)).Funcs(
		funcMap).Parse(string(templateContents))
	if err != nil {
		return err
	}

	for _, fp := range expandPathnameTemplate(templatePathname, params) {

		projectFile := filepath.Join(projectDir, fp.filename)

		if err = os.MkdirAll(filepath.Dir(projectFile),
			os.ModePerm); err != nil {
			return err
		}

		buffer := bytes.NewBufferString("")

		if err = t.Execute(buffer, fp.params); err != nil {
			return err
		}

		newContents := buffer.Bytes()

		oldContents, err := ioutil.ReadFile(projectFile)

		if os.IsNotExist(err) {
			fmt.Println("A", projectFile)
		} else if bytes.Compare(oldContents, newContents) != 0 {
			fmt.Println("U", projectFile)
		} else {
			return nil
		}

		if err = ioutil.WriteFile(projectFile, newContents,
			templateFileMode); err != nil {
			return err
		}
	}

	return nil
}

type fileProcessor func(sourcePathname, relativePathname string,
	sourceFileInfo os.FileInfo) error

func getFileGenerationWalkFunc(sourceDir, targetDir string,
	processFile fileProcessor) filepath.WalkFunc {

	sourceDir = filepath.Clean(sourceDir) + string(filepath.Separator)

	return func(sourcePathname string, sourceFileInfo os.FileInfo,
		err error) error {
		if err != nil {
			return err
		}

		if sourceFileInfo.IsDir() {
			return nil
		}

		// Panic if filepath.Walk() does not behave as expected.
		if !strings.HasPrefix(sourcePathname, sourceDir) {
			panic(sourcePathname + " does not start with " +
				sourceDir)
		}

		// Relative pathname of the source file in the source
		// directory (and the target file in the target directory).
		relativePathname := sourcePathname[len(sourceDir):]

		// Ignore the package definition file.
		if relativePathname == packageDefinitionFilename {
			return nil
		}

		return processFile(sourcePathname, relativePathname,
			sourceFileInfo)
	}
}

// GetTemplateWalkFunc returns a walker function for use with filepath.Walk().
// The returned function interprets each file it visits as a 'text/template'
// file and generates a new file with the same relative pathname in the output
// directory 'projectDir'.
func getTemplateWalkFunc(templateDir, projectDir string,
	params templateParams) filepath.WalkFunc {

	templateDir = filepath.Clean(templateDir) + string(filepath.Separator)

	return func(templateFile string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if info.IsDir() {
			return nil
		}

		// Panic if filepath.Walk() does not behave as expected.
		if !strings.HasPrefix(templateFile, templateDir) {
			panic(templateFile + " does not start with " +
				templateDir)
		}

		// Read the contents of the template file. Cannot use
		// template.ParseFiles() because a Funcs() call must be
		// made between New() and Parse().
		templateContents, err := ioutil.ReadFile(templateFile)
		if err != nil {
			return err
		}

		// Pathname of the template file relative to the
		// template directory.
		templateFile = templateFile[len(templateDir):]

		// Ignore package definition file for the template.
		if templateFile == packageDefinitionFilename {
			return nil
		}

		if err = generateFileFromTemplate(projectDir, templateFile,
			templateContents, info.Mode(), params); err != nil {
			return err
		}

		return nil
	}
}

func linkFilesFromSourceDir(pd *packageDefinition, projectDir string) error {
	sourceDir := filepath.Dir(pd.pathname)

	linkFile := func(sourcePathname, relativePathname string,
		sourceFileInfo os.FileInfo) error {
		targetPathname := filepath.Join(projectDir, relativePathname)
		targetFileInfo, err := os.Lstat(targetPathname)
		if err == nil {
			if (targetFileInfo.Mode() & os.ModeSymlink) != 0 {
				originalLink, err := os.Readlink(targetPathname)

				if err != nil {
					return err
				}

				if originalLink == sourcePathname {
					return nil
				}
			}

			if err = os.Remove(targetPathname); err != nil {
				return err
			}
		}

		fmt.Println("L", targetPathname)

		if err = os.MkdirAll(filepath.Dir(targetPathname),
			os.ModePerm); err != nil {
			return err
		}

		return os.Symlink(sourcePathname, targetPathname)
	}

	return filepath.Walk(sourceDir,
		getFileGenerationWalkFunc(sourceDir, projectDir, linkFile))
}

// For each source file in 'templateDir', generateBuildFilesFromProjectTemplate
// generates an output file with the same relative pathname inside 'projectDir'.
func generateBuildFilesFromProjectTemplate(templateDir,
	projectDir string, pd *packageDefinition) error {
	if err := linkFilesFromSourceDir(pd, projectDir); err != nil {
		return err
	}

	if err := filepath.Walk(templateDir, getTemplateWalkFunc(templateDir,
		projectDir, pd.params)); err != nil {
		return err
	}

	return nil
}

// EmbeddedTemplateFile defines the file mode and the contents
// of a single file that is a part of an embedded project template.
type embeddedTemplateFile struct {
	mode     os.FileMode
	contents []byte
}

// EmbeddedTemplate defines a build-in project template.
type embeddedTemplate map[string]embeddedTemplateFile

// GenerateBuildFilesFromEmbeddedTemplate generates project build
// files from a built-in template pointed to by the 'template' parameter.
func generateBuildFilesFromEmbeddedTemplate(template *embeddedTemplate,
	projectDir string, pd *packageDefinition) error {
	if err := linkFilesFromSourceDir(pd, projectDir); err != nil {
		return err
	}

	for pathname, fileInfo := range *template {
		if err := generateFileFromTemplate(projectDir, pathname,
			fileInfo.contents, fileInfo.mode,
			pd.params); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>Remove code duplication<commit_after>// Copyright (C) 2017 Damon Revoe. All rights reserved.
// Use of this source code is governed by the MIT
// license, which can be found in the LICENSE file.

package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"text/template"
)

type templateParams map[string]interface{}

type fileParams struct {
	filename string
	params   templateParams
}

type pathnameTemplateMultiplier struct {
	paramName    string
	paramValues  []string
	continuation pathnameTemplateText
}

type pathnameTemplateText struct {
	text string
	next *pathnameTemplateMultiplier
}

// Subst updates the 'pathnameTemplateText' receiver by replacing all
// instances of 'name' surrounded by braces with 'value', which can be
// either a string or a slice of strings.  In the latter case, the text
// in the receiver structure gets truncated by the substitution and the
// receiver structure gets extended by a new pathnameTemplateMultiplier
// structure.  Subst returns the number of substitution values.
func (t *pathnameTemplateText) subst(name string, value interface{}) int {
	if textValue, ok := value.(string); ok {
		t.text = strings.Replace(t.text, "{"+name+"}",
			textValue, -1)
	} else if arrayValue, ok := value.([]string); !ok {
		t.text = strings.Replace(t.text, "{"+name+"}",
			fmt.Sprint(value), -1)
	} else if pos := strings.Index(t.text, "{"+name+"}"); pos >= 0 {
		t.next = &pathnameTemplateMultiplier{name, arrayValue,
			pathnameTemplateText{t.text[pos+len(name)+2:], t.next}}
		t.text = t.text[:pos]
		return len(arrayValue)
	}

	return 1
}

// ExpandPathnameTemplate takes a pathname template and substitutes
// template parameter names with their values. Parameter values can be
// either strings or slices of strings. Each template value that is a
// slice of strings multiplies the number of output strings by the number
// of strings in the slice.
func expandPathnameTemplate(pathname string,
	params templateParams) []fileParams {
	root := pathnameTemplateText{pathname, nil}

	resultSize := 1

	for name, value := range params {
		resultSize *= root.subst(name, value)

		for n := root.next; n != nil; n = n.continuation.next {
			resultSize *= n.continuation.subst(name, value)
		}
	}

	result := make([]fileParams, resultSize)

	for i := 0; i < resultSize; i++ {
		result[i].filename = root.text
		copyOfParams := templateParams{}
		for name, value := range params {
			copyOfParams[name] = value
		}
		result[i].params = copyOfParams
	}

	if resultSize == 0 {
		return result
	}

	sliceSize := 1

	for a := root.next; a != nil; a = a.continuation.next {
		numberOfValues := len(a.paramValues)

		continuationText := a.continuation.text

		for i := 0; i < resultSize; {
			for j := 0; j < numberOfValues; j++ {
				value := a.paramValues[j]
				filenameFragment := value + continuationText
				for k := 0; k < sliceSize; k++ {
					result[i].filename += filenameFragment
					result[i].params[a.paramName] = value
					i++
				}
			}
		}

		sliceSize *= numberOfValues
	}

	return result
}

func must(err error) {
	if err != nil {
		panic(err)
	}
}

func varName(arg string) string {
	return strings.Map(func(r rune) rune {
		if r >= 'A' && r <= 'Z' || r >= 'a' && r <= 'z' ||
			r >= '0' && r <= '9' {
			return r
		} else if r == '+' {
			return 'x'
		}
		return '_'
	}, arg)
}

func varNameUC(arg string) string {
	return strings.Map(func(r rune) rune {
		if r >= 'a' && r <= 'z' {
			return r - 'a' + 'A'
		} else if r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' {
			return r
		} else if r == '+' {
			return 'X'
		}
		return '_'
	}, arg)
}

func libName(arg string) string {
	return strings.Map(func(r rune) rune {
		if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' ||
			r >= '0' && r <= '9' ||
			r == '+' || r == '-' || r == '.' {
			return r
		}
		return '_'
	}, arg)
}

var funcMap = template.FuncMap{
	"VarName":   varName,
	"VarNameUC": varNameUC,
	"LibName":   libName,
}

func generateFileFromTemplate(projectDir, templatePathname string,
	templateContents []byte, templateFileMode os.FileMode,
	params templateParams) error {
	// Parse the template file. The parsed template will be
	// reused multiple times if expandPathnameTemplate()
	// returns more than one pathname expansion.
	t, err := template.New(filepath.Base(templatePathname)).Funcs(
		funcMap).Parse(string(templateContents))
	if err != nil {
		return err
	}

	for _, fp := range expandPathnameTemplate(templatePathname, params) {

		projectFile := filepath.Join(projectDir, fp.filename)

		if err = os.MkdirAll(filepath.Dir(projectFile),
			os.ModePerm); err != nil {
			return err
		}

		buffer := bytes.NewBufferString("")

		if err = t.Execute(buffer, fp.params); err != nil {
			return err
		}

		newContents := buffer.Bytes()

		oldContents, err := ioutil.ReadFile(projectFile)

		if os.IsNotExist(err) {
			fmt.Println("A", projectFile)
		} else if bytes.Compare(oldContents, newContents) != 0 {
			fmt.Println("U", projectFile)
		} else {
			return nil
		}

		if err = ioutil.WriteFile(projectFile, newContents,
			templateFileMode); err != nil {
			return err
		}
	}

	return nil
}

type fileProcessor func(sourcePathname, relativePathname string,
	sourceFileInfo os.FileInfo) error

// GetFileGenerationWalkFunc returns a walker function for use with
// filepath.Walk(). The walker function skips directories and package
// definition files.
func getFileGenerationWalkFunc(sourceDir, targetDir string,
	processFile fileProcessor) filepath.WalkFunc {

	sourceDir = filepath.Clean(sourceDir) + string(filepath.Separator)

	return func(sourcePathname string, sourceFileInfo os.FileInfo,
		err error) error {
		if err != nil {
			return err
		}

		if sourceFileInfo.IsDir() {
			return nil
		}

		// Panic if filepath.Walk() does not behave as expected.
		if !strings.HasPrefix(sourcePathname, sourceDir) {
			panic(sourcePathname + " does not start with " +
				sourceDir)
		}

		// Relative pathname of the source file in the source
		// directory (and the target file in the target directory).
		relativePathname := sourcePathname[len(sourceDir):]

		// Ignore the package definition file.
		if relativePathname == packageDefinitionFilename {
			return nil
		}

		return processFile(sourcePathname, relativePathname,
			sourceFileInfo)
	}
}

func linkFilesFromSourceDir(pd *packageDefinition, projectDir string) error {
	sourceDir := filepath.Dir(pd.pathname)

	linkFile := func(sourcePathname, relativePathname string,
		sourceFileInfo os.FileInfo) error {
		targetPathname := filepath.Join(projectDir, relativePathname)
		targetFileInfo, err := os.Lstat(targetPathname)
		if err == nil {
			if (targetFileInfo.Mode() & os.ModeSymlink) != 0 {
				originalLink, err := os.Readlink(targetPathname)

				if err != nil {
					return err
				}

				if originalLink == sourcePathname {
					return nil
				}
			}

			if err = os.Remove(targetPathname); err != nil {
				return err
			}
		}

		fmt.Println("L", targetPathname)

		if err = os.MkdirAll(filepath.Dir(targetPathname),
			os.ModePerm); err != nil {
			return err
		}

		return os.Symlink(sourcePathname, targetPathname)
	}

	return filepath.Walk(sourceDir,
		getFileGenerationWalkFunc(sourceDir, projectDir, linkFile))
}

// For each source file in 'templateDir', generateBuildFilesFromProjectTemplate
// generates an output file with the same relative pathname inside 'projectDir'.
func generateBuildFilesFromProjectTemplate(templateDir,
	projectDir string, pd *packageDefinition) error {
	if err := linkFilesFromSourceDir(pd, projectDir); err != nil {
		return err
	}

	generateFile := func(sourcePathname, relativePathname string,
		sourceFileInfo os.FileInfo) error {
		// Read the contents of the template file. Cannot use
		// template.ParseFiles() because a Funcs() call must be
		// made between New() and Parse().
		templateContents, err := ioutil.ReadFile(sourcePathname)
		if err != nil {
			return err
		}

		if err = generateFileFromTemplate(projectDir, relativePathname,
			templateContents, sourceFileInfo.Mode(),
			pd.params); err != nil {
			return err
		}

		return nil

	}

	return filepath.Walk(templateDir,
		getFileGenerationWalkFunc(templateDir,
			projectDir, generateFile))
}

// EmbeddedTemplateFile defines the file mode and the contents
// of a single file that is a part of an embedded project template.
type embeddedTemplateFile struct {
	mode     os.FileMode
	contents []byte
}

// EmbeddedTemplate defines a build-in project template.
type embeddedTemplate map[string]embeddedTemplateFile

// GenerateBuildFilesFromEmbeddedTemplate generates project build
// files from a built-in template pointed to by the 'template' parameter.
func generateBuildFilesFromEmbeddedTemplate(template *embeddedTemplate,
	projectDir string, pd *packageDefinition) error {
	if err := linkFilesFromSourceDir(pd, projectDir); err != nil {
		return err
	}

	for pathname, fileInfo := range *template {
		if err := generateFileFromTemplate(projectDir, pathname,
			fileInfo.contents, fileInfo.mode,
			pd.params); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package google

import (
	"fmt"
	"regexp"
	"strings"

	"github.com/hashicorp/terraform/helper/hashcode"
	"github.com/hashicorp/terraform/helper/schema"
)

// Compare only the relative path of two self links.
func compareSelfLinkRelativePaths(k, old, new string, d *schema.ResourceData) bool {
	oldStripped, err := getRelativePath(old)
	if err != nil {
		return false
	}

	newStripped, err := getRelativePath(new)
	if err != nil {
		return false
	}

	if oldStripped == newStripped {
		return true
	}

	return false
}

// Use this method when the field accepts either a name or a self_link referencing a resource.
// The value we store (i.e. `old` in this method), must be a self_link.
func compareSelfLinkOrResourceName(k, old, new string, d *schema.ResourceData) bool {
	newParts := strings.Split(new, "/")

	if len(newParts) == 1 {
		// `new` is a name
		// `old` is always a self_link
		if GetResourceNameFromSelfLink(old) == newParts[0] {
			return true
		}
	}

	// The `new` string is a self_link
	return compareSelfLinkRelativePaths(k, old, new, d)
}

// Hash the relative path of a self link.
func selfLinkRelativePathHash(selfLink interface{}) int {
	path, _ := getRelativePath(selfLink.(string))
	return hashcode.String(path)
}

func getRelativePath(selfLink string) (string, error) {
	stringParts := strings.SplitAfterN(selfLink, "projects/", 2)
	if len(stringParts) != 2 {
		return "", fmt.Errorf("String was not a self link: %s", selfLink)
	}

	return "projects/" + stringParts[1], nil
}

func ConvertSelfLinkToV1(link string) string {
	reg := regexp.MustCompile("/compute/[a-zA-Z0-9]*/projects/")
	return reg.ReplaceAllString(link, "/compute/v1/projects/")
}

func GetResourceNameFromSelfLink(link string) string {
	parts := strings.Split(link, "/")
	return parts[len(parts)-1]
}

func StoreResourceName(resourceLink interface{}) string {
	return GetResourceNameFromSelfLink(resourceLink.(string))
}
<commit_msg>StateFunc for GetResourceNameFromSelfLink (#1412)<commit_after>package google

import (
	"fmt"
	"regexp"
	"strings"

	"github.com/hashicorp/terraform/helper/hashcode"
	"github.com/hashicorp/terraform/helper/schema"
)

// Compare only the relative path of two self links.
func compareSelfLinkRelativePaths(k, old, new string, d *schema.ResourceData) bool {
	oldStripped, err := getRelativePath(old)
	if err != nil {
		return false
	}

	newStripped, err := getRelativePath(new)
	if err != nil {
		return false
	}

	if oldStripped == newStripped {
		return true
	}

	return false
}

// Use this method when the field accepts either a name or a self_link referencing a resource.
// The value we store (i.e. `old` in this method), must be a self_link.
func compareSelfLinkOrResourceName(k, old, new string, d *schema.ResourceData) bool {
	newParts := strings.Split(new, "/")

	if len(newParts) == 1 {
		// `new` is a name
		// `old` is always a self_link
		if GetResourceNameFromSelfLink(old) == newParts[0] {
			return true
		}
	}

	// The `new` string is a self_link
	return compareSelfLinkRelativePaths(k, old, new, d)
}

// Hash the relative path of a self link.
func selfLinkRelativePathHash(selfLink interface{}) int {
	path, _ := getRelativePath(selfLink.(string))
	return hashcode.String(path)
}

func getRelativePath(selfLink string) (string, error) {
	stringParts := strings.SplitAfterN(selfLink, "projects/", 2)
	if len(stringParts) != 2 {
		return "", fmt.Errorf("String was not a self link: %s", selfLink)
	}

	return "projects/" + stringParts[1], nil
}

func ConvertSelfLinkToV1(link string) string {
	reg := regexp.MustCompile("/compute/[a-zA-Z0-9]*/projects/")
	return reg.ReplaceAllString(link, "/compute/v1/projects/")
}

func GetResourceNameFromSelfLink(link string) string {
	parts := strings.Split(link, "/")
	return parts[len(parts)-1]
}

func NameFromSelfLinkStateFunc(v interface{}) string {
	return GetResourceNameFromSelfLink(v.(string))
}

func StoreResourceName(resourceLink interface{}) string {
	return GetResourceNameFromSelfLink(resourceLink.(string))
}
<|endoftext|>