<commit_before>package webserver

import (
	"context"
	"log"
	"net/http"
	"os"
	"os/signal"
	"time"

	"github.com/PaluMacil/dwn/configuration"
	"github.com/PaluMacil/dwn/database"
	coreapi "github.com/PaluMacil/dwn/module/core/api"
	oauthapi "github.com/PaluMacil/dwn/module/oauth/api"
	"github.com/PaluMacil/dwn/spa"
	"github.com/PaluMacil/dwn/webserver/handler"
	"github.com/gorilla/mux"
)

type WebServer struct {
	configuration.WebServerConfiguration
	mux *mux.Router
}

func New(db *database.Database, config configuration.Configuration) *WebServer {
	ws := &WebServer{
		WebServerConfiguration: config.WebServer,
		mux:                    mux.NewRouter(),
	}

	apiFactory := handler.Factory{
		Db:                    db,
		Config:                config,
		AssumeJSONContentType: true,
	}
	genericFactory := handler.Factory{
		Db:                    db,
		Config:                config,
		AssumeJSONContentType: false,
	}
	// TODO: refactor config and remove this check on the os args
	prod := len(os.Args) == 2 && os.Args[1] == "prod"

	// Set host subrouters
	var dwnHost *mux.Router
	if prod {
		dwnHost = ws.mux.Host("danwolf.net").Subrouter()
	} else {
		dwnHost = ws.mux.Host("localhost:" + ws.Port).Subrouter()
	}

	// Set module subrouters
	// ...core
	coreRouter := dwnHost.PathPrefix("/api/core/").Subrouter()
	coreapi.RegisterRoutes(coreRouter, apiFactory)
	// ...oauth
	oauthBaseRouter := dwnHost.PathPrefix("/oauth/").Subrouter()
	oauthapi.RegisterBaseRoutes(oauthBaseRouter, genericFactory)
	// ...content roots
	dwnHost.PathPrefix("/").Handler(spa.ContentRoot(config.WebServer.ContentRoot))

	ws.mux.Host("echo.danwolf.net").Subrouter()
	ws.mux.Host("echo-history.danwolf.net").Subrouter()
	return ws
}

func (ws *WebServer) Serve() {
	srv := &http.Server{
		Addr:    ":" + ws.Port,
		Handler: ws.mux,
		//time from when the connection is accepted to when the request body is fully read
		ReadTimeout: 5 * time.Second,
		//time from the end of the request header read to the end of the response write
		WriteTimeout: 10 * time.Second,
		IdleTimeout:  120 * time.Second,
	}
	stop := make(chan os.Signal)
	signal.Notify(stop, os.Interrupt)
	go func() {
		log.Println("Now serving on port", ws.Port)
		log.Println(srv.ListenAndServe())
	}()
	<-stop

	log.Printf("shutting down ...\n")

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()

	if err := srv.Shutdown(ctx); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>use either port or proxy port in dev<commit_after>package webserver

import (
	"context"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/signal"
	"time"

	"github.com/PaluMacil/dwn/configuration"
	"github.com/PaluMacil/dwn/database"
	coreapi "github.com/PaluMacil/dwn/module/core/api"
	oauthapi "github.com/PaluMacil/dwn/module/oauth/api"
	"github.com/PaluMacil/dwn/spa"
	"github.com/PaluMacil/dwn/webserver/handler"
	"github.com/gorilla/mux"
)

type WebServer struct {
	configuration.WebServerConfiguration
	mux *mux.Router
}

func New(db *database.Database, config configuration.Configuration) *WebServer {
	ws := &WebServer{
		WebServerConfiguration: config.WebServer,
		mux:                    mux.NewRouter(),
	}

	apiFactory := handler.Factory{
		Db:                    db,
		Config:                config,
		AssumeJSONContentType: true,
	}
	genericFactory := handler.Factory{
		Db:                    db,
		Config:                config,
		AssumeJSONContentType: false,
	}
	// TODO: refactor config and remove this check on the os args
	prod := len(os.Args) == 2 && os.Args[1] == "prod"

	// Set host subrouters
	var dwnHost *mux.Router
	if prod {
		dwnHost = ws.mux.Host("danwolf.net").Subrouter()
	} else {
		localhostMatchPattern := fmt.Sprintf("localhost:(?:%s|%s)", ws.Port, ws.UIProxyPort)
		dwnHost = ws.mux.Host(localhostMatchPattern).Subrouter()
	}

	// Set module subrouters
	// ...core
	coreRouter := dwnHost.PathPrefix("/api/core/").Subrouter()
	coreapi.RegisterRoutes(coreRouter, apiFactory)
	// ...oauth
	oauthBaseRouter := dwnHost.PathPrefix("/oauth/").Subrouter()
	oauthapi.RegisterBaseRoutes(oauthBaseRouter, genericFactory)
	// ...content roots
	dwnHost.PathPrefix("/").Handler(spa.ContentRoot(config.WebServer.ContentRoot))

	ws.mux.Host("echo.danwolf.net").Subrouter()
	ws.mux.Host("echo-history.danwolf.net").Subrouter()
	return ws
}

func (ws *WebServer) Serve() {
	srv := &http.Server{
		Addr:    ":" + ws.Port,
		Handler: ws.mux,
		//time from when the connection is accepted to when the request body is fully read
		ReadTimeout: 5 * time.Second,
		//time from the end of the request header read to the end of the response write
		WriteTimeout: 10 * time.Second,
		IdleTimeout:  120 * time.Second,
	}
	stop := make(chan os.Signal)
	signal.Notify(stop, os.Interrupt)
	go func() {
		log.Println("Now serving on port", ws.Port)
		log.Println(srv.ListenAndServe())
	}()
	<-stop

	log.Printf("shutting down ...\n")

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()

	if err := srv.Shutdown(ctx); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package downloader

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"time"
)

var (
	errNotModified  = errors.New("not modified")
	errNoETag       = errors.New("no etag")
	defaultInterval = time.Hour
)

// Downloader 用于下载IPIPNet提供的数据文件、检查和通知文件更新
type Downloader struct {
	// 必须的本地文件路径，对应ETag文件被保存在LocalPath+".etag"
	LocalPath string

	// 可选的远程下载地址，此字段为空时将监控本地文件修改，非空时监控远程文件ETag变化
	RemoteURL string

	// 监控周期
	Interval time.Duration

	// 是否检查远程下载中的ETag字段，若远程下载服务器出现异常导致缺少ETag字段，可将此字段设置为false以退化为周期性强制更新
	CheckETag bool

	// 错误事件回调函数，参数为错误对象
	ErrorCallback func(error)

	// 更新事件回调函数，参数为数据文件路径
	UpdateCallback func(string)

	etag     string
	watching bool
}

// EnsureLocal 用于在首次加载前确保本地文件存在
func (d *Downloader) EnsureLocal() error {
	if _, err := os.Stat(d.LocalPath); err == nil {
		if d.CheckETag {
			etag, err := ioutil.ReadFile(d.LocalPath + ".etag")
			if err != nil {
				return fmt.Errorf("load etag fail: %s", err.Error())
			}
			d.etag = string(etag)
		}
		return nil
	}
	if err := d.download(); err != nil {
		return fmt.Errorf("download fail: %s", err.Error())
	}
	return nil
}

// StartWatch 开始监控数据文件变化
func (d *Downloader) StartWatch() {
	d.watching = true
	if d.RemoteURL == "" {
		go d.watchLocal()
	} else {
		go d.watchRemote()
	}
}

// StopWatch 停止监控数据文件变化
func (d *Downloader) StopWatch() {
	d.watching = false
}

func (d *Downloader) watchLocal() {
	interval := d.Interval
	if interval == 0 {
		interval = defaultInterval
	}
	info, _ := os.Stat(d.LocalPath)
	ts := info.ModTime()
	time.Sleep(interval)
	for d.watching {
		info, err := os.Stat(d.LocalPath)
		if err != nil {
			d.onError(err)
		} else if info.ModTime().After(ts) {
			d.onUpdate()
		}
		time.Sleep(interval)
	}
}

func (d *Downloader) watchRemote() {
	interval := d.Interval
	if interval == 0 {
		interval = defaultInterval
	}
	for d.watching {
		if err := d.download(); err == errNotModified {
			// do nothing
		} else if err != nil {
			d.onError(err)
		} else {
			d.onUpdate()
		}
		time.Sleep(interval)
	}
}

func (d *Downloader) onError(err error) {
	if d.ErrorCallback != nil {
		d.ErrorCallback(err)
	}
}

func (d *Downloader) onUpdate() {
	if d.UpdateCallback != nil {
		d.UpdateCallback(d.LocalPath)
	}

}

func (d *Downloader) checkRemoteModification() (bool, error) {
	resp, err := http.Head(d.RemoteURL)
	if err != nil {
		return true, err
	}
	defer resp.Body.Close()
	etag := resp.Header.Get("ETag")
	if etag == "" {
		return true, errNoETag
	}
	return d.etag != etag, nil
}

func (d *Downloader) download() error {
	if d.RemoteURL == "" {
		return errors.New("remote url is unset")
	}
	// check remote modification first
	if d.CheckETag {
		if modified, err := d.checkRemoteModification(); err != nil {
			return fmt.Errorf("check remote modification fail:", err.Error())
		} else if !modified {
			return errNotModified
		}
	}
	// download remote content
	resp, err := http.Get(d.RemoteURL)
	if err != nil {
		return fmt.Errorf("download fail: %s", err.Error())
	}
	defer resp.Body.Close()
    content, err := ioutil.ReadAll(resp.Body)
    if err != nil {
        return fmt.Errorf("read response body fail: %s", err.Error())
    }
	if err := ioutil.WriteFile(d.LocalPath, content, 0755); err != nil {
		return fmt.Errorf("save local file fail: %s", err.Error())
	}
	if err := ioutil.WriteFile(d.LocalPath+".etag", []byte(resp.Header.Get("ETag")), 0755); err != nil {
		return fmt.Errorf("save local etag file fail: %s", err.Error())
	}
	d.etag = resp.Header.Get("ETag")
	return nil
}

<commit_msg>remove default interval, the interval <= 0 will disable StartWatch<commit_after>package downloader

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"time"
)

var (
	errNotModified = errors.New("not modified")
	errNoETag      = errors.New("no etag")
)

// Downloader 用于下载IPIPNet提供的数据文件、检查和通知文件更新
type Downloader struct {
	// 必须的本地文件路径，对应ETag文件被保存在LocalPath+".etag"
	LocalPath string

	// 可选的远程下载地址，此字段为空时将监控本地文件修改，非空时监控远程文件ETag变化
	RemoteURL string

	// 监控周期
	Interval time.Duration

	// 是否检查远程下载中的ETag字段，若远程下载服务器出现异常导致缺少ETag字段，可将此字段设置为false以退化为周期性强制更新
	CheckETag bool

	// 错误事件回调函数，参数为错误对象
	ErrorCallback func(error)

	// 更新事件回调函数，参数为数据文件路径
	UpdateCallback func(string)

	etag     string
	watching bool
}

// EnsureLocal 用于在首次加载前确保本地文件存在
func (d *Downloader) EnsureLocal() error {
	if _, err := os.Stat(d.LocalPath); err == nil {
		if d.CheckETag {
			etag, err := ioutil.ReadFile(d.LocalPath + ".etag")
			if err != nil {
				return fmt.Errorf("load etag fail: %s", err.Error())
			}
			d.etag = string(etag)
		}
		return nil
	}
	if err := d.download(); err != nil {
		return fmt.Errorf("download fail: %s", err.Error())
	}
	return nil
}

// StartWatch 开始监控数据文件变化
func (d *Downloader) StartWatch() {
	if d.Interval <= 0 {
		return
	}
	d.watching = true
	if d.RemoteURL == "" {
		go d.watchLocal()
	} else {
		go d.watchRemote()
	}
}

// StopWatch 停止监控数据文件变化
func (d *Downloader) StopWatch() {
	d.watching = false
}

func (d *Downloader) watchLocal() {
	info, _ := os.Stat(d.LocalPath)
	ts := info.ModTime()
	time.Sleep(d.Interval)
	for d.watching {
		info, err := os.Stat(d.LocalPath)
		if err != nil {
			d.onError(err)
		} else if info.ModTime().After(ts) {
			d.onUpdate()
		}
		time.Sleep(d.Interval)
	}
}

func (d *Downloader) watchRemote() {
	for d.watching {
		if err := d.download(); err == errNotModified {
			// do nothing
		} else if err != nil {
			d.onError(err)
		} else {
			d.onUpdate()
		}
		time.Sleep(d.Interval)
	}
}

func (d *Downloader) onError(err error) {
	if d.ErrorCallback != nil {
		d.ErrorCallback(err)
	}
}

func (d *Downloader) onUpdate() {
	if d.UpdateCallback != nil {
		d.UpdateCallback(d.LocalPath)
	}

}

func (d *Downloader) checkRemoteModification() (bool, error) {
	resp, err := http.Head(d.RemoteURL)
	if err != nil {
		return true, err
	}
	defer resp.Body.Close()
	etag := resp.Header.Get("ETag")
	if etag == "" {
		return true, errNoETag
	}
	return d.etag != etag, nil
}

func (d *Downloader) download() error {
	if d.RemoteURL == "" {
		return errors.New("remote url is unset")
	}
	// check remote modification first
	if d.CheckETag {
		if modified, err := d.checkRemoteModification(); err != nil {
			return fmt.Errorf("check remote modification fail:", err.Error())
		} else if !modified {
			return errNotModified
		}
	}
	// download remote content
	resp, err := http.Get(d.RemoteURL)
	if err != nil {
		return fmt.Errorf("download fail: %s", err.Error())
	}
	defer resp.Body.Close()
	content, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return fmt.Errorf("read response body fail: %s", err.Error())
	}
	if err := ioutil.WriteFile(d.LocalPath, content, 0755); err != nil {
		return fmt.Errorf("save local file fail: %s", err.Error())
	}
	if err := ioutil.WriteFile(d.LocalPath+".etag", []byte(resp.Header.Get("ETag")), 0755); err != nil {
		return fmt.Errorf("save local etag file fail: %s", err.Error())
	}
	d.etag = resp.Header.Get("ETag")
	return nil
}
<|endoftext|>


<commit_before>package jsonapi

import (
	"encoding/json"
	"errors"
	"fmt"
	"reflect"
	"strings"
)

// MarshalIdentifier interface is necessary to give an element
// a unique ID. This interface must be implemented for
// marshal and unmarshal in order to let them store
// elements
type MarshalIdentifier interface {
	GetID() string
}

// ReferenceID contains all necessary information in order
// to reference another struct in jsonapi
type ReferenceID struct {
	ID   string
	Type string
	Name string
}

// Reference information about possible references of a struct
// If IsNotLoaded is set to true, the `data` field will be omitted and only the `links` object will be
// generated. You should do this if there are some references, but you do not want to load them.
// Otherwise, if IsNotLoaded is false and GetReferencedIDs() returns no IDs for this reference name, an
// empty `data` field will be added which means that there are no references.
type Reference struct {
	Type        string
	Name        string
	IsNotLoaded bool
}

// MarshalReferences must be implemented if the struct to be serialized has relations. This must be done
// because jsonapi needs information about relations even if many to many relations or many to one relations
// are empty
type MarshalReferences interface {
	GetReferences() []Reference
}

// MarshalLinkedRelations must be implemented if there are references and the reference IDs should be included
type MarshalLinkedRelations interface {
	MarshalReferences
	MarshalIdentifier
	GetReferencedIDs() []ReferenceID
}

// MarshalIncludedRelations must be implemented if referenced structs should be included
type MarshalIncludedRelations interface {
	MarshalReferences
	MarshalIdentifier
	GetReferencedStructs() []MarshalIdentifier
}

// ServerInformation can be passed to MarshalWithURLs to generate the `self` and `related` urls inside `links`
type ServerInformation interface {
	GetBaseURL() string
	GetPrefix() string
}

var serverInformationNil ServerInformation

// MarshalWithURLs can be used to include the generation of `related` and `self` links
func MarshalWithURLs(data interface{}, information ServerInformation) ([]byte, error) {
	document, err := MarshalToStruct(data, information)
	if err != nil {
		return []byte(""), err
	}

	return json.Marshal(document)
}

// Marshal thats the input from `data` which can be a struct, a slice, or a pointer of it.
// Any struct in `data`or data itself, must at least implement the `MarshalIdentifier` interface.
// If so, it will generate a map[string]interface{} matching the jsonapi specification.
func Marshal(data interface{}) ([]byte, error) {
	document, err := MarshalToStruct(data, serverInformationNil)
	if err != nil {
		return []byte(""), err
	}
	return json.Marshal(document)
}

// MarshalToStruct marshals an api2go compatible struct into a jsonapi Document structure which then can be
// marshaled to JSON. You only need this method if you want to extract or extend parts of the document.
// You should directly use Marshal to get a []byte with JSON in it.
func MarshalToStruct(data interface{}, information ServerInformation) (Document, error) {
	if data == nil {
		return Document{}, nil
	}

	switch reflect.TypeOf(data).Kind() {
	case reflect.Slice:
		return marshalSlice(data, information)
	case reflect.Struct, reflect.Ptr:
		return marshalStruct(data.(MarshalIdentifier), information)
	default:
		return Document{}, errors.New("Marshal only accepts slice, struct or ptr types")
	}
}

func marshalSlice(data interface{}, information ServerInformation) (Document, error) {
	result := Document{}

	val := reflect.ValueOf(data)
	dataElements := []Data{}
	var referencedStructs []MarshalIdentifier

	for i := 0; i < val.Len(); i++ {
		k := val.Index(i).Interface()
		element, ok := k.(MarshalIdentifier)
		if !ok {
			return result, errors.New("all elements within the slice must implement api2go.MarshalIdentifier")
		}

		content, err := marshalData(element, information)
		if err != nil {
			return result, err
		}

		dataElements = append(dataElements, *content)

		included, ok := k.(MarshalIncludedRelations)
		if ok {
			referencedStructs = append(referencedStructs, included.GetReferencedStructs()...)
		}
	}

	includedElements := reduceDuplicates(referencedStructs, information, marshalData)

	//data key is always present
	result.Data = &DataContainer{
		DataArray: dataElements,
	}
	if includedElements != nil && len(*includedElements) > 0 {
		result.Included = *includedElements
	}

	return result, nil
}

// reduceDuplicates eliminates duplicate MarshalIdentifier from input and calls `method` on every unique MarshalIdentifier
func reduceDuplicates(
	input []MarshalIdentifier,
	information ServerInformation,
	method func(MarshalIdentifier, ServerInformation) (*Data, error),
) *[]Data {
	alreadyIncluded := map[string]map[string]bool{}
	includedElements := []Data{}

	for _, referencedStruct := range input {
		structType := getStructType(referencedStruct)
		if alreadyIncluded[structType] == nil {
			alreadyIncluded[structType] = make(map[string]bool)
		}

		if !alreadyIncluded[structType][referencedStruct.GetID()] {
			marshalled, _ := method(referencedStruct, information)
			includedElements = append(includedElements, *marshalled)
			alreadyIncluded[structType][referencedStruct.GetID()] = true
		}
	}

	return &includedElements
}

func marshalData(element MarshalIdentifier, information ServerInformation) (*Data, error) {
	var err error
	result := &Data{}

	refValue := reflect.ValueOf(element)
	if refValue.Kind() == reflect.Ptr && refValue.IsNil() {
		return result, errors.New("MarshalIdentifier must not be nil")
	}

	attributes, err := json.Marshal(element)
	result.Attributes = attributes
	result.ID = element.GetID()
	result.Type = getStructType(element)

	// optional relationship interface for struct
	references, ok := element.(MarshalLinkedRelations)
	if ok {
		result.Relationships = *getStructRelationships(references, information)
	}

	return result, err
}

// getStructRelationships returns the relationships struct with ids
func getStructRelationships(relationer MarshalLinkedRelations, information ServerInformation) *map[string]Relationship {
	referencedIDs := relationer.GetReferencedIDs()
	sortedResults := map[string][]ReferenceID{}
	relationships := map[string]Relationship{}

	for _, referenceID := range referencedIDs {
		sortedResults[referenceID.Name] = append(sortedResults[referenceID.Name], referenceID)
	}

	references := relationer.GetReferences()

	// helper mad to check if all references are included to also include mepty ones
	notIncludedReferences := map[string]Reference{}
	for _, reference := range references {
		notIncludedReferences[reference.Name] = reference
	}

	for name, referenceIDs := range sortedResults {
		relationships[name] = Relationship{}
		// if referenceType is plural, we need to use an array for data, otherwise it's just an object
		container := RelationshipDataContainer{}
		if Pluralize(name) == name {
			// multiple elements in links
			container.DataArray = []RelationshipData{}
			for _, referenceID := range referenceIDs {
				container.DataArray = append(container.DataArray, RelationshipData{
					Type: referenceID.Type,
					ID:   referenceID.ID,
				})
			}
		} else {
			container.DataObject = &RelationshipData{
				Type: referenceIDs[0].Type,
				ID:   referenceIDs[0].ID,
			}
		}

		// set URLs if necessary
		links := getLinksForServerInformation(relationer, name, information)

		relationship := Relationship{
			Data:  &container,
			Links: links,
		}

		relationships[name] = relationship

		// this marks the reference as already included
		delete(notIncludedReferences, referenceIDs[0].Name)
	}

	// check for empty references
	for name, reference := range notIncludedReferences {
		container := RelationshipDataContainer{}
		// Plural empty relationships need an empty array and empty to-one need a null in the json
		if !reference.IsNotLoaded && Pluralize(name) == name {
			container.DataArray = []RelationshipData{}
		}

		links := getLinksForServerInformation(relationer, name, information)
		relationship := Relationship{
			Links: links,
		}

		// skip relationship data completely if IsNotLoaded is set
		if !reference.IsNotLoaded {
			relationship.Data = &container
		}

		relationships[name] = relationship
	}

	return &relationships
}

// helper method to generate URL fields for `links`
func getLinksForServerInformation(relationer MarshalLinkedRelations, name string, information ServerInformation) *Links {
	links := &Links{}

	if information != serverInformationNil {
		prefix := strings.Trim(information.GetBaseURL(), "/")
		namespace := strings.Trim(information.GetPrefix(), "/")
		structType := getStructType(relationer)

		if namespace != "" {
			prefix += "/" + namespace
		}

		links.Self = fmt.Sprintf("%s/%s/%s/relationships/%s", prefix, structType, relationer.GetID(), name)
		links.Related = fmt.Sprintf("%s/%s/%s/%s", prefix, structType, relationer.GetID(), name)

		return links
	}

	return nil
}

func getIncludedStructs(included MarshalIncludedRelations, information ServerInformation) (*[]Data, error) {
	result := []Data{}
	includedStructs := included.GetReferencedStructs()

	for key := range includedStructs {
		marshalled, err := marshalData(includedStructs[key], information)
		if err != nil {
			return &result, err
		}

		result = append(result, *marshalled)
	}

	return &result, nil
}

func marshalStruct(data MarshalIdentifier, information ServerInformation) (Document, error) {
	fmt.Printf("marshalStructing ----------->", data)
	result := Document{}
	contentData, err := marshalData(data, information)
	if err != nil {
		return result, err
	}

	result.Data = &DataContainer{
		DataObject: contentData,
	}

	included, ok := data.(MarshalIncludedRelations)
	if ok {
		included, err := getIncludedStructs(included, information)
		if err != nil {
			return result, err
		}

		if len(*included) > 0 {
			result.Included = *included
		}
	}

	return result, nil
}

func getStructType(data interface{}) string {
	entityName, ok := data.(EntityNamer)
	if ok {
		return entityName.GetName()
	}

	reflectType := reflect.TypeOf(data)
	if reflectType.Kind() == reflect.Ptr {
		return Pluralize(Jsonify(reflectType.Elem().Name()))
	}

	return Pluralize(Jsonify(reflectType.Name()))
}
<commit_msg>remove printf<commit_after>package jsonapi

import (
	"encoding/json"
	"errors"
	"fmt"
	"reflect"
	"strings"
)

// MarshalIdentifier interface is necessary to give an element
// a unique ID. This interface must be implemented for
// marshal and unmarshal in order to let them store
// elements
type MarshalIdentifier interface {
	GetID() string
}

// ReferenceID contains all necessary information in order
// to reference another struct in jsonapi
type ReferenceID struct {
	ID   string
	Type string
	Name string
}

// Reference information about possible references of a struct
// If IsNotLoaded is set to true, the `data` field will be omitted and only the `links` object will be
// generated. You should do this if there are some references, but you do not want to load them.
// Otherwise, if IsNotLoaded is false and GetReferencedIDs() returns no IDs for this reference name, an
// empty `data` field will be added which means that there are no references.
type Reference struct {
	Type        string
	Name        string
	IsNotLoaded bool
}

// MarshalReferences must be implemented if the struct to be serialized has relations. This must be done
// because jsonapi needs information about relations even if many to many relations or many to one relations
// are empty
type MarshalReferences interface {
	GetReferences() []Reference
}

// MarshalLinkedRelations must be implemented if there are references and the reference IDs should be included
type MarshalLinkedRelations interface {
	MarshalReferences
	MarshalIdentifier
	GetReferencedIDs() []ReferenceID
}

// MarshalIncludedRelations must be implemented if referenced structs should be included
type MarshalIncludedRelations interface {
	MarshalReferences
	MarshalIdentifier
	GetReferencedStructs() []MarshalIdentifier
}

// ServerInformation can be passed to MarshalWithURLs to generate the `self` and `related` urls inside `links`
type ServerInformation interface {
	GetBaseURL() string
	GetPrefix() string
}

var serverInformationNil ServerInformation

// MarshalWithURLs can be used to include the generation of `related` and `self` links
func MarshalWithURLs(data interface{}, information ServerInformation) ([]byte, error) {
	document, err := MarshalToStruct(data, information)
	if err != nil {
		return []byte(""), err
	}

	return json.Marshal(document)
}

// Marshal thats the input from `data` which can be a struct, a slice, or a pointer of it.
// Any struct in `data`or data itself, must at least implement the `MarshalIdentifier` interface.
// If so, it will generate a map[string]interface{} matching the jsonapi specification.
func Marshal(data interface{}) ([]byte, error) {
	document, err := MarshalToStruct(data, serverInformationNil)
	if err != nil {
		return []byte(""), err
	}
	return json.Marshal(document)
}

// MarshalToStruct marshals an api2go compatible struct into a jsonapi Document structure which then can be
// marshaled to JSON. You only need this method if you want to extract or extend parts of the document.
// You should directly use Marshal to get a []byte with JSON in it.
func MarshalToStruct(data interface{}, information ServerInformation) (Document, error) {
	if data == nil {
		return Document{}, nil
	}

	switch reflect.TypeOf(data).Kind() {
	case reflect.Slice:
		return marshalSlice(data, information)
	case reflect.Struct, reflect.Ptr:
		return marshalStruct(data.(MarshalIdentifier), information)
	default:
		return Document{}, errors.New("Marshal only accepts slice, struct or ptr types")
	}
}

func marshalSlice(data interface{}, information ServerInformation) (Document, error) {
	result := Document{}

	val := reflect.ValueOf(data)
	dataElements := []Data{}
	var referencedStructs []MarshalIdentifier

	for i := 0; i < val.Len(); i++ {
		k := val.Index(i).Interface()
		element, ok := k.(MarshalIdentifier)
		if !ok {
			return result, errors.New("all elements within the slice must implement api2go.MarshalIdentifier")
		}

		content, err := marshalData(element, information)
		if err != nil {
			return result, err
		}

		dataElements = append(dataElements, *content)

		included, ok := k.(MarshalIncludedRelations)
		if ok {
			referencedStructs = append(referencedStructs, included.GetReferencedStructs()...)
		}
	}

	includedElements := reduceDuplicates(referencedStructs, information, marshalData)

	//data key is always present
	result.Data = &DataContainer{
		DataArray: dataElements,
	}
	if includedElements != nil && len(*includedElements) > 0 {
		result.Included = *includedElements
	}

	return result, nil
}

// reduceDuplicates eliminates duplicate MarshalIdentifier from input and calls `method` on every unique MarshalIdentifier
func reduceDuplicates(
	input []MarshalIdentifier,
	information ServerInformation,
	method func(MarshalIdentifier, ServerInformation) (*Data, error),
) *[]Data {
	alreadyIncluded := map[string]map[string]bool{}
	includedElements := []Data{}

	for _, referencedStruct := range input {
		structType := getStructType(referencedStruct)
		if alreadyIncluded[structType] == nil {
			alreadyIncluded[structType] = make(map[string]bool)
		}

		if !alreadyIncluded[structType][referencedStruct.GetID()] {
			marshalled, _ := method(referencedStruct, information)
			includedElements = append(includedElements, *marshalled)
			alreadyIncluded[structType][referencedStruct.GetID()] = true
		}
	}

	return &includedElements
}

func marshalData(element MarshalIdentifier, information ServerInformation) (*Data, error) {
	var err error
	result := &Data{}

	refValue := reflect.ValueOf(element)
	if refValue.Kind() == reflect.Ptr && refValue.IsNil() {
		return result, errors.New("MarshalIdentifier must not be nil")
	}

	attributes, err := json.Marshal(element)
	result.Attributes = attributes
	result.ID = element.GetID()
	result.Type = getStructType(element)

	// optional relationship interface for struct
	references, ok := element.(MarshalLinkedRelations)
	if ok {
		result.Relationships = *getStructRelationships(references, information)
	}

	return result, err
}

// getStructRelationships returns the relationships struct with ids
func getStructRelationships(relationer MarshalLinkedRelations, information ServerInformation) *map[string]Relationship {
	referencedIDs := relationer.GetReferencedIDs()
	sortedResults := map[string][]ReferenceID{}
	relationships := map[string]Relationship{}

	for _, referenceID := range referencedIDs {
		sortedResults[referenceID.Name] = append(sortedResults[referenceID.Name], referenceID)
	}

	references := relationer.GetReferences()

	// helper mad to check if all references are included to also include mepty ones
	notIncludedReferences := map[string]Reference{}
	for _, reference := range references {
		notIncludedReferences[reference.Name] = reference
	}

	for name, referenceIDs := range sortedResults {
		relationships[name] = Relationship{}
		// if referenceType is plural, we need to use an array for data, otherwise it's just an object
		container := RelationshipDataContainer{}
		if Pluralize(name) == name {
			// multiple elements in links
			container.DataArray = []RelationshipData{}
			for _, referenceID := range referenceIDs {
				container.DataArray = append(container.DataArray, RelationshipData{
					Type: referenceID.Type,
					ID:   referenceID.ID,
				})
			}
		} else {
			container.DataObject = &RelationshipData{
				Type: referenceIDs[0].Type,
				ID:   referenceIDs[0].ID,
			}
		}

		// set URLs if necessary
		links := getLinksForServerInformation(relationer, name, information)

		relationship := Relationship{
			Data:  &container,
			Links: links,
		}

		relationships[name] = relationship

		// this marks the reference as already included
		delete(notIncludedReferences, referenceIDs[0].Name)
	}

	// check for empty references
	for name, reference := range notIncludedReferences {
		container := RelationshipDataContainer{}
		// Plural empty relationships need an empty array and empty to-one need a null in the json
		if !reference.IsNotLoaded && Pluralize(name) == name {
			container.DataArray = []RelationshipData{}
		}

		links := getLinksForServerInformation(relationer, name, information)
		relationship := Relationship{
			Links: links,
		}

		// skip relationship data completely if IsNotLoaded is set
		if !reference.IsNotLoaded {
			relationship.Data = &container
		}

		relationships[name] = relationship
	}

	return &relationships
}

// helper method to generate URL fields for `links`
func getLinksForServerInformation(relationer MarshalLinkedRelations, name string, information ServerInformation) *Links {
	links := &Links{}

	if information != serverInformationNil {
		prefix := strings.Trim(information.GetBaseURL(), "/")
		namespace := strings.Trim(information.GetPrefix(), "/")
		structType := getStructType(relationer)

		if namespace != "" {
			prefix += "/" + namespace
		}

		links.Self = fmt.Sprintf("%s/%s/%s/relationships/%s", prefix, structType, relationer.GetID(), name)
		links.Related = fmt.Sprintf("%s/%s/%s/%s", prefix, structType, relationer.GetID(), name)

		return links
	}

	return nil
}

func getIncludedStructs(included MarshalIncludedRelations, information ServerInformation) (*[]Data, error) {
	result := []Data{}
	includedStructs := included.GetReferencedStructs()

	for key := range includedStructs {
		marshalled, err := marshalData(includedStructs[key], information)
		if err != nil {
			return &result, err
		}

		result = append(result, *marshalled)
	}

	return &result, nil
}

func marshalStruct(data MarshalIdentifier, information ServerInformation) (Document, error) {
	result := Document{}
	contentData, err := marshalData(data, information)
	if err != nil {
		return result, err
	}

	result.Data = &DataContainer{
		DataObject: contentData,
	}

	included, ok := data.(MarshalIncludedRelations)
	if ok {
		included, err := getIncludedStructs(included, information)
		if err != nil {
			return result, err
		}

		if len(*included) > 0 {
			result.Included = *included
		}
	}

	return result, nil
}

func getStructType(data interface{}) string {
	entityName, ok := data.(EntityNamer)
	if ok {
		return entityName.GetName()
	}

	reflectType := reflect.TypeOf(data)
	if reflectType.Kind() == reflect.Ptr {
		return Pluralize(Jsonify(reflectType.Elem().Name()))
	}

	return Pluralize(Jsonify(reflectType.Name()))
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// run runs the docs tests found in this directory.
package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
)

const usage = `go run run.go [tests]

run.go runs the docs tests in this directory.
If no tests are provided, it runs all tests.
Tests may be specified without their .go suffix.
`

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, usage)
		flag.PrintDefaults()
		os.Exit(2)
	}

	flag.Parse()
	if flag.NArg() == 0 {
		// run all tests
		fixcgo()
	} else {
		// run specified tests
		onlyTest(flag.Args()...)
	}

	tmpdir, err := ioutil.TempDir("", "go-progs")
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	// ratec limits the number of tests running concurrently.
	// None of the tests are intensive, so don't bother
	// trying to manually adjust for slow builders.
	ratec := make(chan bool, runtime.NumCPU())
	errc := make(chan error, len(tests))

	for _, tt := range tests {
		tt := tt
		ratec <- true
		go func() {
			errc <- test(tmpdir, tt.file, tt.want)
			<-ratec
		}()
	}

	var rc int
	for range tests {
		if err := <-errc; err != nil {
			fmt.Fprintln(os.Stderr, err)
			rc = 1
		}
	}
	os.Remove(tmpdir)
	os.Exit(rc)
}

// test builds the test in the given file.
// If want is non-empty, test also runs the test
// and checks that the output matches the regexp want.
func test(tmpdir, file, want string) error {
	// Build the program.
	prog := filepath.Join(tmpdir, file)
	cmd := exec.Command("go", "build", "-o", prog, file+".go")
	out, err := cmd.CombinedOutput()
	if err != nil {
		return fmt.Errorf("go build %s.go failed: %v\nOutput:\n%s", file, err, out)
	}
	defer os.Remove(prog)

	// Only run the test if we have output to check.
	if want == "" {
		return nil
	}

	cmd = exec.Command(prog)
	out, err = cmd.CombinedOutput()
	if err != nil {
		return fmt.Errorf("%s failed: %v\nOutput:\n%s", file, err, out)
	}

	// Canonicalize output.
	out = bytes.TrimRight(out, "\n")
	out = bytes.Replace(out, []byte{'\n'}, []byte{' '}, -1)

	// Check the result.
	match, err := regexp.Match(want, out)
	if err != nil {
		return fmt.Errorf("failed to parse regexp %q: %v", want, err)
	}
	if !match {
		return fmt.Errorf("%s.go:\n%q\ndoes not match %s", file, out, want)
	}

	return nil
}

type testcase struct {
	file string
	want string
}

var tests = []testcase{
	// defer_panic_recover
	{"defer", `^0 3210 2$`},
	{"defer2", `^Calling g. Printing in g 0 Printing in g 1 Printing in g 2 Printing in g 3 Panicking! Defer in g 3 Defer in g 2 Defer in g 1 Defer in g 0 Recovered in f 4 Returned normally from f.$`},

	// effective_go
	{"eff_bytesize", `^1.00YB 9.09TB$`},
	{"eff_qr", ""},
	{"eff_sequence", `^\[-1 2 6 16 44\]$`},
	{"eff_unused2", ""},

	// error_handling
	{"error", ""},
	{"error2", ""},
	{"error3", ""},
	{"error4", ""},

	// law_of_reflection
	{"interface", ""},
	{"interface2", `^type: float64$`},

	// c_go_cgo
	{"cgo1", ""},
	{"cgo2", ""},
	{"cgo3", ""},
	{"cgo4", ""},

	// timeout
	{"timeout1", ""},
	{"timeout2", ""},

	// gobs
	{"gobs1", ""},
	{"gobs2", ""},

	// json
	{"json1", `^$`},
	{"json2", `the reciprocal of i is`},
	{"json3", `Age is int 6`},
	{"json4", `^$`},
	{"json5", ""},

	// image_package
	{"image_package1", `^X is 2 Y is 1$`},
	{"image_package2", `^3 4 false$`},
	{"image_package3", `^3 4 true$`},
	{"image_package4", `^image.Point{X:2, Y:1}$`},
	{"image_package5", `^{255 0 0 255}$`},
	{"image_package6", `^8 4 true$`},

	// other
	{"go1", `^Christmas is a holiday: true .*go1.go already exists$`},
	{"slices", ""},
}

func onlyTest(files ...string) {
	var new []testcase
NextFile:
	for _, file := range files {
		file = strings.TrimSuffix(file, ".go")
		for _, tt := range tests {
			if tt.file == file {
				new = append(new, tt)
				continue NextFile
			}
		}
		fmt.Fprintf(os.Stderr, "test %s.go not found\n", file)
		os.Exit(1)
	}
	tests = new
}

func skipTest(file string) {
	for i, tt := range tests {
		if tt.file == file {
			copy(tests[i:], tests[i+1:])
			tests = tests[:len(tests)-1]
			return
		}
	}
	panic("delete(" + file + "): not found")
}

func fixcgo() {
	if os.Getenv("CGO_ENABLED") != "1" {
		skipTest("cgo1")
		skipTest("cgo2")
		skipTest("cgo3")
		skipTest("cgo4")
		return
	}

	switch runtime.GOOS {
	case "freebsd":
		// cgo1 and cgo2 don't run on freebsd, srandom has a different signature
		skipTest("cgo1")
		skipTest("cgo2")
	case "netbsd":
		// cgo1 and cgo2 don't run on netbsd, srandom has a different signature
		skipTest("cgo1")
		skipTest("cgo2")
		// cgo3 and cgo4 don't run on netbsd, since cgo cannot handle stdout correctly
		skipTest("cgo3")
		skipTest("cgo4")
	case "openbsd":
		// cgo3 and cgo4 don't run on openbsd and solaris, since cgo cannot handle stdout correctly
		skipTest("cgo3")
		skipTest("cgo4")
	}
}
<commit_msg>doc/progs: disable cgo tests that use C.Stdout on Solaris<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// run runs the docs tests found in this directory.
package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
)

const usage = `go run run.go [tests]

run.go runs the docs tests in this directory.
If no tests are provided, it runs all tests.
Tests may be specified without their .go suffix.
`

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, usage)
		flag.PrintDefaults()
		os.Exit(2)
	}

	flag.Parse()
	if flag.NArg() == 0 {
		// run all tests
		fixcgo()
	} else {
		// run specified tests
		onlyTest(flag.Args()...)
	}

	tmpdir, err := ioutil.TempDir("", "go-progs")
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	// ratec limits the number of tests running concurrently.
	// None of the tests are intensive, so don't bother
	// trying to manually adjust for slow builders.
	ratec := make(chan bool, runtime.NumCPU())
	errc := make(chan error, len(tests))

	for _, tt := range tests {
		tt := tt
		ratec <- true
		go func() {
			errc <- test(tmpdir, tt.file, tt.want)
			<-ratec
		}()
	}

	var rc int
	for range tests {
		if err := <-errc; err != nil {
			fmt.Fprintln(os.Stderr, err)
			rc = 1
		}
	}
	os.Remove(tmpdir)
	os.Exit(rc)
}

// test builds the test in the given file.
// If want is non-empty, test also runs the test
// and checks that the output matches the regexp want.
func test(tmpdir, file, want string) error {
	// Build the program.
	prog := filepath.Join(tmpdir, file)
	cmd := exec.Command("go", "build", "-o", prog, file+".go")
	out, err := cmd.CombinedOutput()
	if err != nil {
		return fmt.Errorf("go build %s.go failed: %v\nOutput:\n%s", file, err, out)
	}
	defer os.Remove(prog)

	// Only run the test if we have output to check.
	if want == "" {
		return nil
	}

	cmd = exec.Command(prog)
	out, err = cmd.CombinedOutput()
	if err != nil {
		return fmt.Errorf("%s failed: %v\nOutput:\n%s", file, err, out)
	}

	// Canonicalize output.
	out = bytes.TrimRight(out, "\n")
	out = bytes.Replace(out, []byte{'\n'}, []byte{' '}, -1)

	// Check the result.
	match, err := regexp.Match(want, out)
	if err != nil {
		return fmt.Errorf("failed to parse regexp %q: %v", want, err)
	}
	if !match {
		return fmt.Errorf("%s.go:\n%q\ndoes not match %s", file, out, want)
	}

	return nil
}

type testcase struct {
	file string
	want string
}

var tests = []testcase{
	// defer_panic_recover
	{"defer", `^0 3210 2$`},
	{"defer2", `^Calling g. Printing in g 0 Printing in g 1 Printing in g 2 Printing in g 3 Panicking! Defer in g 3 Defer in g 2 Defer in g 1 Defer in g 0 Recovered in f 4 Returned normally from f.$`},

	// effective_go
	{"eff_bytesize", `^1.00YB 9.09TB$`},
	{"eff_qr", ""},
	{"eff_sequence", `^\[-1 2 6 16 44\]$`},
	{"eff_unused2", ""},

	// error_handling
	{"error", ""},
	{"error2", ""},
	{"error3", ""},
	{"error4", ""},

	// law_of_reflection
	{"interface", ""},
	{"interface2", `^type: float64$`},

	// c_go_cgo
	{"cgo1", ""},
	{"cgo2", ""},
	{"cgo3", ""},
	{"cgo4", ""},

	// timeout
	{"timeout1", ""},
	{"timeout2", ""},

	// gobs
	{"gobs1", ""},
	{"gobs2", ""},

	// json
	{"json1", `^$`},
	{"json2", `the reciprocal of i is`},
	{"json3", `Age is int 6`},
	{"json4", `^$`},
	{"json5", ""},

	// image_package
	{"image_package1", `^X is 2 Y is 1$`},
	{"image_package2", `^3 4 false$`},
	{"image_package3", `^3 4 true$`},
	{"image_package4", `^image.Point{X:2, Y:1}$`},
	{"image_package5", `^{255 0 0 255}$`},
	{"image_package6", `^8 4 true$`},

	// other
	{"go1", `^Christmas is a holiday: true .*go1.go already exists$`},
	{"slices", ""},
}

func onlyTest(files ...string) {
	var new []testcase
NextFile:
	for _, file := range files {
		file = strings.TrimSuffix(file, ".go")
		for _, tt := range tests {
			if tt.file == file {
				new = append(new, tt)
				continue NextFile
			}
		}
		fmt.Fprintf(os.Stderr, "test %s.go not found\n", file)
		os.Exit(1)
	}
	tests = new
}

func skipTest(file string) {
	for i, tt := range tests {
		if tt.file == file {
			copy(tests[i:], tests[i+1:])
			tests = tests[:len(tests)-1]
			return
		}
	}
	panic("delete(" + file + "): not found")
}

func fixcgo() {
	if os.Getenv("CGO_ENABLED") != "1" {
		skipTest("cgo1")
		skipTest("cgo2")
		skipTest("cgo3")
		skipTest("cgo4")
		return
	}

	switch runtime.GOOS {
	case "freebsd":
		// cgo1 and cgo2 don't run on freebsd, srandom has a different signature
		skipTest("cgo1")
		skipTest("cgo2")
	case "netbsd":
		// cgo1 and cgo2 don't run on netbsd, srandom has a different signature
		skipTest("cgo1")
		skipTest("cgo2")
		// cgo3 and cgo4 don't run on netbsd, since cgo cannot handle stdout correctly, see issue #10715.
		skipTest("cgo3")
		skipTest("cgo4")
	case "openbsd", "solaris":
		// cgo3 and cgo4 don't run on openbsd and solaris, since cgo cannot handle stdout correctly, see issue #10715.
		skipTest("cgo3")
		skipTest("cgo4")
	}
}
<|endoftext|>


<commit_before>package dpn

import (
	"encoding/json"
	"fmt"
	"github.com/APTrust/bagman/bagman"
	"github.com/nsqio/go-nsq"
	"github.com/crowdmob/goamz/s3"
	"os"
	"path/filepath"
	"sync"
	"time"
)

type Storer struct {
	StorageChannel      chan *DPNResult
	CleanupChannel      chan *DPNResult
	BagCreateChannel    chan *DPNResult
	PostProcessChannel  chan *DPNResult
	ProcUtil            *bagman.ProcessUtil
	DPNConfig           *DPNConfig
	LocalRESTClient     *DPNRestClient
	// WaitGroup is for running local tests only.
	WaitGroup           sync.WaitGroup
}

func NewStorer(procUtil *bagman.ProcessUtil, dpnConfig *DPNConfig) (*Storer, error) {
	// Set up a DPN REST client that talks to our local DPN REST service.
	localClient, err := NewDPNRestClient(
		dpnConfig.RestClient.LocalServiceURL,
		dpnConfig.RestClient.LocalAPIRoot,
		dpnConfig.RestClient.LocalAuthToken,
		dpnConfig.LocalNode,
		dpnConfig,
		procUtil.MessageLog)
	if err != nil {
		return nil, err
	}

	storer := &Storer{
		LocalRESTClient: localClient,
		ProcUtil: procUtil,
		DPNConfig: dpnConfig,
	}
	workerBufferSize := procUtil.Config.DPNStoreWorker.Workers * 10
	storer.StorageChannel = make(chan *DPNResult, workerBufferSize)
	storer.BagCreateChannel = make(chan *DPNResult, workerBufferSize)
	storer.CleanupChannel = make(chan *DPNResult, workerBufferSize)
	storer.PostProcessChannel = make(chan *DPNResult, workerBufferSize)
	for i := 0; i < procUtil.Config.DPNStoreWorker.Workers; i++ {
		go storer.createBagRecord()
		go storer.cleanup()
		go storer.postProcess()
	}
	for i := 0; i < procUtil.Config.DPNStoreWorker.NetworkConnections; i++ {
		go storer.store()
	}
	return storer, nil
}

func (storer *Storer) HandleMessage(message *nsq.Message) error {
	message.DisableAutoResponse()
	result := &DPNResult{}
	err := json.Unmarshal(message.Body, result)
	if err != nil {
		storer.ProcUtil.MessageLog.Error("Could not unmarshal JSON data from nsq:",
			string(message.Body))
		message.Finish()
		return fmt.Errorf("Could not unmarshal JSON data from nsq")
	}
	if result.BagMd5Digest == "" || result.BagSha256Digest == "" {
		errMsg := "Bag cannot be stored because DPNResult is missing either md5 or sha256 checksum."
		storer.ProcUtil.MessageLog.Error(errMsg)
		result.ErrorMessage = errMsg
		SendToTroubleQueue(result, storer.ProcUtil)
		message.Finish()
		return fmt.Errorf(errMsg)
	}
	result.NsqMessage = message
	result.Stage = STAGE_STORE
	bagIdentifier := result.BagIdentifier
	if bagIdentifier == "" {
		bagIdentifier = "DPN Replication Bag"
	}
	storer.ProcUtil.MessageLog.Info("Putting %s into the storage queue (%s)",
		result.TarFilePath(), bagIdentifier)
	storer.StorageChannel <- result
	return nil
}

func (storer *Storer) store() {
	for result := range storer.StorageChannel {
		if result.NsqMessage != nil {
			result.NsqMessage.Touch()
		}

		// By the time we get our hands on this replication request,
		// it may be many hours old. Fetch the request again from the
		// originating node and make sure it hasn't been cancelled.
		if result.TransferRequest != nil {
			storer.refetchReplicationRequest(result)
		}
		if result.ErrorMessage != "" {
			storer.PostProcessChannel <- result
			continue
		}
		// Remember that TransferRequest will be nil
		// for bags we built ourselves.
		if result.TransferRequest != nil && result.TransferRequest.Status == "Cancelled" {
			result.Stage = STAGE_CANCELLED
			result.Retry = false
			storer.PostProcessChannel <- result
			continue
		}

		err := os.MkdirAll(filepath.Dir(result.TarFilePath()), 0755)
		if err != nil {
			result.ErrorMessage = fmt.Sprintf("Error creating directory '%s': %v",
				filepath.Dir(result.TarFilePath()), err)
			storer.PostProcessChannel <- result
			continue
		}

		// Now we'll open a file reader and stream the tar file
		// up to S3.
		reader, err := os.Open(result.TarFilePath())
		if err != nil {
			result.ErrorMessage = fmt.Sprintf("Error opening file '%s': %v",
				result.TarFilePath(), err)
			reader.Close()
			storer.PostProcessChannel <- result
			continue
		}
		fileInfo, err := reader.Stat()
		if err != nil {
			result.ErrorMessage = fmt.Sprintf("Cannot stat file '%s': %v",
				result.TarFilePath(), err)
			reader.Close()
			storer.PostProcessChannel <- result
			continue
		}
		options, err := storer.GetS3Options(result)
		if err != nil {
			result.ErrorMessage = fmt.Sprintf("Error generating S3 options: %v", err)
			reader.Close()
			storer.PostProcessChannel <- result
			continue
		}

		bagUUID := ""
		if result.DPNBag != nil {
			bagUUID = result.DPNBag.UUID
		} else {
			bagUUID = result.PackageResult.BagBuilder.UUID
		}
		fileName := fmt.Sprintf("%s.tar", bagUUID)
		url, err := storer.ProcUtil.S3Client.SaveToS3(
			storer.ProcUtil.Config.DPNPreservationBucket,
			fileName,
			"application/x-tar",
			reader,
			fileInfo.Size(),
			options)

		// Close the reader
		reader.Close()
		if err != nil {
			result.ErrorMessage = fmt.Sprintf("Error saving file to S3/Glacier: %v", err)
			storer.PostProcessChannel <- result
			continue
		}

		// Record where we stored the file
		result.StorageURL = url

		// Update the transfer request, if there is one.
		if result.TransferRequest != nil {
			result.TransferRequest.Status = "Stored"
		}
		if result.ErrorMessage != "" {
			result.ErrorMessage = fmt.Sprintf("Bag was stored, but we couldn't " +
				"update the replication request on the originating node: %v", err)
			storer.PostProcessChannel <- result
			continue
		}

		if result.NsqMessage != nil {
			result.NsqMessage.Touch()
		}

		// This channel really only applies to bags we created
		// at our own node. (Not replication requests.)
		storer.BagCreateChannel <- result
	}
}

// Get the replication request from the originating node and
// make sure it's still valid.
func (storer *Storer) refetchReplicationRequest(result *DPNResult) {
	storer.makeReplicationRequest(result, "GET")
}

func (storer *Storer) makeReplicationRequest(result *DPNResult, whatKind string) {

	if result.TransferRequest == nil {
		// This is a local bag and there is no replication request.
		storer.ProcUtil.MessageLog.Error("makeReplicationRequest was called for bag %s (%s), " +
			"which has no associated replication request. Is this a local bag?",
			result.BagIdentifier, result.PackageResult.BagBuilder.UUID)
		return
	}

	// Get a REST client that can talk the remote node.
	// This client will be configured with the correct URL
	// and API token to talk to the node that issues the
	// replication request.
	remoteClient, err := storer.LocalRESTClient.GetRemoteClient(
		result.TransferRequest.FromNode,  // the node we want to talk to
		storer.DPNConfig,
		storer.ProcUtil.MessageLog)
	if err != nil {
		result.ErrorMessage = fmt.Sprintf("Could not create a REST client to connect to node %s: %v",
			result.TransferRequest.FromNode, err)
		return
	}

	var xferRequest *DPNReplicationTransfer
	if whatKind == "GET" {
		storer.ProcUtil.MessageLog.Debug("Replication request id: %s, " +
			"FromNode: %s, ToNode: %s", result.TransferRequest.ReplicationId,
			result.TransferRequest.FromNode, result.TransferRequest.ToNode)
		xferRequest, err = remoteClient.ReplicationTransferGet(result.TransferRequest.ReplicationId)
	} else if whatKind == "POST" {
		xferRequest, err = remoteClient.ReplicationTransferUpdate(result.TransferRequest)
	} else {
		panic("makeReplicationRequest doesn't understand that kind of request")
	}

	if err != nil {
		result.ErrorMessage = fmt.Sprintf("Could not get replication record %s from node %s: %v",
			result.TransferRequest.ReplicationId, result.TransferRequest.FromNode, err)
		return
	}
	result.TransferRequest = xferRequest

}

func (storer *Storer) createBagRecord() {
	for result := range storer.BagCreateChannel {
		// If result has a local identifier, it's a bag we created
		// here at the local node (as opposed to a bag we're just
		// replicating to fulfill a transfer request). Since we
		// created this bag here, we need to create a Bag record
		// for it in DPN. We could also check result.PackageResult
		// to determine if bag was made here. That will be non-nil
		// for bags we built, nil for bags built elsewhere.
		bagWasCreatedHere := result.BagIdentifier != ""
		bagStoredSuccessfully := (result.ErrorMessage == "" && result.StorageURL != "")
		if bagWasCreatedHere && bagStoredSuccessfully {
			storer.ProcUtil.MessageLog.Debug("Creating bag record for %s with md5 %s and sha256 %s",
				result.BagIdentifier, result.BagMd5Digest, result.BagSha256Digest)
			fixity := &DPNFixity{
				Sha256: result.BagSha256Digest,
			}
			fileInfo, err := os.Stat(result.TarFilePath())
			if err != nil {
				result.ErrorMessage = fmt.Sprintf("Cannot stat %s to get file size: %v",
					result.TarFilePath(), err)
				storer.CleanupChannel <- result
				continue
			}
			newBag := &DPNBag{
				UUID: result.PackageResult.BagBuilder.UUID,
				LocalId: result.BagIdentifier,
				Size: uint64(fileInfo.Size()),
				FirstVersionUUID: result.PackageResult.BagBuilder.UUID,
				Version: 1,
				IngestNode: storer.DPNConfig.LocalNode,
				AdminNode: storer.DPNConfig.LocalNode,
				BagType: "D",
				// No spec yet on how to specify rights; cannot be nil
				Rights: make([]string, 0),
				// No spec yet on how to specify interpretive; cannot be nil
				Interpretive: make([]string, 0),
				// No replicating nodes on a new bag; cannot be nil
				ReplicatingNodes: make([]string, 0),
				Fixities: fixity,
			}
			result.DPNBag = newBag
		}
		storer.CleanupChannel <- result
	}
}

func (storer *Storer) cleanup() {
	for result := range storer.CleanupChannel {
		thisIsNotATest := (result.NsqMessage != nil)
		storageSucceeded := (result.ErrorMessage == "" && result.StorageURL != "")
		if storageSucceeded && thisIsNotATest {
			err := os.Remove(result.TarFilePath())
			if err != nil {
				storer.ProcUtil.MessageLog.Warning("Error cleaning up %s: %v",
					result.TarFilePath(), err)
			} else {
				storer.ProcUtil.MessageLog.Info(
					"After successful upload, deleted local DPN bag at %s",
					result.TarFilePath())
			}
		}
		storer.PostProcessChannel <- result
	}
}

func (storer *Storer) postProcess() {
	for result := range storer.PostProcessChannel {
		bagIdentifier := result.BagIdentifier
		if bagIdentifier == "" {
			bagIdentifier = result.DPNBag.UUID
		}
		if result.ErrorMessage == "" && result.StorageURL != "" {
			// SUCCESS :)
			storer.ProcUtil.MessageLog.Info("Bag %s successfully stored at %s",
				bagIdentifier, result.StorageURL)
			storer.ProcUtil.IncrementSucceeded()
			// Send to queue for recording in Fluctus and/or DPN REST
			if result.NsqMessage != nil {
				result.NsqMessage.Finish()
				SendToRecordQueue(result, storer.ProcUtil)
			}
		} else {
			// FAILURE :(
			storer.ProcUtil.MessageLog.Error(result.ErrorMessage)
			storer.ProcUtil.IncrementFailed()
			// Item failed after max attempts. Put in trouble queue
			// for admin review.
			if result.NsqMessage != nil {
				if result.NsqMessage.Attempts >= uint16(storer.ProcUtil.Config.DPNStoreWorker.MaxAttempts) {
					// No more retries
					result.NsqMessage.Finish()
					SendToTroubleQueue(result, storer.ProcUtil)
				} else {
					storer.ProcUtil.MessageLog.Info("Requeuing %s (%s)",
						bagIdentifier, result.TarFilePath())
					result.NsqMessage.Requeue(1 * time.Minute)
				}
			}
		}
		if result.NsqMessage == nil {
			// This is a test message, running outside production.
			storer.WaitGroup.Done()
		}
	}
}

func (storer *Storer) GetS3Options(result *DPNResult) (s3.Options, error) {
	// Prepare metadata for save to S3
	s3Metadata := make(map[string][]string)
	if result.BagIdentifier != "" {
		s3Metadata["aptrust-bag"] = []string{result.BagIdentifier}
	}
	// Save to S3 with the base64-encoded md5 sum
	base64md5, err := bagman.Base64EncodeMd5(result.BagMd5Digest)
	if err != nil {
		return s3.Options{}, err
	}
	options := storer.ProcUtil.S3Client.MakeOptions(base64md5, s3Metadata)
	return options, nil
}

func (storer *Storer) RunTest(result *DPNResult) {
	storer.WaitGroup.Add(1)
	storer.ProcUtil.MessageLog.Info("Putting %s into digest channel",
		result.BagIdentifier)
	storer.StorageChannel <- result
	storer.WaitGroup.Wait()
	fmt.Println("Storer is done")
}
<commit_msg>Fixed PT #113866839: store large DPN files<commit_after>package dpn

import (
	"encoding/json"
	"fmt"
	"github.com/APTrust/bagman/bagman"
	"github.com/nsqio/go-nsq"
	"github.com/crowdmob/goamz/s3"
	"os"
	"path/filepath"
	"sync"
	"time"
)

type Storer struct {
	StorageChannel      chan *DPNResult
	CleanupChannel      chan *DPNResult
	BagCreateChannel    chan *DPNResult
	PostProcessChannel  chan *DPNResult
	ProcUtil            *bagman.ProcessUtil
	DPNConfig           *DPNConfig
	LocalRESTClient     *DPNRestClient
	// WaitGroup is for running local tests only.
	WaitGroup           sync.WaitGroup
}

func NewStorer(procUtil *bagman.ProcessUtil, dpnConfig *DPNConfig) (*Storer, error) {
	// Set up a DPN REST client that talks to our local DPN REST service.
	localClient, err := NewDPNRestClient(
		dpnConfig.RestClient.LocalServiceURL,
		dpnConfig.RestClient.LocalAPIRoot,
		dpnConfig.RestClient.LocalAuthToken,
		dpnConfig.LocalNode,
		dpnConfig,
		procUtil.MessageLog)
	if err != nil {
		return nil, err
	}

	storer := &Storer{
		LocalRESTClient: localClient,
		ProcUtil: procUtil,
		DPNConfig: dpnConfig,
	}
	workerBufferSize := procUtil.Config.DPNStoreWorker.Workers * 10
	storer.StorageChannel = make(chan *DPNResult, workerBufferSize)
	storer.BagCreateChannel = make(chan *DPNResult, workerBufferSize)
	storer.CleanupChannel = make(chan *DPNResult, workerBufferSize)
	storer.PostProcessChannel = make(chan *DPNResult, workerBufferSize)
	for i := 0; i < procUtil.Config.DPNStoreWorker.Workers; i++ {
		go storer.createBagRecord()
		go storer.cleanup()
		go storer.postProcess()
	}
	for i := 0; i < procUtil.Config.DPNStoreWorker.NetworkConnections; i++ {
		go storer.store()
	}
	return storer, nil
}

func (storer *Storer) HandleMessage(message *nsq.Message) error {
	message.DisableAutoResponse()
	result := &DPNResult{}
	err := json.Unmarshal(message.Body, result)
	if err != nil {
		storer.ProcUtil.MessageLog.Error("Could not unmarshal JSON data from nsq:",
			string(message.Body))
		message.Finish()
		return fmt.Errorf("Could not unmarshal JSON data from nsq")
	}
	if result.BagMd5Digest == "" || result.BagSha256Digest == "" {
		errMsg := "Bag cannot be stored because DPNResult is missing either md5 or sha256 checksum."
		storer.ProcUtil.MessageLog.Error(errMsg)
		result.ErrorMessage = errMsg
		SendToTroubleQueue(result, storer.ProcUtil)
		message.Finish()
		return fmt.Errorf(errMsg)
	}
	result.NsqMessage = message
	result.Stage = STAGE_STORE
	bagIdentifier := result.BagIdentifier
	if bagIdentifier == "" {
		bagIdentifier = "DPN Replication Bag"
	}
	storer.ProcUtil.MessageLog.Info("Putting %s into the storage queue (%s)",
		result.TarFilePath(), bagIdentifier)
	storer.StorageChannel <- result
	return nil
}

func (storer *Storer) store() {
	for result := range storer.StorageChannel {
		if result.NsqMessage != nil {
			result.NsqMessage.Touch()
		}

		// By the time we get our hands on this replication request,
		// it may be many hours old. Fetch the request again from the
		// originating node and make sure it hasn't been cancelled.
		if result.TransferRequest != nil {
			storer.refetchReplicationRequest(result)
		}
		if result.ErrorMessage != "" {
			storer.PostProcessChannel <- result
			continue
		}
		// Remember that TransferRequest will be nil
		// for bags we built ourselves.
		if result.TransferRequest != nil && result.TransferRequest.Status == "Cancelled" {
			result.Stage = STAGE_CANCELLED
			result.Retry = false
			storer.PostProcessChannel <- result
			continue
		}

		err := os.MkdirAll(filepath.Dir(result.TarFilePath()), 0755)
		if err != nil {
			result.ErrorMessage = fmt.Sprintf("Error creating directory '%s': %v",
				filepath.Dir(result.TarFilePath()), err)
			storer.PostProcessChannel <- result
			continue
		}

		// Now we'll open a file reader and stream the tar file
		// up to S3.
		reader, err := os.Open(result.TarFilePath())
		if err != nil {
			result.ErrorMessage = fmt.Sprintf("Error opening file '%s': %v",
				result.TarFilePath(), err)
			reader.Close()
			storer.PostProcessChannel <- result
			continue
		}
		fileInfo, err := reader.Stat()
		if err != nil {
			result.ErrorMessage = fmt.Sprintf("Cannot stat file '%s': %v",
				result.TarFilePath(), err)
			reader.Close()
			storer.PostProcessChannel <- result
			continue
		}
		options, err := storer.GetS3Options(result)
		if err != nil {
			result.ErrorMessage = fmt.Sprintf("Error generating S3 options: %v", err)
			reader.Close()
			storer.PostProcessChannel <- result
			continue
		}

		bagUUID := ""
		if result.DPNBag != nil {
			bagUUID = result.DPNBag.UUID
		} else {
			bagUUID = result.PackageResult.BagBuilder.UUID
		}
		fileName := fmt.Sprintf("%s.tar", bagUUID)
		url := ""
		if fileInfo.Size() > bagman.S3_LARGE_FILE {
			url, err = storer.ProcUtil.S3Client.SaveLargeFileToS3(
				storer.ProcUtil.Config.DPNPreservationBucket,
				fileName,
				"application/x-tar",
				reader,
				fileInfo.Size(),
				options,
				bagman.S3_CHUNK_SIZE)
		} else {
			url, err = storer.ProcUtil.S3Client.SaveToS3(
				storer.ProcUtil.Config.DPNPreservationBucket,
				fileName,
				"application/x-tar",
				reader,
				fileInfo.Size(),
				options)
		}

		// Close the reader
		reader.Close()
		if err != nil {
			result.ErrorMessage = fmt.Sprintf("Error saving file to S3/Glacier: %v", err)
			storer.PostProcessChannel <- result
			continue
		}

		// Record where we stored the file
		result.StorageURL = url

		// Update the transfer request, if there is one.
		if result.TransferRequest != nil {
			result.TransferRequest.Status = "Stored"
		}
		if result.ErrorMessage != "" {
			result.ErrorMessage = fmt.Sprintf("Bag was stored, but we couldn't " +
				"update the replication request on the originating node: %v", err)
			storer.PostProcessChannel <- result
			continue
		}

		if result.NsqMessage != nil {
			result.NsqMessage.Touch()
		}

		// This channel really only applies to bags we created
		// at our own node. (Not replication requests.)
		storer.BagCreateChannel <- result
	}
}

// Get the replication request from the originating node and
// make sure it's still valid.
func (storer *Storer) refetchReplicationRequest(result *DPNResult) {
	storer.makeReplicationRequest(result, "GET")
}

func (storer *Storer) makeReplicationRequest(result *DPNResult, whatKind string) {

	if result.TransferRequest == nil {
		// This is a local bag and there is no replication request.
		storer.ProcUtil.MessageLog.Error("makeReplicationRequest was called for bag %s (%s), " +
			"which has no associated replication request. Is this a local bag?",
			result.BagIdentifier, result.PackageResult.BagBuilder.UUID)
		return
	}

	// Get a REST client that can talk the remote node.
	// This client will be configured with the correct URL
	// and API token to talk to the node that issues the
	// replication request.
	remoteClient, err := storer.LocalRESTClient.GetRemoteClient(
		result.TransferRequest.FromNode,  // the node we want to talk to
		storer.DPNConfig,
		storer.ProcUtil.MessageLog)
	if err != nil {
		result.ErrorMessage = fmt.Sprintf("Could not create a REST client to connect to node %s: %v",
			result.TransferRequest.FromNode, err)
		return
	}

	var xferRequest *DPNReplicationTransfer
	if whatKind == "GET" {
		storer.ProcUtil.MessageLog.Debug("Replication request id: %s, " +
			"FromNode: %s, ToNode: %s", result.TransferRequest.ReplicationId,
			result.TransferRequest.FromNode, result.TransferRequest.ToNode)
		xferRequest, err = remoteClient.ReplicationTransferGet(result.TransferRequest.ReplicationId)
	} else if whatKind == "POST" {
		xferRequest, err = remoteClient.ReplicationTransferUpdate(result.TransferRequest)
	} else {
		panic("makeReplicationRequest doesn't understand that kind of request")
	}

	if err != nil {
		result.ErrorMessage = fmt.Sprintf("Could not get replication record %s from node %s: %v",
			result.TransferRequest.ReplicationId, result.TransferRequest.FromNode, err)
		return
	}
	result.TransferRequest = xferRequest

}

func (storer *Storer) createBagRecord() {
	for result := range storer.BagCreateChannel {
		// If result has a local identifier, it's a bag we created
		// here at the local node (as opposed to a bag we're just
		// replicating to fulfill a transfer request). Since we
		// created this bag here, we need to create a Bag record
		// for it in DPN. We could also check result.PackageResult
		// to determine if bag was made here. That will be non-nil
		// for bags we built, nil for bags built elsewhere.
		bagWasCreatedHere := result.BagIdentifier != ""
		bagStoredSuccessfully := (result.ErrorMessage == "" && result.StorageURL != "")
		if bagWasCreatedHere && bagStoredSuccessfully {
			storer.ProcUtil.MessageLog.Debug("Creating bag record for %s with md5 %s and sha256 %s",
				result.BagIdentifier, result.BagMd5Digest, result.BagSha256Digest)
			fixity := &DPNFixity{
				Sha256: result.BagSha256Digest,
			}
			fileInfo, err := os.Stat(result.TarFilePath())
			if err != nil {
				result.ErrorMessage = fmt.Sprintf("Cannot stat %s to get file size: %v",
					result.TarFilePath(), err)
				storer.CleanupChannel <- result
				continue
			}
			newBag := &DPNBag{
				UUID: result.PackageResult.BagBuilder.UUID,
				LocalId: result.BagIdentifier,
				Size: uint64(fileInfo.Size()),
				FirstVersionUUID: result.PackageResult.BagBuilder.UUID,
				Version: 1,
				IngestNode: storer.DPNConfig.LocalNode,
				AdminNode: storer.DPNConfig.LocalNode,
				BagType: "D",
				// No spec yet on how to specify rights; cannot be nil
				Rights: make([]string, 0),
				// No spec yet on how to specify interpretive; cannot be nil
				Interpretive: make([]string, 0),
				// No replicating nodes on a new bag; cannot be nil
				ReplicatingNodes: make([]string, 0),
				Fixities: fixity,
			}
			result.DPNBag = newBag
		}
		storer.CleanupChannel <- result
	}
}

func (storer *Storer) cleanup() {
	for result := range storer.CleanupChannel {
		thisIsNotATest := (result.NsqMessage != nil)
		storageSucceeded := (result.ErrorMessage == "" && result.StorageURL != "")
		if storageSucceeded && thisIsNotATest {
			err := os.Remove(result.TarFilePath())
			if err != nil {
				storer.ProcUtil.MessageLog.Warning("Error cleaning up %s: %v",
					result.TarFilePath(), err)
			} else {
				storer.ProcUtil.MessageLog.Info(
					"After successful upload, deleted local DPN bag at %s",
					result.TarFilePath())
			}
		}
		storer.PostProcessChannel <- result
	}
}

func (storer *Storer) postProcess() {
	for result := range storer.PostProcessChannel {
		bagIdentifier := result.BagIdentifier
		if bagIdentifier == "" {
			bagIdentifier = result.DPNBag.UUID
		}
		if result.ErrorMessage == "" && result.StorageURL != "" {
			// SUCCESS :)
			storer.ProcUtil.MessageLog.Info("Bag %s successfully stored at %s",
				bagIdentifier, result.StorageURL)
			storer.ProcUtil.IncrementSucceeded()
			// Send to queue for recording in Fluctus and/or DPN REST
			if result.NsqMessage != nil {
				result.NsqMessage.Finish()
				SendToRecordQueue(result, storer.ProcUtil)
			}
		} else {
			// FAILURE :(
			storer.ProcUtil.MessageLog.Error(result.ErrorMessage)
			storer.ProcUtil.IncrementFailed()
			// Item failed after max attempts. Put in trouble queue
			// for admin review.
			if result.NsqMessage != nil {
				if result.NsqMessage.Attempts >= uint16(storer.ProcUtil.Config.DPNStoreWorker.MaxAttempts) {
					// No more retries
					result.NsqMessage.Finish()
					SendToTroubleQueue(result, storer.ProcUtil)
				} else {
					storer.ProcUtil.MessageLog.Info("Requeuing %s (%s)",
						bagIdentifier, result.TarFilePath())
					result.NsqMessage.Requeue(1 * time.Minute)
				}
			}
		}
		if result.NsqMessage == nil {
			// This is a test message, running outside production.
			storer.WaitGroup.Done()
		}
	}
}

func (storer *Storer) GetS3Options(result *DPNResult) (s3.Options, error) {
	// Prepare metadata for save to S3
	s3Metadata := make(map[string][]string)
	if result.BagIdentifier != "" {
		s3Metadata["aptrust-bag"] = []string{result.BagIdentifier}
	}
	// Save to S3 with the base64-encoded md5 sum
	base64md5, err := bagman.Base64EncodeMd5(result.BagMd5Digest)
	if err != nil {
		return s3.Options{}, err
	}
	options := storer.ProcUtil.S3Client.MakeOptions(base64md5, s3Metadata)
	return options, nil
}

func (storer *Storer) RunTest(result *DPNResult) {
	storer.WaitGroup.Add(1)
	storer.ProcUtil.MessageLog.Info("Putting %s into digest channel",
		result.BagIdentifier)
	storer.StorageChannel <- result
	storer.WaitGroup.Wait()
	fmt.Println("Storer is done")
}
<|endoftext|>


<commit_before>/*
 * Copyright 2011 Nan Deng
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package uniqush

import (
    "os"
    "strings"
)

type nullPushFailureHandler struct {}

func (f *nullPushFailureHandler) OnPushFail(pst PushServiceType, id string, err os.Error) {
}

type PushServiceManager struct {
    serviceTypes map[string]PushServiceType
    pfp PushFailureHandler
}

var (
    pushServiceManager *PushServiceManager
)

func init() {
    pushServiceManager = newPushServiceManager()
}

/* This is a singleton */
func newPushServiceManager() *PushServiceManager {
    ret := new(PushServiceManager)
    ret.serviceTypes = make(map[string]PushServiceType, 5)
    ret.pfp = &nullPushFailureHandler{}
    return ret
}

func GetPushServiceManager() *PushServiceManager {
    return pushServiceManager
}

func (m *PushServiceManager) SetAsyncFailureProcessor(pfp PushFailureHandler) {
    if pfp != nil {
        m.pfp = pfp
    }
}

func (m *PushServiceManager) RegisterPushServiceType(pt PushServiceType) os.Error {
    name := pt.Name()
    m.serviceTypes[name] = pt
    pt.SetAsyncFailureHandler(m.pfp)
    return nil
}

func (m *PushServiceManager) BuildPushServiceProviderFromMap(kv map[string]string) (psp *PushServiceProvider, err os.Error) {
    if ptname, ok := kv["pushservicetype"]; ok {
        if pst, ok := m.serviceTypes[ptname]; ok {
            psp, err = pst.BuildPushServiceProviderFromMap(kv)
            if err != nil {
                return nil, err
            }
            if psp == nil {
                return nil, os.NewError("Cannot Build Push Service Provider")
            }
            psp.pushServiceType = pst
            return
        }
        return nil, os.NewError("Unknown Push Service Type")
    }
    return nil, os.NewError("No Push Service Type Specified")
}

func (m *PushServiceManager) BuildPushServiceProviderFromBytes(value []byte) (psp *PushServiceProvider, err os.Error) {
    s := string(value)
    parts := strings.SplitN(s, ":", 2)
    if len(parts) >= 2 {
        ptname := parts[0]
        if pst, ok := m.serviceTypes[ptname]; ok {
            psp = NewEmptyPushServiceProvider()
            psp.pushServiceType = pst
            err = psp.Unmarshal([]byte(parts[1]))
            if err != nil {
                psp = nil
                return
            }
            return
        }
        return nil, os.NewError("Unknown Push Service Type")
    }
    return nil, os.NewError("No Push Service Type Specified")
}

func (m *PushServiceManager) BuildDeliveryPointFromMap(kv map[string]string) (dp *DeliveryPoint, err os.Error) {
    if ptname, ok := kv["pushservicetype"]; ok {
        if pst, ok := m.serviceTypes[ptname]; ok {
            dp, err = pst.BuildDeliveryPointFromMap(kv)
            if err != nil {
                return nil, err
            }
            if dp == nil {
                return nil, os.NewError("Cannot Build Delivery Point")
            }
            dp.pushServiceType = pst
            return
        }
        return nil, os.NewError("Unknown Push Service Type")
    }
    return nil, os.NewError("No Push Service Type Specified")
}

func (m *PushServiceManager) BuildDeliveryPointFromBytes(value []byte) (dp *DeliveryPoint, err os.Error) {
    s := string(value)
    parts := strings.SplitN(s, ":", 2)
    if len(parts) >= 2 {
        ptname := parts[0]
        if pst, ok := m.serviceTypes[ptname]; ok {
            dp = NewEmptyDeliveryPoint()
            dp.pushServiceType = pst
            err = dp.Unmarshal([]byte(parts[1]))
            if err != nil {
                dp = nil
                return
            }
            return
        }
        return nil, os.NewError("Unknown Push Service Type")
    }
    return nil, os.NewError("No Push Service Type Specified")
}

func (m *PushServiceManager) Push(psp *PushServiceProvider, dp *DeliveryPoint, n *Notification) (id string, err os.Error) {
    if psp.pushServiceType != nil {
        id, err = psp.pushServiceType.Push(psp, dp, n)
        return
    }
    return "", NewInvalidPushServiceProviderError(psp, os.NewError("Unknown Service Type"))
}

func (m *PushServiceManager) Finalize() {
    for _, t := range m.serviceTypes {
        t.Finalize()
    }
}

<commit_msg>bug fix: the order of exec. init() diff. on goinstall-ed pkg & gomake install pkg<commit_after>/*
 * Copyright 2011 Nan Deng
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package uniqush

import (
    "os"
    "strings"
)

type nullPushFailureHandler struct {}

func (f *nullPushFailureHandler) OnPushFail(pst PushServiceType, id string, err os.Error) {
}

type PushServiceManager struct {
    serviceTypes map[string]PushServiceType
    pfp PushFailureHandler
}

var (
    pushServiceManager *PushServiceManager
)

func init() {
//    pushServiceManager = newPushServiceManager()
}

/* This is a singleton */
func newPushServiceManager() *PushServiceManager {
    ret := new(PushServiceManager)
    ret.serviceTypes = make(map[string]PushServiceType, 5)
    ret.pfp = &nullPushFailureHandler{}
    return ret
}

func GetPushServiceManager() *PushServiceManager {
    if pushServiceManager == nil {
        pushServiceManager = newPushServiceManager()
    }
    return pushServiceManager
}

func (m *PushServiceManager) SetAsyncFailureProcessor(pfp PushFailureHandler) {
    if pfp != nil {
        m.pfp = pfp
    }
}

func (m *PushServiceManager) RegisterPushServiceType(pt PushServiceType) os.Error {
    name := pt.Name()
    m.serviceTypes[name] = pt
    pt.SetAsyncFailureHandler(m.pfp)
    return nil
}

func (m *PushServiceManager) BuildPushServiceProviderFromMap(kv map[string]string) (psp *PushServiceProvider, err os.Error) {
    if ptname, ok := kv["pushservicetype"]; ok {
        if pst, ok := m.serviceTypes[ptname]; ok {
            psp, err = pst.BuildPushServiceProviderFromMap(kv)
            if err != nil {
                return nil, err
            }
            if psp == nil {
                return nil, os.NewError("Cannot Build Push Service Provider")
            }
            psp.pushServiceType = pst
            return
        }
        return nil, os.NewError("Unknown Push Service Type")
    }
    return nil, os.NewError("No Push Service Type Specified")
}

func (m *PushServiceManager) BuildPushServiceProviderFromBytes(value []byte) (psp *PushServiceProvider, err os.Error) {
    s := string(value)
    parts := strings.SplitN(s, ":", 2)
    if len(parts) >= 2 {
        ptname := parts[0]
        if pst, ok := m.serviceTypes[ptname]; ok {
            psp = NewEmptyPushServiceProvider()
            psp.pushServiceType = pst
            err = psp.Unmarshal([]byte(parts[1]))
            if err != nil {
                psp = nil
                return
            }
            return
        }
        return nil, os.NewError("Unknown Push Service Type")
    }
    return nil, os.NewError("No Push Service Type Specified")
}

func (m *PushServiceManager) BuildDeliveryPointFromMap(kv map[string]string) (dp *DeliveryPoint, err os.Error) {
    if ptname, ok := kv["pushservicetype"]; ok {
        if pst, ok := m.serviceTypes[ptname]; ok {
            dp, err = pst.BuildDeliveryPointFromMap(kv)
            if err != nil {
                return nil, err
            }
            if dp == nil {
                return nil, os.NewError("Cannot Build Delivery Point")
            }
            dp.pushServiceType = pst
            return
        }
        return nil, os.NewError("Unknown Push Service Type")
    }
    return nil, os.NewError("No Push Service Type Specified")
}

func (m *PushServiceManager) BuildDeliveryPointFromBytes(value []byte) (dp *DeliveryPoint, err os.Error) {
    s := string(value)
    parts := strings.SplitN(s, ":", 2)
    if len(parts) >= 2 {
        ptname := parts[0]
        if pst, ok := m.serviceTypes[ptname]; ok {
            dp = NewEmptyDeliveryPoint()
            dp.pushServiceType = pst
            err = dp.Unmarshal([]byte(parts[1]))
            if err != nil {
                dp = nil
                return
            }
            return
        }
        return nil, os.NewError("Unknown Push Service Type")
    }
    return nil, os.NewError("No Push Service Type Specified")
}

func (m *PushServiceManager) Push(psp *PushServiceProvider, dp *DeliveryPoint, n *Notification) (id string, err os.Error) {
    if psp.pushServiceType != nil {
        id, err = psp.pushServiceType.Push(psp, dp, n)
        return
    }
    return "", NewInvalidPushServiceProviderError(psp, os.NewError("Unknown Service Type"))
}

func (m *PushServiceManager) Finalize() {
    for _, t := range m.serviceTypes {
        t.Finalize()
    }
}

<|endoftext|>


<commit_before>// Code for parsing coverage output in various formats.

package test

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"core"
)

// Parses test coverage for a single target from its output file.
func parseTestCoverage(target *core.BuildTarget, outputFile string) (core.TestCoverage, error) {
	coverage := core.NewTestCoverage()
	data, err := ioutil.ReadFile(outputFile)
	if err != nil && os.IsNotExist(err) {
		return coverage, nil // Tests aren't required to produce coverage files.
	} else if err != nil {
		return coverage, err
	} else if len(data) == 0 {
		return coverage, fmt.Errorf("Empty coverage output")
	} else if looksLikeGoCoverageResults(data) {
		// TODO(pebers): this is a little wasteful, we've already read the file once and we must do it again.
		return coverage, parseGoCoverageResults(target, &coverage, outputFile)
	} else {
		return coverage, parseXmlCoverageResults(target, &coverage, data)
	}
}

// Adds empty coverage entries for any files covered by the original query that we
// haven't discovered through tests to the overall report.
// The coverage reports only contain information about files that were covered during
// tests, so it's important that we identify anything with zero coverage here.
// This is made trickier by attempting to reconcile coverage targets from languages like
// Java that don't preserve the original file structure, which requires a slightly fuzzy match.
func AddOriginalTargetsToCoverage(state *core.BuildState) {
	// First we collect all the source files from all relevant targets
	allFiles := map[string]bool{}
	doneTargets := map[*core.BuildTarget]bool{}
	for _, label := range state.ExpandOriginalTargets() {
		collectAllFiles(state, state.Graph.TargetOrDie(label), allFiles, doneTargets)
	}

	// Now merge the recorded coverage so far into them
	recordedCoverage := state.Coverage
	state.Coverage = core.TestCoverage{Tests: recordedCoverage.Tests, Files: map[string][]core.LineCoverage{}}
	mergeCoverage(state, recordedCoverage, allFiles)
}

// Collects all the source files from a single target
func collectAllFiles(state *core.BuildState, target *core.BuildTarget, allFiles map[string]bool, doneTargets map[*core.BuildTarget]bool) {
	doneTargets[target] = true
	// Small hack here; explore these targets when we don't have any sources yet. Helps languages
	// like Java where we generate a wrapper target with a complete one immediately underneath.
	// TODO(pebers): do we still need this now we have Java sourcemaps?
	if !target.OutputIsComplete || len(allFiles) == 0 {
		for _, dep := range target.Dependencies() {
			if !doneTargets[dep] {
				collectAllFiles(state, dep, allFiles, doneTargets)
			}
		}
	}
	if target.IsTest {
		return // Test sources don't count for coverage.
	}
	for _, path := range target.AllSourcePaths(state.Graph) {
		extension := filepath.Ext(path)
		for _, ext := range state.Config.Cover.FileExtension {
			if ext == extension {
				allFiles[path] = target.IsTest || target.TestOnly // Skip test source files from actual coverage display
				break
			}
		}
	}
}

// Merges recorded coverage with the list of all existing files.
func mergeCoverage(state *core.BuildState, recordedCoverage core.TestCoverage, allFiles map[string]bool) {
	for file, coverage := range recordedCoverage.Files {
		state.Coverage.Files[file] = coverage
		allFiles[file] = true
	}
	// For any files left over now, enter them in as 100% uncovered.
	// This is pessimistic but there's not much we can do at this point.
	for file, done := range allFiles {
		if !done {
			s := make([]core.LineCoverage, countLines(file))
			if len(s) > 0 {
				for i := 0; i < len(s); i++ {
					s[i] = core.Uncovered
				}
				state.Coverage.Files[file] = s
			}
		}
	}
}

// Returns the number of lines in a file.
func countLines(path string) int {
	data, _ := ioutil.ReadFile(path)
	return bytes.Count(data, []byte{'\n'})
}

// WriteCoverageToFileOrDie writes the collected coverage data to a file in JSON format. Dies on failure.
func WriteCoverageToFileOrDie(coverage core.TestCoverage, filename string) {
	out := jsonCoverage{Tests: map[string]map[string]string{}}
	for label, coverage := range coverage.Tests {
		out.Tests[label.String()] = convertCoverage(coverage)
	}
	out.Files = convertCoverage(coverage.Files)
	if b, err := json.MarshalIndent(out, "", "    "); err != nil {
		log.Fatalf("Failed to encode json: %s", err)
	} else if err := ioutil.WriteFile(filename, b, 0644); err != nil {
		log.Fatalf("Failed to write coverage results to %s: %s", filename, err)
	}
}

func convertCoverage(in map[string][]core.LineCoverage) map[string]string {
	ret := map[string]string{}
	for k, v := range in {
		ret[k] = core.TestCoverageString(v)
	}
	return ret
}

// Used to prepare core.TestCoverage objects for JSON marshalling.
type jsonCoverage struct {
	Tests map[string]map[string]string `json:"tests"`
	Files map[string]string            `json:"files"`
}

// RemoveFilesFromCoverage removes any files with extensions matching the given set from coverage.
func RemoveFilesFromCoverage(coverage core.TestCoverage, extensions []string) {
	for _, files := range coverage.Tests {
		removeFilesFromCoverage(files, extensions)
	}
	removeFilesFromCoverage(coverage.Files, extensions)
}

func removeFilesFromCoverage(files map[string][]core.LineCoverage, extensions []string) {
	for filename := range files {
		for _, ext := range extensions {
			if strings.HasSuffix(filename, ext) {
				delete(files, filename)
			}
		}
	}
}
<commit_msg>Filter packages for coverage to just the set of packages from all targets specified on the command line.<commit_after>// Code for parsing coverage output in various formats.

package test

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"

	"core"
)

// Parses test coverage for a single target from its output file.
func parseTestCoverage(target *core.BuildTarget, outputFile string) (core.TestCoverage, error) {
	coverage := core.NewTestCoverage()
	data, err := ioutil.ReadFile(outputFile)
	if err != nil && os.IsNotExist(err) {
		return coverage, nil // Tests aren't required to produce coverage files.
	} else if err != nil {
		return coverage, err
	} else if len(data) == 0 {
		return coverage, fmt.Errorf("Empty coverage output")
	} else if looksLikeGoCoverageResults(data) {
		// TODO(pebers): this is a little wasteful, we've already read the file once and we must do it again.
		return coverage, parseGoCoverageResults(target, &coverage, outputFile)
	} else {
		return coverage, parseXmlCoverageResults(target, &coverage, data)
	}
}

// Adds empty coverage entries for any files covered by the original query that we
// haven't discovered through tests to the overall report.
// The coverage reports only contain information about files that were covered during
// tests, so it's important that we identify anything with zero coverage here.
// This is made trickier by attempting to reconcile coverage targets from languages like
// Java that don't preserve the original file structure, which requires a slightly fuzzy match.
func AddOriginalTargetsToCoverage(state *core.BuildState) {
	// First we collect all the source files from all relevant targets
	allFiles := map[string]bool{}
	doneTargets := map[*core.BuildTarget]bool{}
	// Track the set of packages the user ran tests from; we only show coverage metrics from them.
	coveragePackages := map[string]bool{}
	for _, label := range state.OriginalTargets {
		coveragePackages[label.PackageName] = true
	}
	for _, label := range state.ExpandOriginalTargets() {
		collectAllFiles(state, state.Graph.TargetOrDie(label), coveragePackages, allFiles, doneTargets)
	}

	// Now merge the recorded coverage so far into them
	recordedCoverage := state.Coverage
	state.Coverage = core.TestCoverage{Tests: recordedCoverage.Tests, Files: map[string][]core.LineCoverage{}}
	mergeCoverage(state, recordedCoverage, coveragePackages, allFiles)
}

// Collects all the source files from a single target
func collectAllFiles(state *core.BuildState, target *core.BuildTarget, coveragePackages, allFiles map[string]bool, doneTargets map[*core.BuildTarget]bool) {
	doneTargets[target] = true
	if !coveragePackages[target.Label.PackageName] {
		return
	}
	// Small hack here; explore these targets when we don't have any sources yet. Helps languages
	// like Java where we generate a wrapper target with a complete one immediately underneath.
	// TODO(pebers): do we still need this now we have Java sourcemaps?
	if !target.OutputIsComplete || len(allFiles) == 0 {
		for _, dep := range target.Dependencies() {
			if !doneTargets[dep] {
				collectAllFiles(state, dep, coveragePackages, allFiles, doneTargets)
			}
		}
	}
	if target.IsTest {
		return // Test sources don't count for coverage.
	}
	for _, path := range target.AllSourcePaths(state.Graph) {
		extension := filepath.Ext(path)
		for _, ext := range state.Config.Cover.FileExtension {
			if ext == extension {
				allFiles[path] = target.IsTest || target.TestOnly // Skip test source files from actual coverage display
				break
			}
		}
	}
}

// mergeCoverage merges recorded coverage with the list of all existing files.
func mergeCoverage(state *core.BuildState, recordedCoverage core.TestCoverage, coveragePackages, allFiles map[string]bool) {
	for file, coverage := range recordedCoverage.Files {
		if isOwnedBy(file, coveragePackages) {
			state.Coverage.Files[file] = coverage
			allFiles[file] = true
		}
	}
	// For any files left over now, enter them in as 100% uncovered.
	// This is pessimistic but there's not much we can do at this point.
	for file, done := range allFiles {
		if !done {
			s := make([]core.LineCoverage, countLines(file))
			if len(s) > 0 {
				for i := 0; i < len(s); i++ {
					s[i] = core.Uncovered
				}
				state.Coverage.Files[file] = s
			}
		}
	}
}

// isOwnedBy returns true if the given file is owned by any of the given packages.
func isOwnedBy(file string, coveragePackages map[string]bool) bool {
	for file != "." {
		file = path.Dir(file)
		if coveragePackages[file] {
			return true
		}
	}
	return false
}

// countLines returns the number of lines in a file.
func countLines(path string) int {
	data, _ := ioutil.ReadFile(path)
	return bytes.Count(data, []byte{'\n'})
}

// WriteCoverageToFileOrDie writes the collected coverage data to a file in JSON format. Dies on failure.
func WriteCoverageToFileOrDie(coverage core.TestCoverage, filename string) {
	out := jsonCoverage{Tests: map[string]map[string]string{}}
	for label, coverage := range coverage.Tests {
		out.Tests[label.String()] = convertCoverage(coverage)
	}
	out.Files = convertCoverage(coverage.Files)
	if b, err := json.MarshalIndent(out, "", "    "); err != nil {
		log.Fatalf("Failed to encode json: %s", err)
	} else if err := ioutil.WriteFile(filename, b, 0644); err != nil {
		log.Fatalf("Failed to write coverage results to %s: %s", filename, err)
	}
}

func convertCoverage(in map[string][]core.LineCoverage) map[string]string {
	ret := map[string]string{}
	for k, v := range in {
		ret[k] = core.TestCoverageString(v)
	}
	return ret
}

// Used to prepare core.TestCoverage objects for JSON marshalling.
type jsonCoverage struct {
	Tests map[string]map[string]string `json:"tests"`
	Files map[string]string            `json:"files"`
}

// RemoveFilesFromCoverage removes any files with extensions matching the given set from coverage.
func RemoveFilesFromCoverage(coverage core.TestCoverage, extensions []string) {
	for _, files := range coverage.Tests {
		removeFilesFromCoverage(files, extensions)
	}
	removeFilesFromCoverage(coverage.Files, extensions)
}

func removeFilesFromCoverage(files map[string][]core.LineCoverage, extensions []string) {
	for filename := range files {
		for _, ext := range extensions {
			if strings.HasSuffix(filename, ext) {
				delete(files, filename)
			}
		}
	}
}
<|endoftext|>


<commit_before>package flow

import (
	"testing"
)

// Tests run-time process creating with flow.Factory
func TestFactory(t *testing.T) {
	procs := make(map[string]interface{})
	procs["e1"] = Factory("echoer")
	in, out := make(chan int), make(chan int)
	e1 := procs["e1"].(*echoer)
	e1.In = in
	e1.Out = out
	RunProc(procs["e1"])
	for i := 0; i < 10; i++ {
		in <- i
		i2 := <-out
		if i != i2 {
			t.Errorf("%d != %d", i2, i)
		}
	}
	// Shutdown proc
	close(in)
}

type dummyNet struct {
	Graph
}

// Tests connection between 2 processes created at run-time
func TestFactoryConnection(t *testing.T) {
	net := new(dummyNet)
	net.InitGraphState()

	net.AddNew("echoer", "e1")
	net.AddNew("echoer", "e2")

	net.Connect("e1", "Out", "e2", "In")

	net.MapInPort("In", "e1", "In")
	net.MapOutPort("Out", "e2", "Out")

	in, out := make(chan int), make(chan int)

	net.SetInPort("In", in)
	net.SetOutPort("Out", out)

	RunNet(net)

	in <- 123
	i := <-out

	close(in)

	if i != 123 {
		t.Errorf("Error: %d != 123", i)
	}
}
<commit_msg>Added a test for creating subgraphs as factory components at run-time.<commit_after>package flow

import (
	"testing"
)

// Tests run-time process creating with flow.Factory
func TestFactory(t *testing.T) {
	procs := make(map[string]interface{})
	procs["e1"] = Factory("echoer")
	in, out := make(chan int), make(chan int)
	e1 := procs["e1"].(*echoer)
	e1.In = in
	e1.Out = out
	RunProc(procs["e1"])
	for i := 0; i < 10; i++ {
		in <- i
		i2 := <-out
		if i != i2 {
			t.Errorf("%d != %d", i2, i)
		}
	}
	// Shutdown proc
	close(in)
}

type dummyNet struct {
	Graph
}

// Tests connection between 2 processes created at run-time
func TestFactoryConnection(t *testing.T) {
	net := new(dummyNet)
	net.InitGraphState()

	net.AddNew("echoer", "e1")
	net.AddNew("echoer", "e2")

	net.Connect("e1", "Out", "e2", "In")

	net.MapInPort("In", "e1", "In")
	net.MapOutPort("Out", "e2", "Out")

	in, out := make(chan int), make(chan int)

	net.SetInPort("In", in)
	net.SetOutPort("Out", out)

	RunNet(net)

	in <- 123
	i := <-out

	close(in)

	if i != 123 {
		t.Errorf("Error: %d != 123", i)
	}
}

// Creates a graph that will be loaded at run-time
func newDummyNet() interface{} {
	net := new(dummyNet)
	net.InitGraphState()

	net.AddNew("echoer", "e1")
	net.AddNew("echoer", "e2")

	net.Connect("e1", "Out", "e2", "In")

	net.MapInPort("In", "e1", "In")
	net.MapOutPort("Out", "e2", "Out")

	return net
}

func init() {
	Register("DummyNet", newDummyNet)
}

// Tests adding subgraph components at run-time
func TestFactorySubgraph(t *testing.T) {
	net := new(dummyNet)
	net.InitGraphState()

	net.AddNew("DummyNet", "d1")

	net.MapInPort("In", "d1", "In")
	net.MapOutPort("Out", "d1", "Out")

	in, out := make(chan int), make(chan int)

	net.SetInPort("In", in)
	net.SetOutPort("Out", out)

	RunNet(net)

	in <- 123
	i := <-out

	close(in)

	if i != 123 {
		t.Errorf("Error: %d != 123", i)
	}
}
<|endoftext|>