<commit_before>package pkg

import (
	"concierge/config"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/parnurzeal/gorequest"
	"net/http"
	"net/url"
	"time"
)

var client *LookerClient

type LookerClient struct {
	accessToken         string
	accessTokenExpireAt int64
	baseUrl             string
	clientId            string
	clientSecret        string
}

type LookerPatchUserRequest struct {
	IsDisabled bool `json:"is_disabled"`
}

type LookerPatchUserResponse struct {
	IsDisabled bool `json:"is_disabled"`
}

type LookerSearchUserRequest struct {
	Email string `json:"email"`
}

type LookerSearchUserResponse struct {
	Id         int  `json:"id"`
	IsDisabled bool `json:"is_disabled"`
}

func GetLookerClient() *LookerClient {
	if client == nil {
		client = &LookerClient{
			accessToken:         "",
			accessTokenExpireAt: 0,
			baseUrl:             config.LookerConfig.BaseUrl,
			clientId:            config.LookerConfig.ClientId,
			clientSecret:        config.LookerConfig.ClientSecret,
		}
	}
	return client
}

func (c *LookerClient) PatchUser(userId int, req LookerPatchUserRequest) (*LookerPatchUserResponse, error) {
	path := fmt.Sprintf("api/3.1/users/%d", userId)
	method := http.MethodPatch
	body := struct {
		IsDisabled bool `json:"is_disabled"`
	}{req.IsDisabled}

	resp := LookerPatchUserResponse{}

	httpResponse, httpErr := c.executeRequest(path, method, body)

	if httpErr != nil {
		return nil, httpErr
	}

	if decodeErr := json.NewDecoder(httpResponse.Body).Decode(&resp); decodeErr != nil {
		return nil, decodeErr
	}
	return &resp, nil
}

func (c *LookerClient) SearchUser(req LookerSearchUserRequest) ([]LookerSearchUserResponse, error) {
	var resp []LookerSearchUserResponse

	path := fmt.Sprintf("api/3.1/users/search?email=%s", url.QueryEscape(req.Email))

	httpResponse, httpErr := c.executeRequest(path, http.MethodGet, nil)

	if httpErr != nil {
		return nil, httpErr
	}

	if decodeErr := json.NewDecoder(httpResponse.Body).Decode(&resp); decodeErr != nil {
		return nil, decodeErr
	}

	return resp, nil
}

func (c *LookerClient) isAccessTokenExpired() bool {
	return time.Now().Unix() >= c.accessTokenExpireAt
}

func (c *LookerClient) setAccessToken() error {
	requestBody := struct {
		ClientId     string `json:"client_id"`
		ClientSecret string `json:"client_secret"`
	}{c.clientId, c.clientSecret}

	requestJson, _ := json.Marshal(&requestBody)

	requestString := string(requestJson)

	httpResponse, _, errs := gorequest.New().
		Post(c.baseUrl + "login").
		Type(gorequest.TypeForm).
		Send(requestString).
		End()

	if errs != nil {
		return errs[0]
	}

	if httpResponse.StatusCode >= 400 {
		return errors.New("failed to enable user on looker")
	}

	response := struct {
		AccessToken string `json:"access_token"`
		ExpiresIn   int64  `json:"expires_in"`
	}{}

	if decodeErr := json.NewDecoder(httpResponse.Body).Decode(&response); decodeErr != nil {
		return decodeErr
	}

	c.accessToken = response.AccessToken
	c.accessTokenExpireAt = time.Now().Unix() + response.ExpiresIn

	return nil
}

func (c *LookerClient) executeRequest(path string, method string, body interface{}) (*http.Response, error) {
	if c.isAccessTokenExpired() {
		if err := c.setAccessToken(); err != nil {
			return nil, err
		}
	}

	url := c.baseUrl + path

	req := gorequest.New()

	switch method {
	case http.MethodGet:
		req = req.Get(url)
	case http.MethodPatch:
		req = req.Patch(url).Send(body)
	case http.MethodPost:
		req = req.Post(url).Send(body)
	}

	req.Header.Set("Authorization", "Bearer token "+c.accessToken)

	resp, _, errs := req.End()

	if len(errs) > 0 {
		return nil, errs[0]
	}

	if resp.StatusCode >= 400 {
		return nil, errors.New("request failed. please contact admins")
	}

	return resp, nil

}
<commit_msg>code review:<commit_after>package pkg

import (
	"concierge/config"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/parnurzeal/gorequest"
	"net/http"
	"net/url"
	"time"
)

var client *LookerClient

type LookerClient struct {
	accessToken         string
	accessTokenExpireAt int64
	baseUrl             string
	clientId            string
	clientSecret        string
}

type LookerPatchUserRequest struct {
	IsDisabled bool `json:"is_disabled"`
}

type LookerPatchUserResponse struct {
	IsDisabled bool `json:"is_disabled"`
}

type LookerSearchUserRequest struct {
	Email string `json:"email"`
}

type LookerSearchUserResponse struct {
	Id         int  `json:"id"`
	IsDisabled bool `json:"is_disabled"`
}

func GetLookerClient() *LookerClient {
	if client == nil {
		client = &LookerClient{
			accessToken:         "",
			accessTokenExpireAt: 0,
			baseUrl:             config.LookerConfig.BaseUrl,
			clientId:            config.LookerConfig.ClientId,
			clientSecret:        config.LookerConfig.ClientSecret,
		}
	}
	return client
}

func (c *LookerClient) PatchUser(userId int, req LookerPatchUserRequest) (*LookerPatchUserResponse, error) {
	path := fmt.Sprintf("api/3.1/users/%d", userId)
	method := http.MethodPatch
	body := struct {
		IsDisabled bool `json:"is_disabled"`
	}{req.IsDisabled}

	resp := LookerPatchUserResponse{}

	httpResponse, httpErr := c.executeRequest(path, method, body)

	if httpErr != nil {
		return nil, httpErr
	}

	if decodeErr := json.NewDecoder(httpResponse.Body).Decode(&resp); decodeErr != nil {
		return nil, decodeErr
	}
	return &resp, nil
}

func (c *LookerClient) SearchUser(req LookerSearchUserRequest) ([]LookerSearchUserResponse, error) {
	var resp []LookerSearchUserResponse

	path := fmt.Sprintf("api/3.1/users/search?email=%s", url.QueryEscape(req.Email))

	httpResponse, httpErr := c.executeRequest(path, http.MethodGet, nil)

	if httpErr != nil {
		return nil, httpErr
	}

	if decodeErr := json.NewDecoder(httpResponse.Body).Decode(&resp); decodeErr != nil {
		return nil, decodeErr
	}

	return resp, nil
}

func (c *LookerClient) isAccessTokenExpired() bool {
	return time.Now().Unix() >= c.accessTokenExpireAt
}

func (c *LookerClient) setAccessToken() error {
	requestBody := struct {
		ClientId     string `json:"client_id"`
		ClientSecret string `json:"client_secret"`
	}{c.clientId, c.clientSecret}

	requestJson, _ := json.Marshal(&requestBody)

	requestString := string(requestJson)

	httpResponse, _, errs := gorequest.New().
		Post(c.baseUrl + "login").
		Type(gorequest.TypeForm).
		Send(requestString).
		End()

	if errs != nil {
		return errs[0]
	}

	if err := getErrorFromResponseIfApplicable(httpResponse); err != nil {
		return err
	}

	response := struct {
		AccessToken string `json:"access_token"`
		ExpiresIn   int64  `json:"expires_in"`
	}{}

	if decodeErr := json.NewDecoder(httpResponse.Body).Decode(&response); decodeErr != nil {
		return decodeErr
	}

	c.accessToken = response.AccessToken
	c.accessTokenExpireAt = time.Now().Unix() + response.ExpiresIn

	return nil
}

func (c *LookerClient) executeRequest(path string, method string, body interface{}) (*http.Response, error) {
	if c.isAccessTokenExpired() {
		if err := c.setAccessToken(); err != nil {
			return nil, err
		}
	}

	url := c.baseUrl + path

	req := gorequest.New()

	switch method {
	case http.MethodGet:
		req = req.Get(url)
	case http.MethodPatch:
		req = req.Patch(url).Send(body)
	case http.MethodPost:
		req = req.Post(url).Send(body)
	}

	req.Header.Set("Authorization", "Bearer token "+c.accessToken)

	resp, _, errs := req.End()

	if len(errs) > 0 {
		return nil, errs[0]
	}

	if err := getErrorFromResponseIfApplicable(resp); err != nil {
		return nil, err
	}

	return resp, nil

}

func getErrorFromResponseIfApplicable(response gorequest.Response) error {
	if response.StatusCode >= 500 {
		return errors.New("there was an error contacting looker. please try after sometime. If error persists, " +
			"please contact looker admins")
	}

	if response.StatusCode >= 400 {
		return errors.New("failed to perform action on looker. please contact concierge admins")
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build darwin freebsd linux netbsd openbsd

package net

import (
	"io"
	"os"
	"sync"
	"syscall"
	"time"
)

// Network file descriptor.
type netFD struct {
	// locking/lifetime of sysfd
	sysmu  sync.Mutex
	sysref int

	// must lock both sysmu and pollDesc to write
	// can lock either to read
	closing bool

	// immutable until Close
	sysfd       int
	family      int
	sotype      int
	isConnected bool
	sysfile     *os.File
	net         string
	laddr       Addr
	raddr       Addr

	// serialize access to Read and Write methods
	rio, wio sync.Mutex

	// wait server
	pd pollDesc
}

func dialTimeout(net, addr string, timeout time.Duration) (Conn, error) {
	deadline := time.Now().Add(timeout)
	ra, err := resolveAddr("dial", net, addr, deadline)
	if err != nil {
		return nil, err
	}
	return dial(net, addr, noLocalAddr, ra, deadline)
}

func newFD(fd, family, sotype int, net string) (*netFD, error) {
	netfd := &netFD{
		sysfd:  fd,
		family: family,
		sotype: sotype,
		net:    net,
	}
	if err := netfd.pd.Init(netfd); err != nil {
		return nil, err
	}
	return netfd, nil
}

func (fd *netFD) setAddr(laddr, raddr Addr) {
	fd.laddr = laddr
	fd.raddr = raddr
	fd.sysfile = os.NewFile(uintptr(fd.sysfd), fd.net)
}

func (fd *netFD) name() string {
	var ls, rs string
	if fd.laddr != nil {
		ls = fd.laddr.String()
	}
	if fd.raddr != nil {
		rs = fd.raddr.String()
	}
	return fd.net + ":" + ls + "->" + rs
}

func (fd *netFD) connect(ra syscall.Sockaddr) error {
	fd.wio.Lock()
	defer fd.wio.Unlock()
	if err := fd.pd.PrepareWrite(); err != nil {
		return err
	}
	err := syscall.Connect(fd.sysfd, ra)
	if err == syscall.EINPROGRESS {
		if err = fd.pd.WaitWrite(); err != nil {
			return err
		}
		var e int
		e, err = syscall.GetsockoptInt(fd.sysfd, syscall.SOL_SOCKET, syscall.SO_ERROR)
		if err != nil {
			return os.NewSyscallError("getsockopt", err)
		}
		if e != 0 {
			err = syscall.Errno(e)
		}
	}
	return err
}

// Add a reference to this fd.
// If closing==true, pollDesc must be locked; mark the fd as closing.
// Returns an error if the fd cannot be used.
func (fd *netFD) incref(closing bool) error {
	fd.sysmu.Lock()
	if fd.closing {
		fd.sysmu.Unlock()
		return errClosing
	}
	fd.sysref++
	if closing {
		fd.closing = true
	}
	fd.sysmu.Unlock()
	return nil
}

// Remove a reference to this FD and close if we've been asked to do so (and
// there are no references left.
func (fd *netFD) decref() {
	fd.sysmu.Lock()
	fd.sysref--
	if fd.closing && fd.sysref == 0 && fd.sysfile != nil {
		fd.sysfile.Close()
		fd.pd.Close()
		fd.sysfile = nil
		fd.sysfd = -1
	}
	fd.sysmu.Unlock()
}

func (fd *netFD) Close() error {
	fd.pd.Lock() // needed for both fd.incref(true) and pollDesc.Evict
	if err := fd.incref(true); err != nil {
		fd.pd.Unlock()
		return err
	}
	// Unblock any I/O.  Once it all unblocks and returns,
	// so that it cannot be referring to fd.sysfd anymore,
	// the final decref will close fd.sysfd.  This should happen
	// fairly quickly, since all the I/O is non-blocking, and any
	// attempts to block in the pollDesc will return errClosing.
	doWakeup := fd.pd.Evict()
	fd.pd.Unlock()
	fd.decref()
	if doWakeup {
		fd.pd.Wakeup()
	}
	return nil
}

func (fd *netFD) shutdown(how int) error {
	if err := fd.incref(false); err != nil {
		return err
	}
	defer fd.decref()
	err := syscall.Shutdown(fd.sysfd, how)
	if err != nil {
		return &OpError{"shutdown", fd.net, fd.laddr, err}
	}
	return nil
}

func (fd *netFD) CloseRead() error {
	return fd.shutdown(syscall.SHUT_RD)
}

func (fd *netFD) CloseWrite() error {
	return fd.shutdown(syscall.SHUT_WR)
}

func (fd *netFD) Read(p []byte) (n int, err error) {
	fd.rio.Lock()
	defer fd.rio.Unlock()
	if err := fd.incref(false); err != nil {
		return 0, err
	}
	defer fd.decref()
	if err := fd.pd.PrepareRead(); err != nil {
		return 0, &OpError{"read", fd.net, fd.raddr, err}
	}
	for {
		n, err = syscall.Read(int(fd.sysfd), p)
		if err != nil {
			n = 0
			if err == syscall.EAGAIN {
				if err = fd.pd.WaitRead(); err == nil {
					continue
				}
			}
		}
		err = chkReadErr(n, err, fd)
		break
	}
	if err != nil && err != io.EOF {
		err = &OpError{"read", fd.net, fd.raddr, err}
	}
	return
}

func (fd *netFD) ReadFrom(p []byte) (n int, sa syscall.Sockaddr, err error) {
	fd.rio.Lock()
	defer fd.rio.Unlock()
	if err := fd.incref(false); err != nil {
		return 0, nil, err
	}
	defer fd.decref()
	if err := fd.pd.PrepareRead(); err != nil {
		return 0, nil, &OpError{"read", fd.net, fd.laddr, err}
	}
	for {
		n, sa, err = syscall.Recvfrom(fd.sysfd, p, 0)
		if err != nil {
			n = 0
			if err == syscall.EAGAIN {
				if err = fd.pd.WaitRead(); err == nil {
					continue
				}
			}
		}
		err = chkReadErr(n, err, fd)
		break
	}
	if err != nil && err != io.EOF {
		err = &OpError{"read", fd.net, fd.laddr, err}
	}
	return
}

func (fd *netFD) ReadMsg(p []byte, oob []byte) (n, oobn, flags int, sa syscall.Sockaddr, err error) {
	fd.rio.Lock()
	defer fd.rio.Unlock()
	if err := fd.incref(false); err != nil {
		return 0, 0, 0, nil, err
	}
	defer fd.decref()
	if err := fd.pd.PrepareRead(); err != nil {
		return 0, 0, 0, nil, &OpError{"read", fd.net, fd.laddr, err}
	}
	for {
		n, oobn, flags, sa, err = syscall.Recvmsg(fd.sysfd, p, oob, 0)
		if err != nil {
			// TODO(dfc) should n and oobn be set to 0
			if err == syscall.EAGAIN {
				if err = fd.pd.WaitRead(); err == nil {
					continue
				}
			}
		}
		err = chkReadErr(n, err, fd)
		break
	}
	if err != nil && err != io.EOF {
		err = &OpError{"read", fd.net, fd.laddr, err}
	}
	return
}

func chkReadErr(n int, err error, fd *netFD) error {
	if n == 0 && err == nil && fd.sotype != syscall.SOCK_DGRAM && fd.sotype != syscall.SOCK_RAW {
		return io.EOF
	}
	return err
}

func (fd *netFD) Write(p []byte) (nn int, err error) {
	fd.wio.Lock()
	defer fd.wio.Unlock()
	if err := fd.incref(false); err != nil {
		return 0, err
	}
	defer fd.decref()
	if err := fd.pd.PrepareWrite(); err != nil {
		return 0, &OpError{"write", fd.net, fd.raddr, err}
	}
	for {
		var n int
		n, err = syscall.Write(int(fd.sysfd), p[nn:])
		if n > 0 {
			nn += n
		}
		if nn == len(p) {
			break
		}
		if err == syscall.EAGAIN {
			if err = fd.pd.WaitWrite(); err == nil {
				continue
			}
		}
		if err != nil {
			n = 0
			break
		}
		if n == 0 {
			err = io.ErrUnexpectedEOF
			break
		}
	}
	if err != nil {
		err = &OpError{"write", fd.net, fd.raddr, err}
	}
	return nn, err
}

func (fd *netFD) WriteTo(p []byte, sa syscall.Sockaddr) (n int, err error) {
	fd.wio.Lock()
	defer fd.wio.Unlock()
	if err := fd.incref(false); err != nil {
		return 0, err
	}
	defer fd.decref()
	if err := fd.pd.PrepareWrite(); err != nil {
		return 0, &OpError{"write", fd.net, fd.raddr, err}
	}
	for {
		err = syscall.Sendto(fd.sysfd, p, 0, sa)
		if err == syscall.EAGAIN {
			if err = fd.pd.WaitWrite(); err == nil {
				continue
			}
		}
		break
	}
	if err == nil {
		n = len(p)
	} else {
		err = &OpError{"write", fd.net, fd.raddr, err}
	}
	return
}

func (fd *netFD) WriteMsg(p []byte, oob []byte, sa syscall.Sockaddr) (n int, oobn int, err error) {
	fd.wio.Lock()
	defer fd.wio.Unlock()
	if err := fd.incref(false); err != nil {
		return 0, 0, err
	}
	defer fd.decref()
	if err := fd.pd.PrepareWrite(); err != nil {
		return 0, 0, &OpError{"write", fd.net, fd.raddr, err}
	}
	for {
		err = syscall.Sendmsg(fd.sysfd, p, oob, sa, 0)
		if err == syscall.EAGAIN {
			if err = fd.pd.WaitWrite(); err == nil {
				continue
			}
		}
		break
	}
	if err == nil {
		n = len(p)
		oobn = len(oob)
	} else {
		err = &OpError{"write", fd.net, fd.raddr, err}
	}
	return
}

func (fd *netFD) accept(toAddr func(syscall.Sockaddr) Addr) (netfd *netFD, err error) {
	fd.rio.Lock()
	defer fd.rio.Unlock()
	if err := fd.incref(false); err != nil {
		return nil, err
	}
	defer fd.decref()

	var s int
	var rsa syscall.Sockaddr
	if err = fd.pd.PrepareRead(); err != nil {
		return nil, &OpError{"accept", fd.net, fd.laddr, err}
	}
	for {
		s, rsa, err = accept(fd.sysfd)
		if err != nil {
			if err == syscall.EAGAIN {
				if err = fd.pd.WaitRead(); err == nil {
					continue
				}
			} else if err == syscall.ECONNABORTED {
				// This means that a socket on the listen queue was closed
				// before we Accept()ed it; it's a silly error, so try again.
				continue
			}
			return nil, &OpError{"accept", fd.net, fd.laddr, err}
		}
		break
	}

	if netfd, err = newFD(s, fd.family, fd.sotype, fd.net); err != nil {
		closesocket(s)
		return nil, err
	}
	lsa, _ := syscall.Getsockname(netfd.sysfd)
	netfd.setAddr(toAddr(lsa), toAddr(rsa))
	return netfd, nil
}

func (fd *netFD) dup() (f *os.File, err error) {
	syscall.ForkLock.RLock()
	ns, err := syscall.Dup(fd.sysfd)
	if err != nil {
		syscall.ForkLock.RUnlock()
		return nil, &OpError{"dup", fd.net, fd.laddr, err}
	}
	syscall.CloseOnExec(ns)
	syscall.ForkLock.RUnlock()

	// We want blocking mode for the new fd, hence the double negative.
	// This also puts the old fd into blocking mode, meaning that
	// I/O will block the thread instead of letting us use the epoll server.
	// Everything will still work, just with more threads.
	if err = syscall.SetNonblock(ns, false); err != nil {
		return nil, &OpError{"setnonblock", fd.net, fd.laddr, err}
	}

	return os.NewFile(uintptr(ns), fd.name()), nil
}

func closesocket(s int) error {
	return syscall.Close(s)
}
<commit_msg>net: prepare connect() for new network poller The problem is that new network poller can have spurious rediness notifications. This implementation ensures that the socket is actually connected.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build darwin freebsd linux netbsd openbsd

package net

import (
	"io"
	"os"
	"sync"
	"syscall"
	"time"
)

// Network file descriptor.
type netFD struct {
	// locking/lifetime of sysfd
	sysmu  sync.Mutex
	sysref int

	// must lock both sysmu and pollDesc to write
	// can lock either to read
	closing bool

	// immutable until Close
	sysfd       int
	family      int
	sotype      int
	isConnected bool
	sysfile     *os.File
	net         string
	laddr       Addr
	raddr       Addr

	// serialize access to Read and Write methods
	rio, wio sync.Mutex

	// wait server
	pd pollDesc
}

func dialTimeout(net, addr string, timeout time.Duration) (Conn, error) {
	deadline := time.Now().Add(timeout)
	ra, err := resolveAddr("dial", net, addr, deadline)
	if err != nil {
		return nil, err
	}
	return dial(net, addr, noLocalAddr, ra, deadline)
}

func newFD(fd, family, sotype int, net string) (*netFD, error) {
	netfd := &netFD{
		sysfd:  fd,
		family: family,
		sotype: sotype,
		net:    net,
	}
	if err := netfd.pd.Init(netfd); err != nil {
		return nil, err
	}
	return netfd, nil
}

func (fd *netFD) setAddr(laddr, raddr Addr) {
	fd.laddr = laddr
	fd.raddr = raddr
	fd.sysfile = os.NewFile(uintptr(fd.sysfd), fd.net)
}

func (fd *netFD) name() string {
	var ls, rs string
	if fd.laddr != nil {
		ls = fd.laddr.String()
	}
	if fd.raddr != nil {
		rs = fd.raddr.String()
	}
	return fd.net + ":" + ls + "->" + rs
}

func (fd *netFD) connect(ra syscall.Sockaddr) error {
	fd.wio.Lock()
	defer fd.wio.Unlock()
	if err := fd.pd.PrepareWrite(); err != nil {
		return err
	}
	for {
		err := syscall.Connect(fd.sysfd, ra)
		if err == nil || err == syscall.EISCONN {
			break
		}
		if err != syscall.EINPROGRESS && err != syscall.EALREADY && err != syscall.EINTR {
			return err
		}
		if err = fd.pd.WaitWrite(); err != nil {
			return err
		}
	}
	return nil
}

// Add a reference to this fd.
// If closing==true, pollDesc must be locked; mark the fd as closing.
// Returns an error if the fd cannot be used.
func (fd *netFD) incref(closing bool) error {
	fd.sysmu.Lock()
	if fd.closing {
		fd.sysmu.Unlock()
		return errClosing
	}
	fd.sysref++
	if closing {
		fd.closing = true
	}
	fd.sysmu.Unlock()
	return nil
}

// Remove a reference to this FD and close if we've been asked to do so (and
// there are no references left.
func (fd *netFD) decref() {
	fd.sysmu.Lock()
	fd.sysref--
	if fd.closing && fd.sysref == 0 && fd.sysfile != nil {
		fd.sysfile.Close()
		fd.pd.Close()
		fd.sysfile = nil
		fd.sysfd = -1
	}
	fd.sysmu.Unlock()
}

func (fd *netFD) Close() error {
	fd.pd.Lock() // needed for both fd.incref(true) and pollDesc.Evict
	if err := fd.incref(true); err != nil {
		fd.pd.Unlock()
		return err
	}
	// Unblock any I/O.  Once it all unblocks and returns,
	// so that it cannot be referring to fd.sysfd anymore,
	// the final decref will close fd.sysfd.  This should happen
	// fairly quickly, since all the I/O is non-blocking, and any
	// attempts to block in the pollDesc will return errClosing.
	doWakeup := fd.pd.Evict()
	fd.pd.Unlock()
	fd.decref()
	if doWakeup {
		fd.pd.Wakeup()
	}
	return nil
}

func (fd *netFD) shutdown(how int) error {
	if err := fd.incref(false); err != nil {
		return err
	}
	defer fd.decref()
	err := syscall.Shutdown(fd.sysfd, how)
	if err != nil {
		return &OpError{"shutdown", fd.net, fd.laddr, err}
	}
	return nil
}

func (fd *netFD) CloseRead() error {
	return fd.shutdown(syscall.SHUT_RD)
}

func (fd *netFD) CloseWrite() error {
	return fd.shutdown(syscall.SHUT_WR)
}

func (fd *netFD) Read(p []byte) (n int, err error) {
	fd.rio.Lock()
	defer fd.rio.Unlock()
	if err := fd.incref(false); err != nil {
		return 0, err
	}
	defer fd.decref()
	if err := fd.pd.PrepareRead(); err != nil {
		return 0, &OpError{"read", fd.net, fd.raddr, err}
	}
	for {
		n, err = syscall.Read(int(fd.sysfd), p)
		if err != nil {
			n = 0
			if err == syscall.EAGAIN {
				if err = fd.pd.WaitRead(); err == nil {
					continue
				}
			}
		}
		err = chkReadErr(n, err, fd)
		break
	}
	if err != nil && err != io.EOF {
		err = &OpError{"read", fd.net, fd.raddr, err}
	}
	return
}

func (fd *netFD) ReadFrom(p []byte) (n int, sa syscall.Sockaddr, err error) {
	fd.rio.Lock()
	defer fd.rio.Unlock()
	if err := fd.incref(false); err != nil {
		return 0, nil, err
	}
	defer fd.decref()
	if err := fd.pd.PrepareRead(); err != nil {
		return 0, nil, &OpError{"read", fd.net, fd.laddr, err}
	}
	for {
		n, sa, err = syscall.Recvfrom(fd.sysfd, p, 0)
		if err != nil {
			n = 0
			if err == syscall.EAGAIN {
				if err = fd.pd.WaitRead(); err == nil {
					continue
				}
			}
		}
		err = chkReadErr(n, err, fd)
		break
	}
	if err != nil && err != io.EOF {
		err = &OpError{"read", fd.net, fd.laddr, err}
	}
	return
}

func (fd *netFD) ReadMsg(p []byte, oob []byte) (n, oobn, flags int, sa syscall.Sockaddr, err error) {
	fd.rio.Lock()
	defer fd.rio.Unlock()
	if err := fd.incref(false); err != nil {
		return 0, 0, 0, nil, err
	}
	defer fd.decref()
	if err := fd.pd.PrepareRead(); err != nil {
		return 0, 0, 0, nil, &OpError{"read", fd.net, fd.laddr, err}
	}
	for {
		n, oobn, flags, sa, err = syscall.Recvmsg(fd.sysfd, p, oob, 0)
		if err != nil {
			// TODO(dfc) should n and oobn be set to 0
			if err == syscall.EAGAIN {
				if err = fd.pd.WaitRead(); err == nil {
					continue
				}
			}
		}
		err = chkReadErr(n, err, fd)
		break
	}
	if err != nil && err != io.EOF {
		err = &OpError{"read", fd.net, fd.laddr, err}
	}
	return
}

func chkReadErr(n int, err error, fd *netFD) error {
	if n == 0 && err == nil && fd.sotype != syscall.SOCK_DGRAM && fd.sotype != syscall.SOCK_RAW {
		return io.EOF
	}
	return err
}

func (fd *netFD) Write(p []byte) (nn int, err error) {
	fd.wio.Lock()
	defer fd.wio.Unlock()
	if err := fd.incref(false); err != nil {
		return 0, err
	}
	defer fd.decref()
	if err := fd.pd.PrepareWrite(); err != nil {
		return 0, &OpError{"write", fd.net, fd.raddr, err}
	}
	for {
		var n int
		n, err = syscall.Write(int(fd.sysfd), p[nn:])
		if n > 0 {
			nn += n
		}
		if nn == len(p) {
			break
		}
		if err == syscall.EAGAIN {
			if err = fd.pd.WaitWrite(); err == nil {
				continue
			}
		}
		if err != nil {
			n = 0
			break
		}
		if n == 0 {
			err = io.ErrUnexpectedEOF
			break
		}
	}
	if err != nil {
		err = &OpError{"write", fd.net, fd.raddr, err}
	}
	return nn, err
}

func (fd *netFD) WriteTo(p []byte, sa syscall.Sockaddr) (n int, err error) {
	fd.wio.Lock()
	defer fd.wio.Unlock()
	if err := fd.incref(false); err != nil {
		return 0, err
	}
	defer fd.decref()
	if err := fd.pd.PrepareWrite(); err != nil {
		return 0, &OpError{"write", fd.net, fd.raddr, err}
	}
	for {
		err = syscall.Sendto(fd.sysfd, p, 0, sa)
		if err == syscall.EAGAIN {
			if err = fd.pd.WaitWrite(); err == nil {
				continue
			}
		}
		break
	}
	if err == nil {
		n = len(p)
	} else {
		err = &OpError{"write", fd.net, fd.raddr, err}
	}
	return
}

func (fd *netFD) WriteMsg(p []byte, oob []byte, sa syscall.Sockaddr) (n int, oobn int, err error) {
	fd.wio.Lock()
	defer fd.wio.Unlock()
	if err := fd.incref(false); err != nil {
		return 0, 0, err
	}
	defer fd.decref()
	if err := fd.pd.PrepareWrite(); err != nil {
		return 0, 0, &OpError{"write", fd.net, fd.raddr, err}
	}
	for {
		err = syscall.Sendmsg(fd.sysfd, p, oob, sa, 0)
		if err == syscall.EAGAIN {
			if err = fd.pd.WaitWrite(); err == nil {
				continue
			}
		}
		break
	}
	if err == nil {
		n = len(p)
		oobn = len(oob)
	} else {
		err = &OpError{"write", fd.net, fd.raddr, err}
	}
	return
}

func (fd *netFD) accept(toAddr func(syscall.Sockaddr) Addr) (netfd *netFD, err error) {
	fd.rio.Lock()
	defer fd.rio.Unlock()
	if err := fd.incref(false); err != nil {
		return nil, err
	}
	defer fd.decref()

	var s int
	var rsa syscall.Sockaddr
	if err = fd.pd.PrepareRead(); err != nil {
		return nil, &OpError{"accept", fd.net, fd.laddr, err}
	}
	for {
		s, rsa, err = accept(fd.sysfd)
		if err != nil {
			if err == syscall.EAGAIN {
				if err = fd.pd.WaitRead(); err == nil {
					continue
				}
			} else if err == syscall.ECONNABORTED {
				// This means that a socket on the listen queue was closed
				// before we Accept()ed it; it's a silly error, so try again.
				continue
			}
			return nil, &OpError{"accept", fd.net, fd.laddr, err}
		}
		break
	}

	if netfd, err = newFD(s, fd.family, fd.sotype, fd.net); err != nil {
		closesocket(s)
		return nil, err
	}
	lsa, _ := syscall.Getsockname(netfd.sysfd)
	netfd.setAddr(toAddr(lsa), toAddr(rsa))
	return netfd, nil
}

func (fd *netFD) dup() (f *os.File, err error) {
	syscall.ForkLock.RLock()
	ns, err := syscall.Dup(fd.sysfd)
	if err != nil {
		syscall.ForkLock.RUnlock()
		return nil, &OpError{"dup", fd.net, fd.laddr, err}
	}
	syscall.CloseOnExec(ns)
	syscall.ForkLock.RUnlock()

	// We want blocking mode for the new fd, hence the double negative.
	// This also puts the old fd into blocking mode, meaning that
	// I/O will block the thread instead of letting us use the epoll server.
	// Everything will still work, just with more threads.
	if err = syscall.SetNonblock(ns, false); err != nil {
		return nil, &OpError{"setnonblock", fd.net, fd.laddr, err}
	}

	return os.NewFile(uintptr(ns), fd.name()), nil
}

func closesocket(s int) error {
	return syscall.Close(s)
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2011 Matt Jibson <matt.jibson@gmail.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

// Package fft provides forward and inverse fast Fourier transform functions.
package fft

import (
	"math"
	"sync"

	"github.com/mjibson/go-dsp/dsputils"
)

var (
	radix2Lock    sync.RWMutex
	radix2Factors = map[int][]complex128{
		4: {complex(1, 0), complex(0, -1), complex(-1, 0), complex(0, 1)},
	}
)

var (
	bluesteinLock       sync.RWMutex
	bluesteinFactors    = map[int][]complex128{}
	bluesteinInvFactors = map[int][]complex128{}
)

func getRadix2Factors(input_len int) []complex128 {
	radix2Lock.RLock()

	if hasRadix2Factors(input_len) {
		defer radix2Lock.RUnlock()
		return radix2Factors[input_len]
	}

	radix2Lock.RUnlock()
	radix2Lock.Lock()
	defer radix2Lock.Unlock()

	if !hasRadix2Factors(input_len) {
		for i, p := 8, 4; i <= input_len; i, p = i<<1, i {
			if radix2Factors[i] == nil {
				radix2Factors[i] = make([]complex128, i)

				for n, j := 0, 0; n < i; n, j = n+2, j+1 {
					radix2Factors[i][n] = radix2Factors[p][j]
				}

				for n := 1; n < i; n += 2 {
					sin, cos := math.Sincos(-2 * math.Pi / float64(i) * float64(n))
					radix2Factors[i][n] = complex(cos, sin)
				}
			}
		}
	}

	return radix2Factors[input_len]
}

func hasRadix2Factors(idx int) bool {
	return radix2Factors[idx] != nil
}

func getBluesteinFactors(input_len int) ([]complex128, []complex128) {
	bluesteinLock.RLock()

	if hasBluesteinFactors(input_len) {
		defer bluesteinLock.RUnlock()
		return bluesteinFactors[input_len], bluesteinInvFactors[input_len]
	}

	bluesteinLock.RUnlock()
	bluesteinLock.Lock()
	defer bluesteinLock.Unlock()

	if !hasBluesteinFactors(input_len) {
		bluesteinFactors[input_len] = make([]complex128, input_len)
		bluesteinInvFactors[input_len] = make([]complex128, input_len)

		var sin, cos float64
		for i := 0; i < input_len; i++ {
			if i == 0 {
				sin, cos = 0, 1
			} else {
				sin, cos = math.Sincos(math.Pi / float64(input_len) * float64(i*i))
			}
			bluesteinFactors[input_len][i] = complex(cos, sin)
			bluesteinInvFactors[input_len][i] = complex(cos, -sin)
		}
	}

	return bluesteinFactors[input_len], bluesteinInvFactors[input_len]
}

func hasBluesteinFactors(idx int) bool {
	return bluesteinFactors[idx] != nil
}

// FFTReal returns the forward FFT of the real-valued slice.
func FFTReal(x []float64) []complex128 {
	return FFT(dsputils.ToComplex(x))
}

// IFFTReal returns the inverse FFT of the real-valued slice.
func IFFTReal(x []float64) []complex128 {
	return IFFT(dsputils.ToComplex(x))
}

// IFFT returns the inverse FFT of the complex-valued slice.
func IFFT(x []complex128) []complex128 {
	lx := len(x)
	r := make([]complex128, lx)

	// Reverse inputs, which is calculated with modulo N, hence x[0] as an outlier
	r[0] = x[0]
	for i := 1; i < lx; i++ {
		r[i] = x[lx-i]
	}

	r = FFT(r)

	N := complex(float64(lx), 0)
	for n, _ := range r {
		r[n] /= N
	}
	return r
}

// Convolve returns the convolution of x * y.
func Convolve(x, y []complex128) []complex128 {
	if len(x) != len(y) {
		panic("arrays not of equal size")
	}

	fft_x := FFT(x)
	fft_y := FFT(y)

	r := make([]complex128, len(x))
	for i := 0; i < len(r); i++ {
		r[i] = fft_x[i] * fft_y[i]
	}

	return IFFT(r)
}

// FFT returns the forward FFT of the complex-valued slice.
func FFT(x []complex128) []complex128 {
	lx := len(x)

	// todo: non-hack handling length <= 1 cases
	if lx <= 1 {
		r := make([]complex128, lx)
		copy(r, x)
		return r
	}

	if dsputils.IsPowerOf2(lx) {
		return radix2FFT(x)
	}

	return bluesteinFFT(x)
}

// radix2FFT returns the FFT calculated using the radix-2 DIT Cooley-Tukey algorithm.
func radix2FFT(x []complex128) []complex128 {
	lx := len(x)
	factors := getRadix2Factors(lx)

	lx_2 := lx / 2
	t := make([]complex128, lx) // temp
	r := reorderData(x)

	for stage := 2; stage <= lx; stage <<= 1 {
		if stage == 2 { // 2-point transforms
			for n := 0; n < lx_2; n++ {
				t[n*2] = r[n*2] + r[n*2+1]
				t[n*2+1] = r[n*2] - r[n*2+1]
			}
		} else { // >2-point transforms
			blocks := lx / stage
			s_2 := stage / 2

			for n := 0; n < blocks; n++ {
				nb := n * stage
				for j := 0; j < s_2; j++ {
					w_n := r[j+nb+s_2] * factors[blocks*j]
					t[j+nb] = r[j+nb] + w_n
					t[j+nb+s_2] = r[j+nb] - w_n
				}
			}
		}

		copy(r, t)
	}

	return r
}

// reorderData returns a copy of x reordered for the DFT.
func reorderData(x []complex128) []complex128 {
	lx := uint(len(x))
	r := make([]complex128, lx)
	s := log2(lx)

	var n uint
	for ; n < lx; n++ {
		r[reverseBits(n, s)] = x[n]
	}

	return r
}

// log2 returns the log base 2 of v
// from: http://graphics.stanford.edu/~seander/bithacks.html#IntegerLogObvious
func log2(v uint) uint {
	var r uint

	for v >>= 1; v != 0; v >>= 1 {
		r++
	}

	return r
}

// reverseBits returns the first s bits of v in reverse order
// from: http://graphics.stanford.edu/~seander/bithacks.html#BitReverseObvious
func reverseBits(v, s uint) uint {
	var r uint

	// Since we aren't reversing all the bits in v (just the first s bits),
	// we only need the first bit of v instead of a full copy.
	r = v & 1
	s--

	for v >>= 1; v != 0; v >>= 1 {
		r <<= 1
		r |= v & 1
		s--
	}

	return r << s
}

// bluesteinFFT returns the FFT calculated using the Bluestein algorithm.
func bluesteinFFT(x []complex128) []complex128 {
	lx := len(x)
	a := dsputils.ZeroPad(x, dsputils.NextPowerOf2(lx*2-1))
	la := len(a)
	factors, invFactors := getBluesteinFactors(lx)

	for n, v := range x {
		a[n] = v * invFactors[n]
	}

	b := make([]complex128, la)
	for i := 0; i < lx; i++ {
		b[i] = factors[i]

		if i != 0 {
			b[la-i] = factors[i]
		}
	}

	r := Convolve(a, b)

	for i := 0; i < lx; i++ {
		r[i] *= invFactors[i]
	}

	return r[:lx]
}

// FFT2Real returns the 2-dimensional, forward FFT of the real-valued matrix.
func FFT2Real(x [][]float64) [][]complex128 {
	return FFT2(dsputils.ToComplex2(x))
}

// FFT2 returns the 2-dimensional, forward FFT of the complex-valued matrix.
func FFT2(x [][]complex128) [][]complex128 {
	return computeFFT2(x, FFT)
}

// IFFT2Real returns the 2-dimensional, inverse FFT of the real-valued matrix.
func IFFT2Real(x [][]float64) [][]complex128 {
	return IFFT2(dsputils.ToComplex2(x))
}

// IFFT2 returns the 2-dimensional, inverse FFT of the complex-valued matrix.
func IFFT2(x [][]complex128) [][]complex128 {
	return computeFFT2(x, IFFT)
}

func computeFFT2(x [][]complex128, fftFunc func([]complex128) []complex128) [][]complex128 {
	rows := len(x)
	if rows == 0 {
		panic("empty input array")
	}

	cols := len(x[0])
	r := make([][]complex128, rows)
	for i := 0; i < rows; i++ {
		if len(x[i]) != cols {
			panic("ragged input array")
		}
		r[i] = make([]complex128, cols)
	}

	for i := 0; i < cols; i++ {
		t := make([]complex128, rows)
		for j := 0; j < rows; j++ {
			t[j] = x[j][i]
		}

		for n, v := range fftFunc(t) {
			r[n][i] = v
		}
	}

	for n, v := range r {
		r[n] = fftFunc(v)
	}

	return r
}

// FFTN returns the forward FFT of the matrix m, computed in all N dimensions.
func FFTN(m *dsputils.Matrix) *dsputils.Matrix {
	return computeFFTN(m, FFT)
}

// IFFTN returns the forward FFT of the matrix m, computed in all N dimensions.
func IFFTN(m *dsputils.Matrix) *dsputils.Matrix {
	return computeFFTN(m, IFFT)
}

func computeFFTN(m *dsputils.Matrix, fftFunc func([]complex128) []complex128) *dsputils.Matrix {
	dims := m.Dimensions()
	t := m.Copy()
	r := dsputils.MakeEmptyMatrix(dims)

	for n := range dims {
		dims[n] -= 1
	}

	for n := range dims {
		d := make([]int, len(dims))
		copy(d, dims)
		d[n] = -1

		for {
			r.SetDim(fftFunc(t.Dim(d)), d)

			if !decrDim(d, dims) {
				break
			}
		}

		r, t = t, r
	}

	return t
}

// decrDim decrements an element of x by 1, skipping all -1s, and wrapping up to d.
// If a value is 0, it will be reset to its correspending value in d, and will carry one from the next non -1 value to the right.
// Returns true if decremented, else false.
func decrDim(x, d []int) bool {
	for n, v := range x {
		if v == -1 {
			continue
		} else if v == 0 {
			i := n
			// find the next element to decrement
			for ; i < len(x); i++ {
				if x[i] == -1 {
					continue
				} else if x[i] == 0 {
					x[i] = d[i]
				} else {
					x[i] -= 1
					return true
				}
			}

			// no decrement
			return false
		} else {
			x[n] -= 1
			return true
		}
	}

	return false
}
<commit_msg>Use a temporary swap variable instead of copying slices<commit_after>/*
 * Copyright (c) 2011 Matt Jibson <matt.jibson@gmail.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

// Package fft provides forward and inverse fast Fourier transform functions.
package fft

import (
	"math"
	"sync"

	"github.com/mjibson/go-dsp/dsputils"
)

var (
	radix2Lock    sync.RWMutex
	radix2Factors = map[int][]complex128{
		4: {complex(1, 0), complex(0, -1), complex(-1, 0), complex(0, 1)},
	}
)

var (
	bluesteinLock       sync.RWMutex
	bluesteinFactors    = map[int][]complex128{}
	bluesteinInvFactors = map[int][]complex128{}
)

func getRadix2Factors(input_len int) []complex128 {
	radix2Lock.RLock()

	if hasRadix2Factors(input_len) {
		defer radix2Lock.RUnlock()
		return radix2Factors[input_len]
	}

	radix2Lock.RUnlock()
	radix2Lock.Lock()
	defer radix2Lock.Unlock()

	if !hasRadix2Factors(input_len) {
		for i, p := 8, 4; i <= input_len; i, p = i<<1, i {
			if radix2Factors[i] == nil {
				radix2Factors[i] = make([]complex128, i)

				for n, j := 0, 0; n < i; n, j = n+2, j+1 {
					radix2Factors[i][n] = radix2Factors[p][j]
				}

				for n := 1; n < i; n += 2 {
					sin, cos := math.Sincos(-2 * math.Pi / float64(i) * float64(n))
					radix2Factors[i][n] = complex(cos, sin)
				}
			}
		}
	}

	return radix2Factors[input_len]
}

func hasRadix2Factors(idx int) bool {
	return radix2Factors[idx] != nil
}

func getBluesteinFactors(input_len int) ([]complex128, []complex128) {
	bluesteinLock.RLock()

	if hasBluesteinFactors(input_len) {
		defer bluesteinLock.RUnlock()
		return bluesteinFactors[input_len], bluesteinInvFactors[input_len]
	}

	bluesteinLock.RUnlock()
	bluesteinLock.Lock()
	defer bluesteinLock.Unlock()

	if !hasBluesteinFactors(input_len) {
		bluesteinFactors[input_len] = make([]complex128, input_len)
		bluesteinInvFactors[input_len] = make([]complex128, input_len)

		var sin, cos float64
		for i := 0; i < input_len; i++ {
			if i == 0 {
				sin, cos = 0, 1
			} else {
				sin, cos = math.Sincos(math.Pi / float64(input_len) * float64(i*i))
			}
			bluesteinFactors[input_len][i] = complex(cos, sin)
			bluesteinInvFactors[input_len][i] = complex(cos, -sin)
		}
	}

	return bluesteinFactors[input_len], bluesteinInvFactors[input_len]
}

func hasBluesteinFactors(idx int) bool {
	return bluesteinFactors[idx] != nil
}

// FFTReal returns the forward FFT of the real-valued slice.
func FFTReal(x []float64) []complex128 {
	return FFT(dsputils.ToComplex(x))
}

// IFFTReal returns the inverse FFT of the real-valued slice.
func IFFTReal(x []float64) []complex128 {
	return IFFT(dsputils.ToComplex(x))
}

// IFFT returns the inverse FFT of the complex-valued slice.
func IFFT(x []complex128) []complex128 {
	lx := len(x)
	r := make([]complex128, lx)

	// Reverse inputs, which is calculated with modulo N, hence x[0] as an outlier
	r[0] = x[0]
	for i := 1; i < lx; i++ {
		r[i] = x[lx-i]
	}

	r = FFT(r)

	N := complex(float64(lx), 0)
	for n, _ := range r {
		r[n] /= N
	}
	return r
}

// Convolve returns the convolution of x * y.
func Convolve(x, y []complex128) []complex128 {
	if len(x) != len(y) {
		panic("arrays not of equal size")
	}

	fft_x := FFT(x)
	fft_y := FFT(y)

	r := make([]complex128, len(x))
	for i := 0; i < len(r); i++ {
		r[i] = fft_x[i] * fft_y[i]
	}

	return IFFT(r)
}

// FFT returns the forward FFT of the complex-valued slice.
func FFT(x []complex128) []complex128 {
	lx := len(x)

	// todo: non-hack handling length <= 1 cases
	if lx <= 1 {
		r := make([]complex128, lx)
		copy(r, x)
		return r
	}

	if dsputils.IsPowerOf2(lx) {
		return radix2FFT(x)
	}

	return bluesteinFFT(x)
}

// radix2FFT returns the FFT calculated using the radix-2 DIT Cooley-Tukey algorithm.
func radix2FFT(x []complex128) []complex128 {
	lx := len(x)
	factors := getRadix2Factors(lx)

	lx_2 := lx / 2
	t := make([]complex128, lx) // temp
	r := reorderData(x)
	var swp []complex128

	for stage := 2; stage <= lx; stage <<= 1 {
		if stage == 2 { // 2-point transforms
			for n := 0; n < lx_2; n++ {
				t[n*2] = r[n*2] + r[n*2+1]
				t[n*2+1] = r[n*2] - r[n*2+1]
			}
		} else { // >2-point transforms
			blocks := lx / stage
			s_2 := stage / 2

			for n := 0; n < blocks; n++ {
				nb := n * stage
				for j := 0; j < s_2; j++ {
					w_n := r[j+nb+s_2] * factors[blocks*j]
					t[j+nb] = r[j+nb] + w_n
					t[j+nb+s_2] = r[j+nb] - w_n
				}
			}
		}

		swp = t
		t = r
		r = swp
	}

	return r
}

// reorderData returns a copy of x reordered for the DFT.
func reorderData(x []complex128) []complex128 {
	lx := uint(len(x))
	r := make([]complex128, lx)
	s := log2(lx)

	var n uint
	for ; n < lx; n++ {
		r[reverseBits(n, s)] = x[n]
	}

	return r
}

// log2 returns the log base 2 of v
// from: http://graphics.stanford.edu/~seander/bithacks.html#IntegerLogObvious
func log2(v uint) uint {
	var r uint

	for v >>= 1; v != 0; v >>= 1 {
		r++
	}

	return r
}

// reverseBits returns the first s bits of v in reverse order
// from: http://graphics.stanford.edu/~seander/bithacks.html#BitReverseObvious
func reverseBits(v, s uint) uint {
	var r uint

	// Since we aren't reversing all the bits in v (just the first s bits),
	// we only need the first bit of v instead of a full copy.
	r = v & 1
	s--

	for v >>= 1; v != 0; v >>= 1 {
		r <<= 1
		r |= v & 1
		s--
	}

	return r << s
}

// bluesteinFFT returns the FFT calculated using the Bluestein algorithm.
func bluesteinFFT(x []complex128) []complex128 {
	lx := len(x)
	a := dsputils.ZeroPad(x, dsputils.NextPowerOf2(lx*2-1))
	la := len(a)
	factors, invFactors := getBluesteinFactors(lx)

	for n, v := range x {
		a[n] = v * invFactors[n]
	}

	b := make([]complex128, la)
	for i := 0; i < lx; i++ {
		b[i] = factors[i]

		if i != 0 {
			b[la-i] = factors[i]
		}
	}

	r := Convolve(a, b)

	for i := 0; i < lx; i++ {
		r[i] *= invFactors[i]
	}

	return r[:lx]
}

// FFT2Real returns the 2-dimensional, forward FFT of the real-valued matrix.
func FFT2Real(x [][]float64) [][]complex128 {
	return FFT2(dsputils.ToComplex2(x))
}

// FFT2 returns the 2-dimensional, forward FFT of the complex-valued matrix.
func FFT2(x [][]complex128) [][]complex128 {
	return computeFFT2(x, FFT)
}

// IFFT2Real returns the 2-dimensional, inverse FFT of the real-valued matrix.
func IFFT2Real(x [][]float64) [][]complex128 {
	return IFFT2(dsputils.ToComplex2(x))
}

// IFFT2 returns the 2-dimensional, inverse FFT of the complex-valued matrix.
func IFFT2(x [][]complex128) [][]complex128 {
	return computeFFT2(x, IFFT)
}

func computeFFT2(x [][]complex128, fftFunc func([]complex128) []complex128) [][]complex128 {
	rows := len(x)
	if rows == 0 {
		panic("empty input array")
	}

	cols := len(x[0])
	r := make([][]complex128, rows)
	for i := 0; i < rows; i++ {
		if len(x[i]) != cols {
			panic("ragged input array")
		}
		r[i] = make([]complex128, cols)
	}

	for i := 0; i < cols; i++ {
		t := make([]complex128, rows)
		for j := 0; j < rows; j++ {
			t[j] = x[j][i]
		}

		for n, v := range fftFunc(t) {
			r[n][i] = v
		}
	}

	for n, v := range r {
		r[n] = fftFunc(v)
	}

	return r
}

// FFTN returns the forward FFT of the matrix m, computed in all N dimensions.
func FFTN(m *dsputils.Matrix) *dsputils.Matrix {
	return computeFFTN(m, FFT)
}

// IFFTN returns the forward FFT of the matrix m, computed in all N dimensions.
func IFFTN(m *dsputils.Matrix) *dsputils.Matrix {
	return computeFFTN(m, IFFT)
}

func computeFFTN(m *dsputils.Matrix, fftFunc func([]complex128) []complex128) *dsputils.Matrix {
	dims := m.Dimensions()
	t := m.Copy()
	r := dsputils.MakeEmptyMatrix(dims)

	for n := range dims {
		dims[n] -= 1
	}

	for n := range dims {
		d := make([]int, len(dims))
		copy(d, dims)
		d[n] = -1

		for {
			r.SetDim(fftFunc(t.Dim(d)), d)

			if !decrDim(d, dims) {
				break
			}
		}

		r, t = t, r
	}

	return t
}

// decrDim decrements an element of x by 1, skipping all -1s, and wrapping up to d.
// If a value is 0, it will be reset to its correspending value in d, and will carry one from the next non -1 value to the right.
// Returns true if decremented, else false.
func decrDim(x, d []int) bool {
	for n, v := range x {
		if v == -1 {
			continue
		} else if v == 0 {
			i := n
			// find the next element to decrement
			for ; i < len(x); i++ {
				if x[i] == -1 {
					continue
				} else if x[i] == 0 {
					x[i] = d[i]
				} else {
					x[i] -= 1
					return true
				}
			}

			// no decrement
			return false
		} else {
			x[n] -= 1
			return true
		}
	}

	return false
}
<|endoftext|>


<commit_before>// Defines indexing API. APIs defined here are applicable to all packages
// under `indexing`.

package api

// TODO: Define the semantics of buffer size of channels that are returned by
// the following method receiver.

// Known index types
type IndexType string
const (
    View    IndexType = "view"
    Llrb              = "llrb"
    LevelDB           = "leveldb"
    RocksDB           = "rocksdb"
    HyperDB           = "hyperdb"
    RBTree            = "rbtree"
    CBTree            = "cbtree"
)

// Inclusion, controls how the boundaries values of a range are treated
type Inclusion string
const (
    Neither Inclusion = "none"
    Left              = "left"
    Right             = "right"
    Both              = "both"
)

// Uniqueness, characterizes if the algorithm demands unique keys
type Uniqueness bool
const (
    Unique    Uniqueness = true
    NonUnique            = false
)

// SortOrder characterizes if the algorithm emits keys in a predictable order
type SortOrder string
const (
    Unsorted SortOrder = "none"
    Asc                = "asc"
    Desc               = "desc"
)

// Expression to be applied on the document to get the secondary key.
type ExprType string
const (
    Simple     string = "simple"
    JavaScript        = "javascript"
    N1QL              = "n1ql"
)

// Every index ever created and maintained by this package will have an
// associated index-info structure.
type IndexInfo struct {
    Name       string    // Name of the index
    Uuid       string    // unique id for every new index created.
    Using      IndexType // indexing algorithm to use / used.
    OnExprList []string  // expression list
    Bucket     string    // bucket name, for which the index is created.
    IsPrimary  bool      // true/false based on index
    Exprtype   ExprType  // type of `Expression`
    Engine     Finder    // instance of index algorithm.
}

// Accuracy characterizes if the results of the index is subject to probabilistic errors.
// When an algorithm that is not Perfect is used, the caller must verify the results.
type Accuracy float64
const (
    Useless Accuracy = 0.0
    Perfect          = 1.0
)

// Complexity characterizes space and time characteristics of the algorithm
type Complexity int
const (
    O1 Complexity = iota
    Ologm
    Ologn
    Om
    Omlogm
    Omlogn
    On
    Onlogn
    Om2
    On2
    Ounknown
)

// TraitInfo is collection of traits of an algorithm. One can query the traits of an
// entire indexing algorithm, or traits of a specific operation. May change soon.
type TraitInfo struct {
    Unique     Uniqueness
    Order      SortOrder
    Accuracy   Accuracy
    AvgTime    Complexity
    AvgSpace   Complexity
    WorstTime  Complexity
    WorstSpace Complexity
}

// IndexCatalog is the interface for disk-based catalog for Index Manager
type IndexCatalog interface {
    // Create builds an instance of index
    Create(indexInfo IndexInfo) (string, error)

    // Drop kills an instance of an index
    Drop(uuid string) (string, error)

    // If `ServerUuid` is not nil, then check to see if the local ServerUUID
    // matches it. A match means client already has latest server
    // information and index data is not sent. A zero value makes server send
    // the latest index data unconditionally.
    //
    // Returned list IndexInfo won't contain the index instance.
    List(ServerUuid string) (string, []IndexInfo, error)

    // Gets a specific instance
    Index(uuid string) (IndexInfo, error)

    // Get Uuid
    GetUuid() string

    //Check if index already exists for a given bucket
    Exists(name string, bucket string) error
}

type Key interface {
    Bytes() []byte        // content of key as byte representation
    Less(than Key) bool   // compare whether `this` key is less than `than` key
    Compare(than Key) Ord // compare whether `this` key is less than `than` key
}

type Value interface {
    Bytes() []byte // content of value, typically document-id
}

type KV [2]interface{} // [Key, Value]

// Algorithm is the basic capability of any index algorithm
type Finder interface {
    Name() string
    Trait(operator interface{}) TraitInfo
}

// Counter is a class of algorithms that return total node count efficiently
type Counter interface {
    Finder
    CountTotal() (uint64, error)
}

// Exister is a class of algorithms that allow testing if a key exists in the
// index
type Exister interface {
    Finder
    Exists(key Key) bool // TODO: Should we have the `error` part of return ?
}

// Looker is a class of algorithms that allow looking up a key in an index.
// Usually, being able to look up a key means we can iterate through all keys
// too, and so that is introduced here as well.
type Looker interface {
    Exister
    Lookup(key Key) (chan Value, chan error)
    KeySet() (chan Key, chan error)
    ValueSet() (chan Value, chan error)
    KVSet() (chan KV, chan error)
}

// Ranger is a class of algorithms that can extract a range of keys from the
// index.
type Ranger interface {
    Looker
    KeyRange(low, high Key, inclusion Inclusion) (chan Key, chan error, SortOrder)
    ValueRange(low, high Key, inclusion Inclusion) (chan Value, chan error, SortOrder)
    KVRange(low Key, high Key, inclusion Inclusion) (chan KV, chan error, SortOrder)
}

// RangeCounter is a class of algorithms that can count a range efficiently
type RangeCounter interface {
    Finder
    CountRange(low Key, high Key, inclusion Inclusion) (uint64, error)
}
<commit_msg>Another iteration on indexing apis.<commit_after>// Defines indexing API. APIs defined here are applicable to all packages
// under `indexing`.

package api

// TODO: Define the semantics of buffer size of channels that are returned by
// the following method receiver.

// Known index types
type IndexType string
const (
    View    IndexType = "view"
    Llrb              = "llrb"
    LevelDB           = "leveldb"
    RocksDB           = "rocksdb"
    HyperDB           = "hyperdb"
    RBTree            = "rbtree"
    CBTree            = "cbtree"
)

// Inclusion, controls how the boundaries values of a range are treated
type Inclusion string
const (
    Neither Inclusion = "none"
    Left              = "left"
    Right             = "right"
    Both              = "both"
)

// Uniqueness, characterizes if the algorithm demands unique keys
type Uniqueness bool
const (
    Unique    Uniqueness = true
    NonUnique            = false
)

// SortOrder characterizes if the algorithm emits keys in a predictable order
type SortOrder string
const (
    Unsorted SortOrder = "none"
    Asc                = "asc"
    Desc               = "desc"
)

// Expression to be applied on the document to get the secondary key.
type ExprType string
const (
    Simple     string = "simple"
    JavaScript        = "javascript"
    N1QL              = "n1ql"
)

// Every index ever created and maintained by this package will have an
// associated index-info structure.
type IndexInfo struct {
    Name       string    // Name of the index
    Uuid       string    // unique id for every new index created.
    Using      IndexType // indexing algorithm to use / used.
    OnExprList []string  // expression list
    Bucket     string    // bucket name, for which the index is created.
    IsPrimary  bool      // true/false based on index
    Exprtype   ExprType  // type of `Expression`
    Engine     Finder    // instance of index algorithm.
}

// Accuracy characterizes if the results of the index is subject to probabilistic errors.
// When an algorithm that is not Perfect is used, the caller must verify the results.
type Accuracy float64
const (
    Useless Accuracy = 0.0
    Perfect          = 1.0
)

// Complexity characterizes space and time characteristics of the algorithm
type Complexity int
const (
    O1     Complexity = iota
    Ologm
    Ologn
    Om
    Omlogm
    Omlogn
    On
    Onlogn
    Om2
    On2
    Ounknown
)

// TraitInfo is collection of traits of an algorithm. One can query the traits of an
// entire indexing algorithm, or traits of a specific operation. May change soon.
type TraitInfo struct {
    Unique     Uniqueness
    Order      SortOrder
    Accuracy   Accuracy
    AvgTime    Complexity
    AvgSpace   Complexity
    WorstTime  Complexity
    WorstSpace Complexity
}

// Indexer is the interface into the index engine
type IndexManager interface {
    // Create builds an instance of index
    Create(indexInfo IndexInfo) (string, IndexInfo, error)

    // Drop kills an instance of an index
    Drop(uuid string) (string, error)

    // If `ServerUuid` is not nil, then check to see if the local ServerUUID
    // matches it. A match means client already has latest server
    // information and index data is not sent. A zero value makes server send
    // the latest index data unconditionally.
    //
    // Returned list IndexInfo won't contain the index instance.
    List(ServerUuid string) (string, []IndexInfo, error)

    // Gets a specific instance
    Index(uuid string) (IndexInfo, error)

    // Get Uuid
    GetUuid() string

    //Check if index already exists for a given bucket
    Exists(name string, bucket string) error
}

type Key interface {
    Bytes() []byte        // content of key as byte representation
    Less(than Key) bool   // compare whether `this` key is less than `than` key
    Compare(than Key) Ord // compare whether `this` key is less than `than` key
}

type Value interface {
=======
}

type Key interface {
    Bytes() []byte      // content of key as byte representation
    Less(than Key) bool // compare whether `this` key is less than `than` key
}

type Value interface{
>>>>>>> Another iteration on indexing apis.
    Bytes() []byte // content of value, typically document-id
}

type KV [2]interface{} // [Key, Value]

// Algorithm is the basic capability of any index algorithm
type Finder interface {
    Name() string
    Purge()
    Trait(operator interface{}) TraitInfo
}

// Counter is a class of algorithms that return total node count efficiently
type Counter interface {
    Finder
    CountTotal() (uint64, error)
}

// Exister is a class of algorithms that allow testing if a key exists in the
// index
type Exister interface {
    Finder
    Exists(key Key) bool // TODO: Should we have the `error` part of return ?
}

// Looker is a class of algorithms that allow looking up a key in an index.
// Usually, being able to look up a key means we can iterate through all keys
// too, and so that is introduced here as well.
//
// TODO: Define the semantics of buffer size of channels that are returned by
// the following method receiver.
type Looker interface {
    Exister
    Lookup(key Key) (chan Value, chan error)
    KeySet() (chan Key, chan error)
    ValueSet() (chan Value, chan error)
    KVSet() (chan KV, chan error)
}

// Ranger is a class of algorithms that can extract a range of keys from the
// index.
type Ranger interface {
    Looker
    KeyRange(low, high Key, inclusion Inclusion) (chan Key, chan error, SortOrder)
    ValueRange(low, high Key, inclusion Inclusion) (chan Value, chan error, SortOrder)
    KVRange(low Key, high Key, inclusion Inclusion) (chan KV, chan error, SortOrder)
}

// RangeCounter is a class of algorithms that can count a range efficiently
type RangeCounter interface {
    Finder
    CountRange(low Key, high Key, inclusion Inclusion) (uint64, error)
}
<|endoftext|>


<commit_before>// +build gofuzz

package types

import "encoding/json"

func FuzzPlaceCountainer(data []byte) int {
	var pc = &PlaceCountainer{}

	// Let's unmarshal, this is not our job so "bleh"
	err := json.Unmarshal(data, pc)
	if err != nil {
		return 0
	}

	// Now that it is unmarshalled, let's test the Place method !
	place, err := pc.Place()
	if err != nil {
		return 0
	}

	_ = place.PlaceName()
	_ = place.PlaceID()
	_ = place.PlaceType()

	return 1
}
<commit_msg>Add String method testing to FuzzJourney<commit_after>// +build gofuzz

package types

import "encoding/json"

func FuzzPlaceCountainer(data []byte) int {
	var pc = &PlaceCountainer{}

	// Let's unmarshal, this is not our job so "bleh"
	err := json.Unmarshal(data, pc)
	if err != nil {
		return 0
	}

	// Now that it is unmarshalled, let's test the Place method !
	place, err := pc.Place()
	if err != nil {
		return 0
	}

	_ = place.PlaceName()
	_ = place.PlaceID()
	_ = place.PlaceType()
	_ = place.String()

	return 1
}
<|endoftext|>


<commit_before>package pop

import (
	"fmt"
	"reflect"
	"strconv"

	"github.com/satori/go.uuid"
)

// Find the first record of the model in the database with a particular id.
//
//	c.Find(&User{}, 1)
func (c *Connection) Find(model interface{}, id interface{}) error {
	return Q(c).Find(model, id)
}

// Find the first record of the model in the database with a particular id.
//
//	q.Find(&User{}, 1)
func (q *Query) Find(model interface{}, id interface{}) error {
	m := &Model{Value: model}
	idq := fmt.Sprintf("%s.id = ?", m.TableName())
	switch t := id.(type) {
	case uuid.UUID:
		return q.Where(idq, t.String()).First(model)
	case string:
		var err error
		id, err = strconv.Atoi(t)
		if err != nil {
			return q.Where(idq, t).First(model)
		}
	}
	return q.Where(idq, id).First(model)
}

// First record of the model in the database that matches the query.
//
//	c.First(&User{})
func (c *Connection) First(model interface{}) error {
	return Q(c).First(model)
}

// First record of the model in the database that matches the query.
//
//	q.Where("name = ?", "mark").First(&User{})
func (q *Query) First(model interface{}) error {
	return q.Connection.timeFunc("First", func() error {
		q.Limit(1)
		m := &Model{Value: model}
		if err := q.Connection.Dialect.SelectOne(q.Connection.Store, m, *q); err != nil {
			return err
		}
		return m.afterFind(q.Connection)
	})
}

// Last record of the model in the database that matches the query.
//
//	c.Last(&User{})
func (c *Connection) Last(model interface{}) error {
	return Q(c).Last(model)
}

// Last record of the model in the database that matches the query.
//
//	q.Where("name = ?", "mark").Last(&User{})
func (q *Query) Last(model interface{}) error {
	return q.Connection.timeFunc("Last", func() error {
		q.Limit(1)
		q.Order("id desc")
		m := &Model{Value: model}
		if err := q.Connection.Dialect.SelectOne(q.Connection.Store, m, *q); err != nil {
			return err
		}
		return m.afterFind(q.Connection)
	})
}

// All retrieves all of the records in the database that match the query.
//
//	c.All(&[]User{})
func (c *Connection) All(models interface{}) error {
	return Q(c).All(models)
}

// All retrieves all of the records in the database that match the query.
//
//	q.Where("name = ?", "mark").All(&[]User{})
func (q *Query) All(models interface{}) error {
	return q.Connection.timeFunc("All", func() error {
		m := &Model{Value: models}
		err := q.Connection.Dialect.SelectMany(q.Connection.Store, m, *q)
		if err == nil && q.Paginator != nil {
			ct, err := q.Count(models)
			if err == nil {
				q.Paginator.TotalEntriesSize = ct
				st := reflect.ValueOf(models).Elem()
				q.Paginator.CurrentEntriesSize = st.Len()
				q.Paginator.TotalPages = (q.Paginator.TotalEntriesSize / q.Paginator.PerPage)
				if q.Paginator.TotalEntriesSize%q.Paginator.PerPage > 0 {
					q.Paginator.TotalPages = q.Paginator.TotalPages + 1
				}
			}
		}
		if err != nil {
			return err
		}
		return m.afterFind(q.Connection)
	})
}

// Exists returns true/false if a record exists in the database that matches
// the query.
//
// 	q.Where("name = ?", "mark").Exists(&User{})
func (q *Query) Exists(model interface{}) (bool, error) {
	i, err := q.Count(model)
	return i != 0, err
}

//only works with postgresql
func (q Query) WrapCount(model interface{}, field string) (int, error) {
	tmpQuery := Q(q.Connection)
	q.Clone(tmpQuery) //avoid mendling with original query

	res := &rowCount{}
	err := tmpQuery.Connection.timeFunc("Count", func() error {
		tmpQuery.Paginator = nil
		tmpQuery.orderClauses = clauses{}
		query, args := tmpQuery.ToSQL(&Model{Value: model})

		countQuery := fmt.Sprintf("select count(%s) as row_count from (%s) a", field, query)

		Log(countQuery, args...)
		return q.Connection.Store.Get(res, countQuery, args...)
	})
	return res.Count, err
}

// Count the number of records in the database.
//
//	c.Count(&User{})
func (c *Connection) Count(model interface{}) (int, error) {
	return Q(c).Count(model)
}

// Count the number of records in the database.
//
//	q.Where("name = ?", "mark").Count(&User{})
func (q Query) Count(model interface{}) (int, error) {
	return q.CountByField(model, "*")
}

func (q Query) CountByField(model interface{}, field string) (int, error) {
	tmpQuery := Q(q.Connection)
	q.Clone(tmpQuery) //avoid mendling with original query

	res := &rowCount{}
	err := tmpQuery.Connection.timeFunc("Count", func() error {
		tmpQuery.Paginator = nil
		tmpQuery.orderClauses = clauses{}
		query, args := tmpQuery.ToSQL(&Model{Value: model})

		countQuery := fmt.Sprintf("select count(%s) as row_count from (%s) a", field, query)

		Log(countQuery, args...)
		return q.Connection.Store.Get(res, countQuery, args...)
	})
	return res.Count, err
}

type rowCount struct {
	Count int `db:"row_count"`
}
<commit_msg>count works with group by on sqlite, mysql n postgresql<commit_after>package pop

import (
	"fmt"
	"reflect"
	"strconv"

	"github.com/satori/go.uuid"
)

// Find the first record of the model in the database with a particular id.
//
//	c.Find(&User{}, 1)
func (c *Connection) Find(model interface{}, id interface{}) error {
	return Q(c).Find(model, id)
}

// Find the first record of the model in the database with a particular id.
//
//	q.Find(&User{}, 1)
func (q *Query) Find(model interface{}, id interface{}) error {
	m := &Model{Value: model}
	idq := fmt.Sprintf("%s.id = ?", m.TableName())
	switch t := id.(type) {
	case uuid.UUID:
		return q.Where(idq, t.String()).First(model)
	case string:
		var err error
		id, err = strconv.Atoi(t)
		if err != nil {
			return q.Where(idq, t).First(model)
		}
	}
	return q.Where(idq, id).First(model)
}

// First record of the model in the database that matches the query.
//
//	c.First(&User{})
func (c *Connection) First(model interface{}) error {
	return Q(c).First(model)
}

// First record of the model in the database that matches the query.
//
//	q.Where("name = ?", "mark").First(&User{})
func (q *Query) First(model interface{}) error {
	return q.Connection.timeFunc("First", func() error {
		q.Limit(1)
		m := &Model{Value: model}
		if err := q.Connection.Dialect.SelectOne(q.Connection.Store, m, *q); err != nil {
			return err
		}
		return m.afterFind(q.Connection)
	})
}

// Last record of the model in the database that matches the query.
//
//	c.Last(&User{})
func (c *Connection) Last(model interface{}) error {
	return Q(c).Last(model)
}

// Last record of the model in the database that matches the query.
//
//	q.Where("name = ?", "mark").Last(&User{})
func (q *Query) Last(model interface{}) error {
	return q.Connection.timeFunc("Last", func() error {
		q.Limit(1)
		q.Order("id desc")
		m := &Model{Value: model}
		if err := q.Connection.Dialect.SelectOne(q.Connection.Store, m, *q); err != nil {
			return err
		}
		return m.afterFind(q.Connection)
	})
}

// All retrieves all of the records in the database that match the query.
//
//	c.All(&[]User{})
func (c *Connection) All(models interface{}) error {
	return Q(c).All(models)
}

// All retrieves all of the records in the database that match the query.
//
//	q.Where("name = ?", "mark").All(&[]User{})
func (q *Query) All(models interface{}) error {
	return q.Connection.timeFunc("All", func() error {
		m := &Model{Value: models}
		err := q.Connection.Dialect.SelectMany(q.Connection.Store, m, *q)
		if err == nil && q.Paginator != nil {
			ct, err := q.Count(models)
			if err == nil {
				q.Paginator.TotalEntriesSize = ct
				st := reflect.ValueOf(models).Elem()
				q.Paginator.CurrentEntriesSize = st.Len()
				q.Paginator.TotalPages = (q.Paginator.TotalEntriesSize / q.Paginator.PerPage)
				if q.Paginator.TotalEntriesSize%q.Paginator.PerPage > 0 {
					q.Paginator.TotalPages = q.Paginator.TotalPages + 1
				}
			}
		}
		if err != nil {
			return err
		}
		return m.afterFind(q.Connection)
	})
}

// Exists returns true/false if a record exists in the database that matches
// the query.
//
// 	q.Where("name = ?", "mark").Exists(&User{})
func (q *Query) Exists(model interface{}) (bool, error) {
	i, err := q.Count(model)
	return i != 0, err
}

// Count the number of records in the database.
//
//	c.Count(&User{})
func (c *Connection) Count(model interface{}) (int, error) {
	return Q(c).Count(model)
}

// Count the number of records in the database.
//
//	q.Where("name = ?", "mark").Count(&User{})
func (q Query) Count(model interface{}) (int, error) {
	return q.CountByField(model, "*")
}

func (q Query) CountByField(model interface{}, field string) (int, error) {
	tmpQuery := Q(q.Connection)
	q.Clone(tmpQuery) //avoid mendling with original query

	res := &rowCount{}
	err := tmpQuery.Connection.timeFunc("Count", func() error {
		tmpQuery.Paginator = nil
		tmpQuery.orderClauses = clauses{}
		query, args := tmpQuery.ToSQL(&Model{Value: model})

		countQuery := fmt.Sprintf("select count(%s) as row_count from (%s) a", field, query)

		Log(countQuery, args...)
		return q.Connection.Store.Get(res, countQuery, args...)
	})
	return res.Count, err
}

type rowCount struct {
	Count int `db:"row_count"`
}
<|endoftext|>


<commit_before>package main

import (
	"net/http"
	"fmt"
	"log"
	"bytes"
	"github.com/go-martini/martini"
	"os"
)

var clipboard string

/*
func handler(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
		case "GET":
			log.Printf("Copied %s\n", clipboard)
			fmt.Fprintf(w, clipboard)
		case "POST":
			buf:= new(bytes.Buffer)
			buf.ReadFrom(r.Body)
			s := buf.String()
			clipboard = s
			log.Printf("Pasted %s\n", clipboard)
			fmt.Fprintf(w, "ok")
		default:
			log.Println("Error")
	}
}
*/

func ClipboardGroup(r martini.Router) {
	r.Get("/", CopyClipboard)
	r.Post("/", PasteClipboard)
}

func CopyClipboard()(int, string) {
}

func PasteClipboard()(int, string) {
}

func main(){
	m := martini.Classic()
	//http.HandleFunc("/realclipper/api/clipboard", handler)
	m.Group("/realclipper/api/clipboard", ClipboardGroup)
	env_port := os.Getenv("REALCLIPPER_API_PORT")
	if env_port == "" {
		env_port = "8080"
	}
	port := fmt.Sprintf(":%s", env_port)
	log.Fatal(http.ListenAndServe(port, m))
}
<commit_msg>API updated with martini framework<commit_after>package main

import (
	"net/http"
	"fmt"
	"log"
	"bytes"
	"github.com/go-martini/martini"
	"os"
)

var clipboard string

func ClipboardGroup(r martini.Router) {
	r.Get("", CopyClipboard)
	r.Post("", PasteClipboard)
}

func CopyClipboard()(int, string) {
	log.Printf("Copied %s\n", clipboard)
	return 200, clipboard
}

func PasteClipboard(w http.ResponseWriter, r *http.Request) {
	buf:= new(bytes.Buffer)
	buf.ReadFrom(r.Body)
	s := buf.String()
	clipboard = s
	log.Printf("Pasted %s\n", clipboard)
	fmt.Fprintf(w, "ok")
}

func main(){
	m := martini.Classic()
	//http.HandleFunc("/realclipper/api/clipboard", handler)
	m.Group("/realclipper/api/clipboard", ClipboardGroup)
	env_port := os.Getenv("REALCLIPPER_API_PORT")
	if env_port == "" {
		env_port = "8080"
	}
	port := fmt.Sprintf(":%s", env_port)
	log.Fatal(http.ListenAndServe(port, m))
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"fmt"
	"math"
	"math/rand"
	"runtime"
	"time"

	"github.com/brunoga/workerpool"
)

var (
	flagNumWorkers = flag.Int("num_workers", runtime.NumCPU(),
		"number of workers to use")
	flagMaxNumber = flag.Uint64("max_number", 100000,
		"max number to compute the prime of")
)

// generateNumbers starts a goroutine that will return random uint64 numbers
// between 0 and maxNumber (using the given Rand instance) and send them
// through the returned channel. It will generate maxNumber numbers and close
// the channel.
func generateNumbers(maxNumber uint64, r *rand.Rand) <-chan interface{} {
	// Create output channel.
	output := make(chan interface{})

	i := uint64(0)
	go func() {
		for {
			output <- (r.Uint64() % maxNumber)

			i++
			if i >= maxNumber {
				break
			}
		}

		close(output)
	}()

	return output
}

// isPrime is a workerpool.WorkerFunc implementation that expects an uint64
// number as input and determines if it is a prime number or not. It returns the
// original uint64 number and a nil error on success and nil and a non-nil error
// on failure (including when the number is not prime).
func isPrime(i interface{}, ctx context.Context) (interface{}, error) {
	n, ok := i.(uint64)
	if !ok {
		return nil, fmt.Errorf("expected uint64. Got %t", i)
	}

	// Optimization. We can only check the square root of the given number.
	n2 := uint64(math.Ceil(math.Sqrt(float64(n))))

	if n2 < 2 {
		return nil, fmt.Errorf("not prime")
	}

	for divisor := uint64(2); divisor < n2; divisor++ {
		if (n2 % divisor) == 0 {
			return nil, fmt.Errorf("not prime")
		}
	}

	return n, nil
}

func main() {
	flag.Parse()

	// Start a WorkerPool for the isPrime function.
	workerPool, err := workerpool.New(isPrime, *flagNumWorkers)
	if err != nil {
		panic(err)
	}

	// Initialize random generator for the generateNumbers() function.
	r := rand.New(rand.NewSource(time.Now().UnixNano()))

	// WorkerPool input is the channel where numbers will be sent to.
	err = workerPool.SetInputChannel(generateNumbers(*flagMaxNumber, r))
	if err != nil {
		panic(err)
	}

	// Get WorkerPool output channel.
	outputChannel := workerPool.GetOutputChannel()

	// Start work.
	err = workerPool.Start(context.Background())
	if err != nil {
		panic(err)
	}

	// Read re3sults and handle them.
	for result := range outputChannel {
		switch result.(type) {
		case uint64:
			// Prime number.
			n := result.(uint64)
			fmt.Println(n, "is prime.")
		case workerpool.WorkerError:
			// Error (including non-prime numbers).
			workerError := result.(workerpool.WorkerError)
			if workerError.Error.Error() != "not prime" {
				// Only print anything if it is an actual error.
				fmt.Println(workerError)
			}
		default:
			panic("should never happend")
		}
	}
}
<commit_msg>Fix prime computation.<commit_after>package main

import (
	"context"
	"flag"
	"fmt"
	"math"
	"math/rand"
	"runtime"
	"time"

	"github.com/brunoga/workerpool"
)

var (
	flagNumWorkers = flag.Int("num_workers", runtime.NumCPU(),
		"number of workers to use")
	flagMaxNumber = flag.Uint64("max_number", 100000,
		"max number to compute the prime of")
)

// generateNumbers starts a goroutine that will return random uint64 numbers
// between 0 and maxNumber (using the given Rand instance) and send them
// through the returned channel. It will generate maxNumber numbers and close
// the channel.
func generateNumbers(maxNumber uint64, r *rand.Rand) <-chan interface{} {
	// Create output channel.
	output := make(chan interface{})

	i := uint64(0)
	go func() {
		for {
			output <- (r.Uint64() % maxNumber)

			i++
			if i >= maxNumber {
				break
			}
		}

		close(output)
	}()

	return output
}

// isPrime is a workerpool.WorkerFunc implementation that expects an uint64
// number as input and determines if it is a prime number or not. It returns the
// original uint64 number and a nil error on success and nil and a non-nil error
// on failure (including when the number is not prime).
func isPrime(i interface{}, ctx context.Context) (interface{}, error) {
	n, ok := i.(uint64)
	if !ok {
		return nil, fmt.Errorf("expected uint64. Got %t", i)
	}

	// Optimization. Only need to check divisors up to the suqare root of
	// the given number.
	n2 := uint64(math.Ceil(math.Sqrt(float64(n))))

	if n2 < 2 {
		return nil, fmt.Errorf("not prime")
	}

	for divisor := uint64(2); divisor <= n2; divisor++ {
		if (n % divisor) == 0 {
			return nil, fmt.Errorf("not prime")
		}
	}

	return n, nil
}

func main() {
	flag.Parse()

	// Start a WorkerPool for the isPrime function.
	workerPool, err := workerpool.New(isPrime, *flagNumWorkers)
	if err != nil {
		panic(err)
	}

	// Initialize random generator for the generateNumbers() function.
	r := rand.New(rand.NewSource(time.Now().UnixNano()))

	// WorkerPool input is the channel where numbers will be sent to.
	err = workerPool.SetInputChannel(generateNumbers(*flagMaxNumber, r))
	if err != nil {
		panic(err)
	}

	// Get WorkerPool output channel.
	outputChannel := workerPool.GetOutputChannel()

	// Start work.
	err = workerPool.Start(context.Background())
	if err != nil {
		panic(err)
	}

	// Read re3sults and handle them.
	for result := range outputChannel {
		switch result.(type) {
		case uint64:
			// Prime number.
			n := result.(uint64)
			fmt.Println(n, "is prime.")
		case workerpool.WorkerError:
			// Error (including non-prime numbers).
			workerError := result.(workerpool.WorkerError)
			if workerError.Error.Error() != "not prime" {
				// Only print anything if it is an actual error.
				fmt.Println(workerError)
			}
		default:
			panic("should never happend")
		}
	}
}
<|endoftext|>


<commit_before>package api

import (
	"fmt"
	"github.com/kamoljan/sushiobrol/conf"
	"html"
	"log"
	"net/http"
)

/*
 * http://localhost:9090/fid/
 * 0001-webp-xlarge-view-da39a3ee5e6b4b0d3255bfef95601890afd80709-EBEBEB-640-871
 */
func Get(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", conf.Mime)
	w.Header().Set("Cache-Control", fmt.Sprintf("max-age=%d", conf.CacheMaxAge))
	fid := html.EscapeString(r.URL.Path[5:]) // cut "/fid/"
	path := parsePath(fid)
	log.Println("GET: fid = " + fid)
	log.Println("GET: path = " + path)
	http.ServeFile(w, r, path)
}

func parsePath(eid string) string {
	return fmt.Sprintf("%s/%s/%s/%s/%s/%s/%s/%s", conf.SushiobrolStore, eid[:4], eid[5:9], eid[10:16], eid[17:21], eid[22:24], eid[24:26], eid)
}
<commit_msg>BUG: GET parses fid wrongly<commit_after>package api

import (
	"fmt"
	"github.com/kamoljan/sushiobrol/conf"
	"html"
	"log"
	"net/http"
	"strings"
)

/*
 * http://localhost:9090/fid/
 * 0001-webp-xlarge-view-da39a3ee5e6b4b0d3255bfef95601890afd80709-EBEBEB-640-871
 */
func Get(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", conf.Mime)
	w.Header().Set("Cache-Control", fmt.Sprintf("max-age=%d", conf.CacheMaxAge))
	fid := html.EscapeString(r.URL.Path[5:]) // cut "/fid/"
	path := parsePath(fid)
	log.Println("GET: fid = " + fid)
	log.Println("GET: path = " + path)
	http.ServeFile(w, r, path)
}

func parsePath(fid string) string {
	a := strings.Split(fid, "-")
	return fmt.Sprintf("%s/%s/%s/%s/%s/%s/%s/%s", conf.SushiobrolStore, a[0], a[1], a[2], a[3], a[4][:2], a[4][2:4], fid)
}
<|endoftext|>


<commit_before>package dpn

import (
	"encoding/json"
	"fmt"
	"github.com/APTrust/bagman/bagman"
	"github.com/bitly/go-nsq"
	"os"
	"time"
)

// Packager creates DPN bags from APTrust IntellectualObjects
// through the following steps:
//
// 1. Fetch all data files from S3.
// 2. Build the DPN bag with data files, manifests and tag files.
// 3. Tar the bag.
//
// The packager pushes data through the following channels:
//
// 1. LookupChannel retrieves the IntellectualObject (and a list of
//    its files) from Fluctus.
// 2. FetchChannel fetches the bag's data files from S3.
// 3. BuildChannel builds the DPN bag.
// 4. TarChannel tars the DPN bag.
// 5. CleanupChannel deletes the files that went into the
//    tarred bag (but keeps the tar file).
// 6. ResultsChannel logs results, tells NSQ the work is done,
//    and queues items from post-processing.
//
// Steps 5 and 6 are guaranteed to occur, no matter what happens
// in the other steps.

type PackageStatus struct {
	BagIdentifier   string
	NsqMessage      *nsq.Message `json:"-"`
	BagBuilder      *BagBuilder
	DPNFetchResults []*DPNFetchResult
	TarFilePath     string
	ErrorMessage    string
	Retry           bool
}

func (status *PackageStatus) Errors() ([]string) {
	errors := make([]string, 0)
	if status.ErrorMessage != "" {
		errors = append(errors, status.ErrorMessage)
	}
	if status.BagBuilder.ErrorMessage != "" {
		errors = append(errors, status.BagBuilder.ErrorMessage)
	}
	for _, result := range status.DPNFetchResults {
		if result.FetchResult.ErrorMessage != "" {
			errors = append(errors, result.FetchResult.ErrorMessage)
		}
	}
	return errors
}

func (status *PackageStatus) Succeeded() (bool) {
	return status.TarFilePath != "" && len(status.Errors()) == 0
}

type Packager struct {
	LookupChannel       chan *PackageStatus
	FetchChannel        chan *PackageStatus
	BuildChannel        chan *PackageStatus
	TarChannel          chan *PackageStatus
	CleanupChannel      chan *PackageStatus
	PostProcessChannel  chan *PackageStatus
	DefaultMetadata     *DefaultMetadata
	ProcUtil            *bagman.ProcessUtil
}

func NewPackager(procUtil *bagman.ProcessUtil, obj *bagman.IntellectualObject, defaultMetadata *DefaultMetadata) (*Packager) {
	packager := &Packager {
		DefaultMetadata: defaultMetadata,
		ProcUtil: procUtil,
	}
	fetcherBufferSize := procUtil.Config.DPNPackageWorker.NetworkConnections * 4
	workerBufferSize := procUtil.Config.DPNPackageWorker.Workers * 4
	packager.FetchChannel = make(chan *PackageStatus, fetcherBufferSize)
	packager.BuildChannel = make(chan *PackageStatus, workerBufferSize)
	packager.TarChannel = make(chan *PackageStatus, workerBufferSize)
	packager.CleanupChannel = make(chan *PackageStatus, workerBufferSize)
	packager.PostProcessChannel = make(chan *PackageStatus, workerBufferSize)
	for i := 0; i < procUtil.Config.DPNPackageWorker.Workers; i++ {
		go packager.doLookup()
		go packager.doBuild()
		go packager.doTar()
		go packager.doCleanup()
		go packager.postProcess()
	}
	for i := 0; i <  procUtil.Config.PrepareWorker.NetworkConnections; i++ {
		go packager.doFetch()
	}
	return packager
}

// MessageHandler handles messages from NSQ, putting each
// item into the pipleline.
func (packager *Packager) HandleMessage(message *nsq.Message) error {
	message.DisableAutoResponse()

	var packageStatus *PackageStatus
	err := json.Unmarshal(message.Body, packageStatus)
	if err != nil {
		detailedError := fmt.Errorf("Could not unmarshal JSON data from nsq:",
			string(message.Body))
		packager.ProcUtil.MessageLog.Error(detailedError.Error())
		message.Finish()
		return detailedError
	}

	// Start processing.
	packager.LookupChannel <- packageStatus
	packager.ProcUtil.MessageLog.Info("Put %s into lookup channel",
		packageStatus.BagIdentifier)
	return nil
}

// doLookup gets information about the intellectual object from
// Fluctus, builds a package status object, and moves the data
// into the FetchChannel.
func (packager *Packager) doLookup() {
	for status := range packager.LookupChannel {
		if status.BagBuilder != nil {
			// We started work on this bag before. No need to look it up again.
			packager.FetchChannel <- status
			continue
		}

		// Get the bag, with a list of GenericFiles
		intelObj, err := packager.ProcUtil.FluctusClient.IntellectualObjectGet(status.BagIdentifier, true)
		if err != nil {
			// FAIL - Can't get intel obj data (HTTP or Fluctus error)
			status.ErrorMessage = fmt.Sprintf("Could not fetch info about IntellectualObject " +
				"'%s' from Fluctus: %s", status.BagIdentifier, err.Error())
			status.Retry = true
			packager.PostProcessChannel <- status
			continue
		}
		if intelObj == nil {
			// FAIL - Can't get intel obj data (Object not found)
			status.ErrorMessage = fmt.Sprintf("Fluctus returned nil for IntellectualObject %s",
				status.BagIdentifier)
			status.Retry = true
			packager.PostProcessChannel <- status
			continue
		}
		err = packager.ProcUtil.Volume.Reserve(uint64(intelObj.TotalFileSize() * 2))
		if err != nil {
			// FAIL - Not enough disk space in staging area to build this bag
			packager.ProcUtil.MessageLog.Warning("Requeueing bag %s, %d bytes - not enough disk space",
				status.BagIdentifier, intelObj.TotalFileSize())
			status.ErrorMessage = err.Error()
			status.Retry = true
			packager.PostProcessChannel <- status
			continue
		} else {
			// Woo-hoo!
			status.BagBuilder = NewBagBuilder(packager.ProcUtil.Config.DPNStagingDirectory,
				intelObj, packager.DefaultMetadata)
			packager.FetchChannel <- status
		}
	}
}

// doFetch fetches the IntellectualObject's files from S3 and
// stores them locally. Data then goes into the BuildChannel
// so we can build the DPN bag.
func (packager *Packager) doFetch() {
	// for status := range packager.FetchChannel {

	// }
}

// doBuild builds the DPN bag, creating all of the necessary
// tag files, manifests and directories. Data then goes into the
// TarChannel, so the bag can be tarred up.
func (packager *Packager) doBuild() {
	// for status := range packager.BuildChannel {

	// }
}

// doTar tars up the DPN bag and then sends data along to the
// CleanupChannel.
func (packager *Packager) doTar() {
	// for status := range packager.TarChannel {

	// }
}

// doCleanup cleans up the the directory containing all of the
// data files, tag files, manifests, etc. that went into the
// DPN bag. When this is done, the tar file will still be around,
// but the directories whose contents went into the tar file will
// be gone. From here, data goes into the PostProcessChannel.
func (packager *Packager) doCleanup() {
	log := packager.ProcUtil.MessageLog
	for status := range packager.CleanupChannel {
		if packager.shouldCleanup(status) {
			packager.cleanup(status)
		}
		status.Retry = packager.shouldRetry(status)
		if status.Succeeded() {
			status.NsqMessage.Finish()
		} else {
			log.Info("Requeuing %s (%s)", status.BagIdentifier, status.BagBuilder.LocalPath)
			status.NsqMessage.Requeue(1 * time.Minute)
		}
		packager.PostProcessChannel <- status
	}
}

func (packager *Packager) reachedMaxAttempts(status *PackageStatus) (bool) {
	return status.NsqMessage.Attempts >= uint16(packager.ProcUtil.Config.DPNPackageWorker.MaxAttempts)
}

// shouldCleanup tells us whether we should delete all of the files,
// except the tar files, for a DPN bag. See inline comments for the logic.
func (packager *Packager) shouldCleanup(status *PackageStatus) (cleanItUp bool) {
	log := packager.ProcUtil.MessageLog
	downloadStarted := status.DPNFetchResults != nil && len(status.DPNFetchResults) > 0
	cleanItUp = false

	if status.Succeeded() && status.TarFilePath != "" {
		// We have the tar file & no longer need the untarred files.
		cleanItUp = true
	} else if packager.reachedMaxAttempts(status) {
		// Failed, and we're done with this bag.
		cleanItUp = true
		status.ErrorMessage += fmt.Sprintf("Processing failed after %d attempts.", status.NsqMessage.Attempts)
	} else if downloadStarted == false {
		// No use leaving an empty directory laying around.
		cleanItUp = true
	} else {
		log.Info("Skipping cleanup on bag %s at %s. Leaving files in place for retry.",
			status.BagIdentifier, status.BagBuilder.LocalPath)
	}

	if cleanItUp {
		log.Info("Cleaning up bag %s at %s", status.BagIdentifier, status.BagBuilder.LocalPath)
	}

	// If we get to this point, leave cleanItUp = false, because
	// we've downloaded some files, and we still have some retries
	// left. This may be a bag with 1000 files, and we may already
	// have 900 of them on disk. The next retry will resume dowloading
	// at file #901, which is what we want. The only downside here is
	// that we might be using up a lot of disk space.
	return cleanItUp
}

func (packager *Packager) shouldRetry(status *PackageStatus) (retry bool) {
	retry = true
	if status.Succeeded() && status.TarFilePath != "" {
		retry = false
	} else if packager.reachedMaxAttempts(status) {
		retry = false
	}
	return retry
}

func (packager *Packager) cleanup(status *PackageStatus) {
	err := os.RemoveAll(status.BagBuilder.LocalPath)
	if err != nil {
		packager.ProcUtil.MessageLog.Error("Error cleaning up %s: %v",
			status.BagBuilder.LocalPath, err)
	} else {
		packager.ProcUtil.Volume.Release(uint64(status.BagBuilder.IntellectualObject.TotalFileSize() * 2))
	}
}

// postProcess logs the results of our DPN bagging operation, tells
// NSQ that the worker is done with the job (whether successful or not),
// and sends data to the next NSQ topic for post-processing.
func (packager *Packager) postProcess() {
	for status := range packager.PostProcessChannel {
		if status.Succeeded() {
			packager.ProcUtil.MessageLog.Info("Bag %s successfully packaged at %s",
				status.BagIdentifier, status.TarFilePath)
			packager.ProcUtil.IncrementSucceeded()
			// TODO: Send to DPN storage queue
		} else {
			if packager.reachedMaxAttempts(status) {
				packager.ProcUtil.MessageLog.Error(status.ErrorMessage)
				packager.ProcUtil.IncrementFailed()
				// TODO: Send to DPN trouble queue
			} else {  // Failed, but we can still retry
				packager.ProcUtil.MessageLog.Warning(
					"Bag %s failed, but will retry. %s",
					status.BagIdentifier, status.ErrorMessage)
			}
		}
	}
}
<commit_msg>Interim commit. Working on packager.<commit_after>package dpn

import (
	"encoding/json"
	"fmt"
	"github.com/APTrust/bagman/bagman"
	"github.com/bitly/go-nsq"
	"os"
	"strings"
	"time"
)

// Packager creates DPN bags from APTrust IntellectualObjects
// through the following steps:
//
// 1. Fetch all data files from S3.
// 2. Build the DPN bag with data files, manifests and tag files.
// 3. Tar the bag.
//
// The packager pushes data through the following channels:
//
// 1. LookupChannel retrieves the IntellectualObject (and a list of
//    its files) from Fluctus.
// 2. FetchChannel fetches the bag's data files from S3.
// 3. BuildChannel builds the DPN bag.
// 4. TarChannel tars the DPN bag.
// 5. CleanupChannel deletes the files that went into the
//    tarred bag (but keeps the tar file).
// 6. ResultsChannel logs results, tells NSQ the work is done,
//    and queues items from post-processing.
//
// Steps 5 and 6 are guaranteed to occur, no matter what happens
// in the other steps.

type PackageStatus struct {
	BagIdentifier   string
	NsqMessage      *nsq.Message `json:"-"`
	BagBuilder      *BagBuilder
	DPNFetchResults []*DPNFetchResult
	TarFilePath     string
	ErrorMessage    string
	Retry           bool
}

func (status *PackageStatus) Errors() ([]string) {
	errors := make([]string, 0)
	if status.ErrorMessage != "" {
		errors = append(errors, status.ErrorMessage)
	}
	if status.BagBuilder.ErrorMessage != "" {
		errors = append(errors, status.BagBuilder.ErrorMessage)
	}
	for _, result := range status.DPNFetchResults {
		if result.FetchResult.ErrorMessage != "" {
			errors = append(errors, result.FetchResult.ErrorMessage)
		}
	}
	return errors
}

func (status *PackageStatus) Succeeded() (bool) {
	return status.TarFilePath != "" && len(status.Errors()) == 0
}

type Packager struct {
	LookupChannel       chan *PackageStatus
	FetchChannel        chan *PackageStatus
	BuildChannel        chan *PackageStatus
	TarChannel          chan *PackageStatus
	CleanupChannel      chan *PackageStatus
	PostProcessChannel  chan *PackageStatus
	DefaultMetadata     *DefaultMetadata
	ProcUtil            *bagman.ProcessUtil
}

func NewPackager(procUtil *bagman.ProcessUtil, obj *bagman.IntellectualObject, defaultMetadata *DefaultMetadata) (*Packager) {
	packager := &Packager {
		DefaultMetadata: defaultMetadata,
		ProcUtil: procUtil,
	}
	fetcherBufferSize := procUtil.Config.DPNPackageWorker.NetworkConnections * 4
	workerBufferSize := procUtil.Config.DPNPackageWorker.Workers * 4
	packager.FetchChannel = make(chan *PackageStatus, fetcherBufferSize)
	packager.BuildChannel = make(chan *PackageStatus, workerBufferSize)
	packager.TarChannel = make(chan *PackageStatus, workerBufferSize)
	packager.CleanupChannel = make(chan *PackageStatus, workerBufferSize)
	packager.PostProcessChannel = make(chan *PackageStatus, workerBufferSize)
	for i := 0; i < procUtil.Config.DPNPackageWorker.Workers; i++ {
		go packager.doLookup()
		go packager.doBuild()
		go packager.doTar()
		go packager.doCleanup()
		go packager.postProcess()
	}
	for i := 0; i <  procUtil.Config.PrepareWorker.NetworkConnections; i++ {
		go packager.doFetch()
	}
	return packager
}

// MessageHandler handles messages from NSQ, putting each
// item into the pipleline.
func (packager *Packager) HandleMessage(message *nsq.Message) error {
	message.DisableAutoResponse()

	var packageStatus *PackageStatus
	err := json.Unmarshal(message.Body, packageStatus)
	if err != nil {
		detailedError := fmt.Errorf("Could not unmarshal JSON data from nsq:",
			string(message.Body))
		packager.ProcUtil.MessageLog.Error(detailedError.Error())
		message.Finish()
		return detailedError
	}

	// Start processing.
	packager.LookupChannel <- packageStatus
	packager.ProcUtil.MessageLog.Info("Put %s into lookup channel",
		packageStatus.BagIdentifier)
	return nil
}

// doLookup gets information about the intellectual object from
// Fluctus, builds a package status object, and moves the data
// into the FetchChannel.
func (packager *Packager) doLookup() {
	for status := range packager.LookupChannel {
		if status.BagBuilder != nil {
			// We started work on this bag before. No need to look it up again.
			packager.FetchChannel <- status
			continue
		}

		// Get the bag, with a list of GenericFiles
		intelObj, err := packager.ProcUtil.FluctusClient.IntellectualObjectGet(status.BagIdentifier, true)
		if err != nil {
			// FAIL - Can't get intel obj data (HTTP or Fluctus error)
			status.ErrorMessage = fmt.Sprintf("Could not fetch info about IntellectualObject " +
				"'%s' from Fluctus: %s", status.BagIdentifier, err.Error())
			status.Retry = true
			packager.PostProcessChannel <- status
			continue
		}
		if intelObj == nil {
			// FAIL - Can't get intel obj data (Object not found)
			status.ErrorMessage = fmt.Sprintf("Fluctus returned nil for IntellectualObject %s",
				status.BagIdentifier)
			status.Retry = true
			packager.PostProcessChannel <- status
			continue
		}
		err = packager.ProcUtil.Volume.Reserve(uint64(intelObj.TotalFileSize() * 2))
		if err != nil {
			// FAIL - Not enough disk space in staging area to build this bag
			packager.ProcUtil.MessageLog.Warning("Requeueing bag %s, %d bytes - not enough disk space",
				status.BagIdentifier, intelObj.TotalFileSize())
			status.ErrorMessage = err.Error()
			status.Retry = true
			packager.PostProcessChannel <- status
			continue
		} else {
			// Woo-hoo!
			status.BagBuilder = NewBagBuilder(packager.ProcUtil.Config.DPNStagingDirectory,
				intelObj, packager.DefaultMetadata)
			packager.FetchChannel <- status
		}
	}
}

// doFetch fetches the IntellectualObject's files from S3 and
// stores them locally. Data then goes into the BuildChannel
// so we can build the DPN bag.
//
// TODO: If we really want to resume fetching, we'll
// need to have a constant UUID for the DPN bag. That
// UUID is assigned in NewBagBuilder. We'll have to assign
// it beforehand. Here's the case:
//
// - We fetched 50 of 100 files for DPN bag X, then we fail & requeue.
// - We have the 50 files we fetched still on the local disk.
// - A new worker picks up the the requeued task. It needs to know
//   where to look to find the 50 files. But the new BagBuilder has
//   assigned this bag UUID Y. We'll look in the wrong directory
//   (Y instead of X). Maybe we should use the DPN bag identifier
//   as the dirname? If we tar it up that way, and change the
//   tar file name at the end to the UUID, then when the bag is
//   untarred, it comes out as test.edu/my_bag. That could be a
//   benefit.
func (packager *Packager) doFetch() {
	// for status := range packager.FetchChannel {

	// }
}

func (packager *Packager) filesAlreadyFetched(status *PackageStatus) ([]string, error) {
	// Get a list of all files we've already fetched.
	// These would have been fetched in a prior run
	// that eventually errored out. Maybe we have 50
	// of the 100 files we need for a bag.
	files, err := bagman.RecursiveFileList(status.BagBuilder.LocalPath)
	if err != nil {
		return nil, err
	}
	// Convert the absolute paths returned by RecursiveFileList
	// to GenericFile.Identifiers.
	gfIdentifiers := make([]string, len(files))
	for i, f := range files {
		gfIdentifiers[i] = strings.Replace(f,
			status.BagBuilder.LocalPath,
			status.BagIdentifier, 1)
	}
	return gfIdentifiers, err
}

// doBuild builds the DPN bag, creating all of the necessary
// tag files, manifests and directories. Data then goes into the
// TarChannel, so the bag can be tarred up.
func (packager *Packager) doBuild() {
	// for status := range packager.BuildChannel {

	// }
}

// doTar tars up the DPN bag and then sends data along to the
// CleanupChannel.
func (packager *Packager) doTar() {
	// for status := range packager.TarChannel {

	// }
}

// doCleanup cleans up the the directory containing all of the
// data files, tag files, manifests, etc. that went into the
// DPN bag. When this is done, the tar file will still be around,
// but the directories whose contents went into the tar file will
// be gone. From here, data goes into the PostProcessChannel.
func (packager *Packager) doCleanup() {
	log := packager.ProcUtil.MessageLog
	for status := range packager.CleanupChannel {
		if packager.shouldCleanup(status) {
			packager.cleanup(status)
		}
		status.Retry = packager.shouldRetry(status)
		if status.Succeeded() {
			status.NsqMessage.Finish()
		} else {
			log.Info("Requeuing %s (%s)", status.BagIdentifier, status.BagBuilder.LocalPath)
			status.NsqMessage.Requeue(1 * time.Minute)
		}
		packager.PostProcessChannel <- status
	}
}

func (packager *Packager) reachedMaxAttempts(status *PackageStatus) (bool) {
	return status.NsqMessage.Attempts >= uint16(packager.ProcUtil.Config.DPNPackageWorker.MaxAttempts)
}

// shouldCleanup tells us whether we should delete all of the files,
// except the tar files, for a DPN bag. See inline comments for the logic.
func (packager *Packager) shouldCleanup(status *PackageStatus) (cleanItUp bool) {
	log := packager.ProcUtil.MessageLog
	downloadStarted := status.DPNFetchResults != nil && len(status.DPNFetchResults) > 0
	cleanItUp = false

	if status.Succeeded() && status.TarFilePath != "" {
		// We have the tar file & no longer need the untarred files.
		cleanItUp = true
	} else if packager.reachedMaxAttempts(status) {
		// Failed, and we're done with this bag.
		cleanItUp = true
		status.ErrorMessage += fmt.Sprintf("Processing failed after %d attempts.", status.NsqMessage.Attempts)
	} else if downloadStarted == false {
		// No use leaving an empty directory laying around.
		cleanItUp = true
	} else {
		log.Info("Skipping cleanup on bag %s at %s. Leaving files in place for retry.",
			status.BagIdentifier, status.BagBuilder.LocalPath)
	}

	if cleanItUp {
		log.Info("Cleaning up bag %s at %s", status.BagIdentifier, status.BagBuilder.LocalPath)
	}

	// If we get to this point, leave cleanItUp = false, because
	// we've downloaded some files, and we still have some retries
	// left. This may be a bag with 1000 files, and we may already
	// have 900 of them on disk. The next retry will resume dowloading
	// at file #901, which is what we want. The only downside here is
	// that we might be using up a lot of disk space.
	return cleanItUp
}

func (packager *Packager) shouldRetry(status *PackageStatus) (retry bool) {
	retry = true
	if status.Succeeded() && status.TarFilePath != "" {
		retry = false
	} else if packager.reachedMaxAttempts(status) {
		retry = false
	}
	return retry
}

func (packager *Packager) cleanup(status *PackageStatus) {
	err := os.RemoveAll(status.BagBuilder.LocalPath)
	if err != nil {
		packager.ProcUtil.MessageLog.Error("Error cleaning up %s: %v",
			status.BagBuilder.LocalPath, err)
	} else {
		packager.ProcUtil.Volume.Release(uint64(status.BagBuilder.IntellectualObject.TotalFileSize() * 2))
	}
}

// postProcess logs the results of our DPN bagging operation, tells
// NSQ that the worker is done with the job (whether successful or not),
// and sends data to the next NSQ topic for post-processing.
func (packager *Packager) postProcess() {
	for status := range packager.PostProcessChannel {
		if status.Succeeded() {
			packager.ProcUtil.MessageLog.Info("Bag %s successfully packaged at %s",
				status.BagIdentifier, status.TarFilePath)
			packager.ProcUtil.IncrementSucceeded()
			// TODO: Send to DPN storage queue
		} else {
			if packager.reachedMaxAttempts(status) {
				packager.ProcUtil.MessageLog.Error(status.ErrorMessage)
				packager.ProcUtil.IncrementFailed()
				// TODO: Send to DPN trouble queue
			} else {  // Failed, but we can still retry
				packager.ProcUtil.MessageLog.Warning(
					"Bag %s failed, but will retry. %s",
					status.BagIdentifier, status.ErrorMessage)
			}
		}
	}
}
<|endoftext|>