<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package fc

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/golang/glog"
	"k8s.io/kubernetes/pkg/volume"
)

type ioHandler interface {
	ReadDir(dirname string) ([]os.FileInfo, error)
	Lstat(name string) (os.FileInfo, error)
	EvalSymlinks(path string) (string, error)
	WriteFile(filename string, data []byte, perm os.FileMode) error
}

type osIOHandler struct{}

func (handler *osIOHandler) ReadDir(dirname string) ([]os.FileInfo, error) {
	return ioutil.ReadDir(dirname)
}
func (handler *osIOHandler) Lstat(name string) (os.FileInfo, error) {
	return os.Lstat(name)
}
func (handler *osIOHandler) EvalSymlinks(path string) (string, error) {
	return filepath.EvalSymlinks(path)
}
func (handler *osIOHandler) WriteFile(filename string, data []byte, perm os.FileMode) error {
	return ioutil.WriteFile(filename, data, perm)
}

// given a disk path like /dev/sdx, find the devicemapper parent
// TODO #23192 Convert this code to use the generic code in ../util
// which is used by the iSCSI implementation
func findMultipathDeviceMapper(disk string, io ioHandler) string {
	sys_path := "/sys/block/"
	if dirs, err := io.ReadDir(sys_path); err == nil {
		for _, f := range dirs {
			name := f.Name()
			if strings.HasPrefix(name, "dm-") {
				if _, err1 := io.Lstat(sys_path + name + "/slaves/" + disk); err1 == nil {
					return "/dev/" + name
				}
			}
		}
	}
	return ""
}

// given a wwn and lun, find the device and associated devicemapper parent
func findDisk(wwn, lun string, io ioHandler) (string, string) {
	fc_path := "-fc-0x" + wwn + "-lun-" + lun
	dev_path := "/dev/disk/by-path/"
	if dirs, err := io.ReadDir(dev_path); err == nil {
		for _, f := range dirs {
			name := f.Name()
			if strings.Contains(name, fc_path) {
				if disk, err1 := io.EvalSymlinks(dev_path + name); err1 == nil {
					arr := strings.Split(disk, "/")
					l := len(arr) - 1
					dev := arr[l]
					dm := findMultipathDeviceMapper(dev, io)
					return disk, dm
				}
			}
		}
	}
	return "", ""
}

// given a wwid, find the device and associated devicemapper parent
func findDiskWWIDs(wwid string, io ioHandler) (string, string) {
	// Example wwid format:
	//   3600508b400105e210000900000490000
	//   <VENDOR NAME> <IDENTIFIER NUMBER>
	// Example of symlink under by-id:
	//   /dev/by-id/scsi-3600508b400105e210000900000490000
	//   /dev/by-id/scsi-<VENDOR NAME>_<IDENTIFIER NUMBER>
	// The wwid could contain white space and it will be replaced
	// underscore when wwid is exposed under /dev/by-id.

	fc_path := "scsi-" + wwid
	dev_id := "/dev/disk/by-id/"
	if dirs, err := io.ReadDir(dev_id); err == nil {
		for _, f := range dirs {
			name := f.Name()
			if name == fc_path {
				disk, err := io.EvalSymlinks(dev_id + name)
				if err != nil {
					glog.V(2).Infof("fc: failed to find a corresponding disk from symlink[%s], error %v", dev_id+name, err)
					return "", ""
				}
				arr := strings.Split(disk, "/")
				l := len(arr) - 1
				dev := arr[l]
				dm := findMultipathDeviceMapper(dev, io)
				return disk, dm
			}
		}
	}
	glog.V(2).Infof("fc: failed to find a disk [%s]", dev_id+fc_path)
	return "", ""
}

// Removes a scsi device based upon /dev/sdX name
func removeFromScsiSubsystem(deviceName string, io ioHandler) {
	fileName := "/sys/block/" + deviceName + "/device/delete"
	glog.V(4).Infof("fc: remove device from scsi-subsystem: path: %s", fileName)
	data := []byte("1")
	io.WriteFile(fileName, data, 0666)
}

// rescan scsi bus
func scsiHostRescan(io ioHandler) {
	scsi_path := "/sys/class/scsi_host/"
	if dirs, err := io.ReadDir(scsi_path); err == nil {
		for _, f := range dirs {
			name := scsi_path + f.Name() + "/scan"
			data := []byte("- - -")
			io.WriteFile(name, data, 0666)
		}
	}
}

// make a directory like /var/lib/kubelet/plugins/kubernetes.io/pod/fc/target-lun-0
func makePDNameInternal(host volume.VolumeHost, wwns []string, lun string, wwids []string) string {
	if len(wwns) != 0 {
		return path.Join(host.GetPluginDir(fcPluginName), wwns[0]+"-lun-"+lun)
	} else {
		return path.Join(host.GetPluginDir(fcPluginName), wwids[0])
	}
}

type FCUtil struct{}

func (util *FCUtil) MakeGlobalPDName(fc fcDisk) string {
	return makePDNameInternal(fc.plugin.host, fc.wwns, fc.lun, fc.wwids)
}

func searchDisk(b fcDiskMounter) (string, string) {
	var diskIds []string
	var disk string
	var dm string
	io := b.io
	wwids := b.wwids
	wwns := b.wwns
	lun := b.lun

	if len(wwns) != 0 {
		diskIds = wwns
	} else {
		diskIds = wwids
	}

	rescaned := false
	// two-phase search:
	// first phase, search existing device path, if a multipath dm is found, exit loop
	// otherwise, in second phase, rescan scsi bus and search again, return with any findings
	for true {
		for _, diskId := range diskIds {
			if len(wwns) != 0 {
				disk, dm = findDisk(diskId, lun, io)
			} else {
				disk, dm = findDiskWWIDs(diskId, io)
			}
			// if multipath device is found, break
			if dm != "" {
				break
			}
		}
		// if a dm is found, exit loop
		if rescaned || dm != "" {
			break
		}
		// rescan and search again
		// rescan scsi bus
		scsiHostRescan(io)
		rescaned = true
	}
	return disk, dm
}

func (util *FCUtil) AttachDisk(b fcDiskMounter) (string, error) {
	devicePath := ""
	var disk, dm string

	disk, dm = searchDisk(b)
	// if no disk matches input wwn and lun, exit
	if disk == "" && dm == "" {
		return "", fmt.Errorf("no fc disk found")
	}

	// if multipath devicemapper device is found, use it; otherwise use raw disk
	if dm != "" {
		devicePath = dm
	} else {
		devicePath = disk
	}
	// mount it
	globalPDPath := util.MakeGlobalPDName(*b.fcDisk)
	noMnt, err := b.mounter.IsLikelyNotMountPoint(globalPDPath)
	if err != nil {
		return devicePath, fmt.Errorf("Heuristic determination of mount point failed:%v", err)
	}
	if !noMnt {
		glog.Infof("fc: %s already mounted", globalPDPath)
		return devicePath, nil
	}

	if err := os.MkdirAll(globalPDPath, 0750); err != nil {
		return devicePath, fmt.Errorf("fc: failed to mkdir %s, error", globalPDPath)
	}

	err = b.mounter.FormatAndMount(devicePath, globalPDPath, b.fsType, nil)
	if err != nil {
		return devicePath, fmt.Errorf("fc: failed to mount fc volume %s [%s] to %s, error %v", devicePath, b.fsType, globalPDPath, err)
	}

	return devicePath, err
}

func (util *FCUtil) DetachDisk(c fcDiskUnmounter, devName string) error {
	// Remove scsi device from the node.
	if !strings.HasPrefix(devName, "/dev/") {
		return fmt.Errorf("fc detach disk: invalid device name: %s", devName)
	}
	arr := strings.Split(devName, "/")
	dev := arr[len(arr)-1]
	removeFromScsiSubsystem(dev, c.io)
	return nil
}
<commit_msg>Fix FC WaitForAttach not mounting a volume<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package fc

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/golang/glog"
	"k8s.io/kubernetes/pkg/volume"
)

type ioHandler interface {
	ReadDir(dirname string) ([]os.FileInfo, error)
	Lstat(name string) (os.FileInfo, error)
	EvalSymlinks(path string) (string, error)
	WriteFile(filename string, data []byte, perm os.FileMode) error
}

type osIOHandler struct{}

func (handler *osIOHandler) ReadDir(dirname string) ([]os.FileInfo, error) {
	return ioutil.ReadDir(dirname)
}
func (handler *osIOHandler) Lstat(name string) (os.FileInfo, error) {
	return os.Lstat(name)
}
func (handler *osIOHandler) EvalSymlinks(path string) (string, error) {
	return filepath.EvalSymlinks(path)
}
func (handler *osIOHandler) WriteFile(filename string, data []byte, perm os.FileMode) error {
	return ioutil.WriteFile(filename, data, perm)
}

// given a disk path like /dev/sdx, find the devicemapper parent
// TODO #23192 Convert this code to use the generic code in ../util
// which is used by the iSCSI implementation
func findMultipathDeviceMapper(disk string, io ioHandler) string {
	sys_path := "/sys/block/"
	if dirs, err := io.ReadDir(sys_path); err == nil {
		for _, f := range dirs {
			name := f.Name()
			if strings.HasPrefix(name, "dm-") {
				if _, err1 := io.Lstat(sys_path + name + "/slaves/" + disk); err1 == nil {
					return "/dev/" + name
				}
			}
		}
	}
	return ""
}

// given a wwn and lun, find the device and associated devicemapper parent
func findDisk(wwn, lun string, io ioHandler) (string, string) {
	fc_path := "-fc-0x" + wwn + "-lun-" + lun
	dev_path := "/dev/disk/by-path/"
	if dirs, err := io.ReadDir(dev_path); err == nil {
		for _, f := range dirs {
			name := f.Name()
			if strings.Contains(name, fc_path) {
				if disk, err1 := io.EvalSymlinks(dev_path + name); err1 == nil {
					arr := strings.Split(disk, "/")
					l := len(arr) - 1
					dev := arr[l]
					dm := findMultipathDeviceMapper(dev, io)
					return disk, dm
				}
			}
		}
	}
	return "", ""
}

// given a wwid, find the device and associated devicemapper parent
func findDiskWWIDs(wwid string, io ioHandler) (string, string) {
	// Example wwid format:
	//   3600508b400105e210000900000490000
	//   <VENDOR NAME> <IDENTIFIER NUMBER>
	// Example of symlink under by-id:
	//   /dev/by-id/scsi-3600508b400105e210000900000490000
	//   /dev/by-id/scsi-<VENDOR NAME>_<IDENTIFIER NUMBER>
	// The wwid could contain white space and it will be replaced
	// underscore when wwid is exposed under /dev/by-id.

	fc_path := "scsi-" + wwid
	dev_id := "/dev/disk/by-id/"
	if dirs, err := io.ReadDir(dev_id); err == nil {
		for _, f := range dirs {
			name := f.Name()
			if name == fc_path {
				disk, err := io.EvalSymlinks(dev_id + name)
				if err != nil {
					glog.V(2).Infof("fc: failed to find a corresponding disk from symlink[%s], error %v", dev_id+name, err)
					return "", ""
				}
				arr := strings.Split(disk, "/")
				l := len(arr) - 1
				dev := arr[l]
				dm := findMultipathDeviceMapper(dev, io)
				return disk, dm
			}
		}
	}
	glog.V(2).Infof("fc: failed to find a disk [%s]", dev_id+fc_path)
	return "", ""
}

// Removes a scsi device based upon /dev/sdX name
func removeFromScsiSubsystem(deviceName string, io ioHandler) {
	fileName := "/sys/block/" + deviceName + "/device/delete"
	glog.V(4).Infof("fc: remove device from scsi-subsystem: path: %s", fileName)
	data := []byte("1")
	io.WriteFile(fileName, data, 0666)
}

// rescan scsi bus
func scsiHostRescan(io ioHandler) {
	scsi_path := "/sys/class/scsi_host/"
	if dirs, err := io.ReadDir(scsi_path); err == nil {
		for _, f := range dirs {
			name := scsi_path + f.Name() + "/scan"
			data := []byte("- - -")
			io.WriteFile(name, data, 0666)
		}
	}
}

// make a directory like /var/lib/kubelet/plugins/kubernetes.io/pod/fc/target-lun-0
func makePDNameInternal(host volume.VolumeHost, wwns []string, lun string, wwids []string) string {
	if len(wwns) != 0 {
		return path.Join(host.GetPluginDir(fcPluginName), wwns[0]+"-lun-"+lun)
	} else {
		return path.Join(host.GetPluginDir(fcPluginName), wwids[0])
	}
}

type FCUtil struct{}

func (util *FCUtil) MakeGlobalPDName(fc fcDisk) string {
	return makePDNameInternal(fc.plugin.host, fc.wwns, fc.lun, fc.wwids)
}

func searchDisk(b fcDiskMounter) (string, string) {
	var diskIds []string
	var disk string
	var dm string
	io := b.io
	wwids := b.wwids
	wwns := b.wwns
	lun := b.lun

	if len(wwns) != 0 {
		diskIds = wwns
	} else {
		diskIds = wwids
	}

	rescaned := false
	// two-phase search:
	// first phase, search existing device path, if a multipath dm is found, exit loop
	// otherwise, in second phase, rescan scsi bus and search again, return with any findings
	for true {
		for _, diskId := range diskIds {
			if len(wwns) != 0 {
				disk, dm = findDisk(diskId, lun, io)
			} else {
				disk, dm = findDiskWWIDs(diskId, io)
			}
			// if multipath device is found, break
			if dm != "" {
				break
			}
		}
		// if a dm is found, exit loop
		if rescaned || dm != "" {
			break
		}
		// rescan and search again
		// rescan scsi bus
		scsiHostRescan(io)
		rescaned = true
	}
	return disk, dm
}

func (util *FCUtil) AttachDisk(b fcDiskMounter) (string, error) {
	devicePath := ""
	var disk, dm string

	disk, dm = searchDisk(b)
	// if no disk matches input wwn and lun, exit
	if disk == "" && dm == "" {
		return "", fmt.Errorf("no fc disk found")
	}

	// if multipath devicemapper device is found, use it; otherwise use raw disk
	if dm != "" {
		devicePath = dm
	} else {
		devicePath = disk
	}
	// mount it
	globalPDPath := util.MakeGlobalPDName(*b.fcDisk)
	if err := os.MkdirAll(globalPDPath, 0750); err != nil {
		return devicePath, fmt.Errorf("fc: failed to mkdir %s, error", globalPDPath)
	}

	noMnt, err := b.mounter.IsLikelyNotMountPoint(globalPDPath)
	if err != nil {
		return devicePath, fmt.Errorf("Heuristic determination of mount point failed:%v", err)
	}
	if !noMnt {
		glog.Infof("fc: %s already mounted", globalPDPath)
		return devicePath, nil
	}

	err = b.mounter.FormatAndMount(devicePath, globalPDPath, b.fsType, nil)
	if err != nil {
		return devicePath, fmt.Errorf("fc: failed to mount fc volume %s [%s] to %s, error %v", devicePath, b.fsType, globalPDPath, err)
	}

	return devicePath, err
}

func (util *FCUtil) DetachDisk(c fcDiskUnmounter, devName string) error {
	// Remove scsi device from the node.
	if !strings.HasPrefix(devName, "/dev/") {
		return fmt.Errorf("fc detach disk: invalid device name: %s", devName)
	}
	arr := strings.Split(devName, "/")
	dev := arr[len(arr)-1]
	removeFromScsiSubsystem(dev, c.io)
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris

package net

import (
	"context"
	"internal/poll"
	"os"
	"runtime"
	"syscall"
)

const (
	readSyscallName     = "read"
	readFromSyscallName = "recvfrom"
	readMsgSyscallName  = "recvmsg"
	writeSyscallName    = "write"
	writeToSyscallName  = "sendto"
	writeMsgSyscallName = "sendmsg"
)

func newFD(sysfd, family, sotype int, net string) (*netFD, error) {
	ret := &netFD{
		pfd: poll.FD{
			Sysfd:         sysfd,
			IsStream:      sotype == syscall.SOCK_STREAM,
			ZeroReadIsEOF: sotype != syscall.SOCK_DGRAM && sotype != syscall.SOCK_RAW,
		},
		family: family,
		sotype: sotype,
		net:    net,
	}
	return ret, nil
}

func (fd *netFD) init() error {
	return fd.pfd.Init(fd.net, true)
}

func (fd *netFD) name() string {
	var ls, rs string
	if fd.laddr != nil {
		ls = fd.laddr.String()
	}
	if fd.raddr != nil {
		rs = fd.raddr.String()
	}
	return fd.net + ":" + ls + "->" + rs
}

func (fd *netFD) connect(ctx context.Context, la, ra syscall.Sockaddr) (rsa syscall.Sockaddr, ret error) {
	// Do not need to call fd.writeLock here,
	// because fd is not yet accessible to user,
	// so no concurrent operations are possible.
	switch err := connectFunc(fd.pfd.Sysfd, ra); err {
	case syscall.EINPROGRESS, syscall.EALREADY, syscall.EINTR:
	case nil, syscall.EISCONN:
		select {
		case <-ctx.Done():
			return nil, mapErr(ctx.Err())
		default:
		}
		if err := fd.pfd.Init(fd.net, true); err != nil {
			return nil, err
		}
		runtime.KeepAlive(fd)
		return nil, nil
	case syscall.EINVAL:
		// On Solaris and illumos we can see EINVAL if the socket has
		// already been accepted and closed by the server.  Treat this
		// as a successful connection--writes to the socket will see
		// EOF.  For details and a test case in C see
		// https://golang.org/issue/6828.
		if runtime.GOOS == "solaris" || runtime.GOOS == "illumos" {
			return nil, nil
		}
		fallthrough
	default:
		return nil, os.NewSyscallError("connect", err)
	}
	if err := fd.pfd.Init(fd.net, true); err != nil {
		return nil, err
	}
	if deadline, hasDeadline := ctx.Deadline(); hasDeadline {
		fd.pfd.SetWriteDeadline(deadline)
		defer fd.pfd.SetWriteDeadline(noDeadline)
	}

	// Start the "interrupter" goroutine, if this context might be canceled.
	// (The background context cannot)
	//
	// The interrupter goroutine waits for the context to be done and
	// interrupts the dial (by altering the fd's write deadline, which
	// wakes up waitWrite).
	if ctx != context.Background() {
		// Wait for the interrupter goroutine to exit before returning
		// from connect.
		done := make(chan struct{})
		interruptRes := make(chan error)
		defer func() {
			close(done)
			if ctxErr := <-interruptRes; ctxErr != nil && ret == nil {
				// The interrupter goroutine called SetWriteDeadline,
				// but the connect code below had returned from
				// waitWrite already and did a successful connect (ret
				// == nil). Because we've now poisoned the connection
				// by making it unwritable, don't return a successful
				// dial. This was issue 16523.
				ret = mapErr(ctxErr)
				fd.Close() // prevent a leak
			}
		}()
		go func() {
			select {
			case <-ctx.Done():
				// Force the runtime's poller to immediately give up
				// waiting for writability, unblocking waitWrite
				// below.
				fd.pfd.SetWriteDeadline(aLongTimeAgo)
				testHookCanceledDial()
				interruptRes <- ctx.Err()
			case <-done:
				interruptRes <- nil
			}
		}()
	}

	for {
		// Performing multiple connect system calls on a
		// non-blocking socket under Unix variants does not
		// necessarily result in earlier errors being
		// returned. Instead, once runtime-integrated network
		// poller tells us that the socket is ready, get the
		// SO_ERROR socket option to see if the connection
		// succeeded or failed. See issue 7474 for further
		// details.
		if err := fd.pfd.WaitWrite(); err != nil {
			select {
			case <-ctx.Done():
				return nil, mapErr(ctx.Err())
			default:
			}
			return nil, err
		}
		nerr, err := getsockoptIntFunc(fd.pfd.Sysfd, syscall.SOL_SOCKET, syscall.SO_ERROR)
		if err != nil {
			return nil, os.NewSyscallError("getsockopt", err)
		}
		switch err := syscall.Errno(nerr); err {
		case syscall.EINPROGRESS, syscall.EALREADY, syscall.EINTR:
		case syscall.EISCONN:
			return nil, nil
		case syscall.Errno(0):
			// The runtime poller can wake us up spuriously;
			// see issues 14548 and 19289. Check that we are
			// really connected; if not, wait again.
			if rsa, err := syscall.Getpeername(fd.pfd.Sysfd); err == nil {
				return rsa, nil
			}
		default:
			return nil, os.NewSyscallError("connect", err)
		}
		runtime.KeepAlive(fd)
	}
}

func (fd *netFD) accept() (netfd *netFD, err error) {
	d, rsa, errcall, err := fd.pfd.Accept()
	if err != nil {
		if errcall != "" {
			err = wrapSyscallError(errcall, err)
		}
		return nil, err
	}

	if netfd, err = newFD(d, fd.family, fd.sotype, fd.net); err != nil {
		poll.CloseFunc(d)
		return nil, err
	}
	if err = netfd.init(); err != nil {
		netfd.Close()
		return nil, err
	}
	lsa, _ := syscall.Getsockname(netfd.pfd.Sysfd)
	netfd.setAddr(netfd.addrFunc()(lsa), netfd.addrFunc()(rsa))
	return netfd, nil
}

func (fd *netFD) dup() (f *os.File, err error) {
	ns, call, err := fd.pfd.Dup()
	if err != nil {
		if call != "" {
			err = os.NewSyscallError(call, err)
		}
		return nil, err
	}

	return os.NewFile(uintptr(ns), fd.name()), nil
}
<commit_msg>net: use Done rather than comparing with context.Background<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris

package net

import (
	"context"
	"internal/poll"
	"os"
	"runtime"
	"syscall"
)

const (
	readSyscallName     = "read"
	readFromSyscallName = "recvfrom"
	readMsgSyscallName  = "recvmsg"
	writeSyscallName    = "write"
	writeToSyscallName  = "sendto"
	writeMsgSyscallName = "sendmsg"
)

func newFD(sysfd, family, sotype int, net string) (*netFD, error) {
	ret := &netFD{
		pfd: poll.FD{
			Sysfd:         sysfd,
			IsStream:      sotype == syscall.SOCK_STREAM,
			ZeroReadIsEOF: sotype != syscall.SOCK_DGRAM && sotype != syscall.SOCK_RAW,
		},
		family: family,
		sotype: sotype,
		net:    net,
	}
	return ret, nil
}

func (fd *netFD) init() error {
	return fd.pfd.Init(fd.net, true)
}

func (fd *netFD) name() string {
	var ls, rs string
	if fd.laddr != nil {
		ls = fd.laddr.String()
	}
	if fd.raddr != nil {
		rs = fd.raddr.String()
	}
	return fd.net + ":" + ls + "->" + rs
}

func (fd *netFD) connect(ctx context.Context, la, ra syscall.Sockaddr) (rsa syscall.Sockaddr, ret error) {
	// Do not need to call fd.writeLock here,
	// because fd is not yet accessible to user,
	// so no concurrent operations are possible.
	switch err := connectFunc(fd.pfd.Sysfd, ra); err {
	case syscall.EINPROGRESS, syscall.EALREADY, syscall.EINTR:
	case nil, syscall.EISCONN:
		select {
		case <-ctx.Done():
			return nil, mapErr(ctx.Err())
		default:
		}
		if err := fd.pfd.Init(fd.net, true); err != nil {
			return nil, err
		}
		runtime.KeepAlive(fd)
		return nil, nil
	case syscall.EINVAL:
		// On Solaris and illumos we can see EINVAL if the socket has
		// already been accepted and closed by the server.  Treat this
		// as a successful connection--writes to the socket will see
		// EOF.  For details and a test case in C see
		// https://golang.org/issue/6828.
		if runtime.GOOS == "solaris" || runtime.GOOS == "illumos" {
			return nil, nil
		}
		fallthrough
	default:
		return nil, os.NewSyscallError("connect", err)
	}
	if err := fd.pfd.Init(fd.net, true); err != nil {
		return nil, err
	}
	if deadline, hasDeadline := ctx.Deadline(); hasDeadline {
		fd.pfd.SetWriteDeadline(deadline)
		defer fd.pfd.SetWriteDeadline(noDeadline)
	}

	// Start the "interrupter" goroutine, if this context might be canceled.
	//
	// The interrupter goroutine waits for the context to be done and
	// interrupts the dial (by altering the fd's write deadline, which
	// wakes up waitWrite).
	if ctxDone := ctx.Done(); ctxDone != nil {
		// Wait for the interrupter goroutine to exit before returning
		// from connect.
		done := make(chan struct{})
		interruptRes := make(chan error)
		defer func() {
			close(done)
			if ctxErr := <-interruptRes; ctxErr != nil && ret == nil {
				// The interrupter goroutine called SetWriteDeadline,
				// but the connect code below had returned from
				// waitWrite already and did a successful connect (ret
				// == nil). Because we've now poisoned the connection
				// by making it unwritable, don't return a successful
				// dial. This was issue 16523.
				ret = mapErr(ctxErr)
				fd.Close() // prevent a leak
			}
		}()
		go func() {
			select {
			case <-ctxDone:
				// Force the runtime's poller to immediately give up
				// waiting for writability, unblocking waitWrite
				// below.
				fd.pfd.SetWriteDeadline(aLongTimeAgo)
				testHookCanceledDial()
				interruptRes <- ctx.Err()
			case <-done:
				interruptRes <- nil
			}
		}()
	}

	for {
		// Performing multiple connect system calls on a
		// non-blocking socket under Unix variants does not
		// necessarily result in earlier errors being
		// returned. Instead, once runtime-integrated network
		// poller tells us that the socket is ready, get the
		// SO_ERROR socket option to see if the connection
		// succeeded or failed. See issue 7474 for further
		// details.
		if err := fd.pfd.WaitWrite(); err != nil {
			select {
			case <-ctx.Done():
				return nil, mapErr(ctx.Err())
			default:
			}
			return nil, err
		}
		nerr, err := getsockoptIntFunc(fd.pfd.Sysfd, syscall.SOL_SOCKET, syscall.SO_ERROR)
		if err != nil {
			return nil, os.NewSyscallError("getsockopt", err)
		}
		switch err := syscall.Errno(nerr); err {
		case syscall.EINPROGRESS, syscall.EALREADY, syscall.EINTR:
		case syscall.EISCONN:
			return nil, nil
		case syscall.Errno(0):
			// The runtime poller can wake us up spuriously;
			// see issues 14548 and 19289. Check that we are
			// really connected; if not, wait again.
			if rsa, err := syscall.Getpeername(fd.pfd.Sysfd); err == nil {
				return rsa, nil
			}
		default:
			return nil, os.NewSyscallError("connect", err)
		}
		runtime.KeepAlive(fd)
	}
}

func (fd *netFD) accept() (netfd *netFD, err error) {
	d, rsa, errcall, err := fd.pfd.Accept()
	if err != nil {
		if errcall != "" {
			err = wrapSyscallError(errcall, err)
		}
		return nil, err
	}

	if netfd, err = newFD(d, fd.family, fd.sotype, fd.net); err != nil {
		poll.CloseFunc(d)
		return nil, err
	}
	if err = netfd.init(); err != nil {
		netfd.Close()
		return nil, err
	}
	lsa, _ := syscall.Getsockname(netfd.pfd.Sysfd)
	netfd.setAddr(netfd.addrFunc()(lsa), netfd.addrFunc()(rsa))
	return netfd, nil
}

func (fd *netFD) dup() (f *os.File, err error) {
	ns, call, err := fd.pfd.Dup()
	if err != nil {
		if call != "" {
			err = os.NewSyscallError(call, err)
		}
		return nil, err
	}

	return os.NewFile(uintptr(ns), fd.name()), nil
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package net

import (
	"context"
	"internal/itoa"
	"io"
	"net/netip"
	"os"
	"syscall"
	"time"
)

// BUG(mikio): On JS and Windows, the File method of TCPConn and
// TCPListener is not implemented.

// TCPAddr represents the address of a TCP end point.
type TCPAddr struct {
	IP   IP
	Port int
	Zone string // IPv6 scoped addressing zone
}

// AddrPort returns the TCPAddr a as a netip.AddrPort.
//
// If a.Port does not fit in a uint16, it's silently truncated.
//
// If a is nil, a zero value is returned.
func (a *TCPAddr) AddrPort() netip.AddrPort {
	if a == nil {
		return netip.AddrPort{}
	}
	na, _ := netip.AddrFromSlice(a.IP)
	na = na.WithZone(a.Zone)
	return netip.AddrPortFrom(na, uint16(a.Port))
}

// Network returns the address's network name, "tcp".
func (a *TCPAddr) Network() string { return "tcp" }

func (a *TCPAddr) String() string {
	if a == nil {
		return "<nil>"
	}
	ip := ipEmptyString(a.IP)
	if a.Zone != "" {
		return JoinHostPort(ip+"%"+a.Zone, itoa.Itoa(a.Port))
	}
	return JoinHostPort(ip, itoa.Itoa(a.Port))
}

func (a *TCPAddr) isWildcard() bool {
	if a == nil || a.IP == nil {
		return true
	}
	return a.IP.IsUnspecified()
}

func (a *TCPAddr) opAddr() Addr {
	if a == nil {
		return nil
	}
	return a
}

// ResolveTCPAddr returns an address of TCP end point.
//
// The network must be a TCP network name.
//
// If the host in the address parameter is not a literal IP address or
// the port is not a literal port number, ResolveTCPAddr resolves the
// address to an address of TCP end point.
// Otherwise, it parses the address as a pair of literal IP address
// and port number.
// The address parameter can use a host name, but this is not
// recommended, because it will return at most one of the host name's
// IP addresses.
//
// See func Dial for a description of the network and address
// parameters.
func ResolveTCPAddr(network, address string) (*TCPAddr, error) {
	switch network {
	case "tcp", "tcp4", "tcp6":
	case "": // a hint wildcard for Go 1.0 undocumented behavior
		network = "tcp"
	default:
		return nil, UnknownNetworkError(network)
	}
	addrs, err := DefaultResolver.internetAddrList(context.Background(), network, address)
	if err != nil {
		return nil, err
	}
	return addrs.forResolve(network, address).(*TCPAddr), nil
}

// TCPConn is an implementation of the Conn interface for TCP network
// connections.
type TCPConn struct {
	conn
}

// SyscallConn returns a raw network connection.
// This implements the syscall.Conn interface.
func (c *TCPConn) SyscallConn() (syscall.RawConn, error) {
	if !c.ok() {
		return nil, syscall.EINVAL
	}
	return newRawConn(c.fd)
}

// ReadFrom implements the io.ReaderFrom ReadFrom method.
func (c *TCPConn) ReadFrom(r io.Reader) (int64, error) {
	if !c.ok() {
		return 0, syscall.EINVAL
	}
	n, err := c.readFrom(r)
	if err != nil && err != io.EOF {
		err = &OpError{Op: "readfrom", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err}
	}
	return n, err
}

// CloseRead shuts down the reading side of the TCP connection.
// Most callers should just use Close.
func (c *TCPConn) CloseRead() error {
	if !c.ok() {
		return syscall.EINVAL
	}
	if err := c.fd.closeRead(); err != nil {
		return &OpError{Op: "close", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err}
	}
	return nil
}

// CloseWrite shuts down the writing side of the TCP connection.
// Most callers should just use Close.
func (c *TCPConn) CloseWrite() error {
	if !c.ok() {
		return syscall.EINVAL
	}
	if err := c.fd.closeWrite(); err != nil {
		return &OpError{Op: "close", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err}
	}
	return nil
}

// SetLinger sets the behavior of Close on a connection which still
// has data waiting to be sent or to be acknowledged.
//
// If sec < 0 (the default), the operating system finishes sending the
// data in the background.
//
// If sec == 0, the operating system discards any unsent or
// unacknowledged data.
//
// If sec > 0, the data is sent in the background as with sec < 0. On
// some operating systems after sec seconds have elapsed any remaining
// unsent data may be discarded.
func (c *TCPConn) SetLinger(sec int) error {
	if !c.ok() {
		return syscall.EINVAL
	}
	if err := setLinger(c.fd, sec); err != nil {
		return &OpError{Op: "set", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err}
	}
	return nil
}

// SetKeepAlive sets whether the operating system should send
// keep-alive messages on the connection.
func (c *TCPConn) SetKeepAlive(keepalive bool) error {
	if !c.ok() {
		return syscall.EINVAL
	}
	if err := setKeepAlive(c.fd, keepalive); err != nil {
		return &OpError{Op: "set", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err}
	}
	return nil
}

// SetKeepAlivePeriod sets period between keep-alives.
func (c *TCPConn) SetKeepAlivePeriod(d time.Duration) error {
	if !c.ok() {
		return syscall.EINVAL
	}
	if err := setKeepAlivePeriod(c.fd, d); err != nil {
		return &OpError{Op: "set", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err}
	}
	return nil
}

// SetNoDelay controls whether the operating system should delay
// packet transmission in hopes of sending fewer packets (Nagle's
// algorithm).  The default is true (no delay), meaning that data is
// sent as soon as possible after a Write.
func (c *TCPConn) SetNoDelay(noDelay bool) error {
	if !c.ok() {
		return syscall.EINVAL
	}
	if err := setNoDelay(c.fd, noDelay); err != nil {
		return &OpError{Op: "set", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err}
	}
	return nil
}

func newTCPConn(fd *netFD) *TCPConn {
	c := &TCPConn{conn{fd}}
	setNoDelay(c.fd, true)
	return c
}

// DialTCP acts like Dial for TCP networks.
//
// The network must be a TCP network name; see func Dial for details.
//
// If laddr is nil, a local address is automatically chosen.
// If the IP field of raddr is nil or an unspecified IP address, the
// local system is assumed.
func DialTCP(network string, laddr, raddr *TCPAddr) (*TCPConn, error) {
	switch network {
	case "tcp", "tcp4", "tcp6":
	default:
		return nil, &OpError{Op: "dial", Net: network, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: UnknownNetworkError(network)}
	}
	if raddr == nil {
		return nil, &OpError{Op: "dial", Net: network, Source: laddr.opAddr(), Addr: nil, Err: errMissingAddress}
	}
	sd := &sysDialer{network: network, address: raddr.String()}
	c, err := sd.dialTCP(context.Background(), laddr, raddr)
	if err != nil {
		return nil, &OpError{Op: "dial", Net: network, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err}
	}
	return c, nil
}

// TCPListener is a TCP network listener. Clients should typically
// use variables of type Listener instead of assuming TCP.
type TCPListener struct {
	fd *netFD
	lc ListenConfig
}

// SyscallConn returns a raw network connection.
// This implements the syscall.Conn interface.
//
// The returned RawConn only supports calling Control. Read and
// Write return an error.
func (l *TCPListener) SyscallConn() (syscall.RawConn, error) {
	if !l.ok() {
		return nil, syscall.EINVAL
	}
	return newRawListener(l.fd)
}

// AcceptTCP accepts the next incoming call and returns the new
// connection.
func (l *TCPListener) AcceptTCP() (*TCPConn, error) {
	if !l.ok() {
		return nil, syscall.EINVAL
	}
	c, err := l.accept()
	if err != nil {
		return nil, &OpError{Op: "accept", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err}
	}
	return c, nil
}

// Accept implements the Accept method in the Listener interface; it
// waits for the next call and returns a generic Conn.
func (l *TCPListener) Accept() (Conn, error) {
	if !l.ok() {
		return nil, syscall.EINVAL
	}
	c, err := l.accept()
	if err != nil {
		return nil, &OpError{Op: "accept", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err}
	}
	return c, nil
}

// Close stops listening on the TCP address.
// Already Accepted connections are not closed.
func (l *TCPListener) Close() error {
	if !l.ok() {
		return syscall.EINVAL
	}
	if err := l.close(); err != nil {
		return &OpError{Op: "close", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err}
	}
	return nil
}

// Addr returns the listener's network address, a *TCPAddr.
// The Addr returned is shared by all invocations of Addr, so
// do not modify it.
func (l *TCPListener) Addr() Addr { return l.fd.laddr }

// SetDeadline sets the deadline associated with the listener.
// A zero time value disables the deadline.
func (l *TCPListener) SetDeadline(t time.Time) error {
	if !l.ok() {
		return syscall.EINVAL
	}
	if err := l.fd.pfd.SetDeadline(t); err != nil {
		return &OpError{Op: "set", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err}
	}
	return nil
}

// File returns a copy of the underlying os.File.
// It is the caller's responsibility to close f when finished.
// Closing l does not affect f, and closing f does not affect l.
//
// The returned os.File's file descriptor is different from the
// connection's. Attempting to change properties of the original
// using this duplicate may or may not have the desired effect.
func (l *TCPListener) File() (f *os.File, err error) {
	if !l.ok() {
		return nil, syscall.EINVAL
	}
	f, err = l.file()
	if err != nil {
		return nil, &OpError{Op: "file", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err}
	}
	return
}

// ListenTCP acts like Listen for TCP networks.
//
// The network must be a TCP network name; see func Dial for details.
//
// If the IP field of laddr is nil or an unspecified IP address,
// ListenTCP listens on all available unicast and anycast IP addresses
// of the local system.
// If the Port field of laddr is 0, a port number is automatically
// chosen.
func ListenTCP(network string, laddr *TCPAddr) (*TCPListener, error) {
	switch network {
	case "tcp", "tcp4", "tcp6":
	default:
		return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: laddr.opAddr(), Err: UnknownNetworkError(network)}
	}
	if laddr == nil {
		laddr = &TCPAddr{}
	}
	sl := &sysListener{network: network, address: laddr.String()}
	ln, err := sl.listenTCP(context.Background(), laddr)
	if err != nil {
		return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: laddr.opAddr(), Err: err}
	}
	return ln, nil
}

// roundDurationUp rounds d to the next multiple of to.
func roundDurationUp(d time.Duration, to time.Duration) time.Duration {
	return (d + to - 1) / to
}
<commit_msg>net: add conversion from AddrPort to TCPAddr to complement existing inverse<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package net

import (
	"context"
	"internal/itoa"
	"io"
	"net/netip"
	"os"
	"syscall"
	"time"
)

// BUG(mikio): On JS and Windows, the File method of TCPConn and
// TCPListener is not implemented.

// TCPAddr represents the address of a TCP end point.
type TCPAddr struct {
	IP   IP
	Port int
	Zone string // IPv6 scoped addressing zone
}

// AddrPort returns the TCPAddr a as a netip.AddrPort.
//
// If a.Port does not fit in a uint16, it's silently truncated.
//
// If a is nil, a zero value is returned.
func (a *TCPAddr) AddrPort() netip.AddrPort {
	if a == nil {
		return netip.AddrPort{}
	}
	na, _ := netip.AddrFromSlice(a.IP)
	na = na.WithZone(a.Zone)
	return netip.AddrPortFrom(na, uint16(a.Port))
}

// Network returns the address's network name, "tcp".
func (a *TCPAddr) Network() string { return "tcp" }

func (a *TCPAddr) String() string {
	if a == nil {
		return "<nil>"
	}
	ip := ipEmptyString(a.IP)
	if a.Zone != "" {
		return JoinHostPort(ip+"%"+a.Zone, itoa.Itoa(a.Port))
	}
	return JoinHostPort(ip, itoa.Itoa(a.Port))
}

func (a *TCPAddr) isWildcard() bool {
	if a == nil || a.IP == nil {
		return true
	}
	return a.IP.IsUnspecified()
}

func (a *TCPAddr) opAddr() Addr {
	if a == nil {
		return nil
	}
	return a
}

// ResolveTCPAddr returns an address of TCP end point.
//
// The network must be a TCP network name.
//
// If the host in the address parameter is not a literal IP address or
// the port is not a literal port number, ResolveTCPAddr resolves the
// address to an address of TCP end point.
// Otherwise, it parses the address as a pair of literal IP address
// and port number.
// The address parameter can use a host name, but this is not
// recommended, because it will return at most one of the host name's
// IP addresses.
//
// See func Dial for a description of the network and address
// parameters.
func ResolveTCPAddr(network, address string) (*TCPAddr, error) {
	switch network {
	case "tcp", "tcp4", "tcp6":
	case "": // a hint wildcard for Go 1.0 undocumented behavior
		network = "tcp"
	default:
		return nil, UnknownNetworkError(network)
	}
	addrs, err := DefaultResolver.internetAddrList(context.Background(), network, address)
	if err != nil {
		return nil, err
	}
	return addrs.forResolve(network, address).(*TCPAddr), nil
}

// TCPAddrFromAddrPort returns addr as a TCPAddr. If addr.IsValid() is false,
// then the returned TCPAddr will contain a nil IP field, indicating an
// address family-agnostic unspecified address.
func TCPAddrFromAddrPort(addr netip.AddrPort) *TCPAddr {
	return &TCPAddr{
		IP:   addr.Addr().AsSlice(),
		Zone: addr.Addr().Zone(),
		Port: int(addr.Port()),
	}
}

// TCPConn is an implementation of the Conn interface for TCP network
// connections.
type TCPConn struct {
	conn
}

// SyscallConn returns a raw network connection.
// This implements the syscall.Conn interface.
func (c *TCPConn) SyscallConn() (syscall.RawConn, error) {
	if !c.ok() {
		return nil, syscall.EINVAL
	}
	return newRawConn(c.fd)
}

// ReadFrom implements the io.ReaderFrom ReadFrom method.
func (c *TCPConn) ReadFrom(r io.Reader) (int64, error) {
	if !c.ok() {
		return 0, syscall.EINVAL
	}
	n, err := c.readFrom(r)
	if err != nil && err != io.EOF {
		err = &OpError{Op: "readfrom", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err}
	}
	return n, err
}

// CloseRead shuts down the reading side of the TCP connection.
// Most callers should just use Close.
func (c *TCPConn) CloseRead() error {
	if !c.ok() {
		return syscall.EINVAL
	}
	if err := c.fd.closeRead(); err != nil {
		return &OpError{Op: "close", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err}
	}
	return nil
}

// CloseWrite shuts down the writing side of the TCP connection.
// Most callers should just use Close.
func (c *TCPConn) CloseWrite() error {
	if !c.ok() {
		return syscall.EINVAL
	}
	if err := c.fd.closeWrite(); err != nil {
		return &OpError{Op: "close", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err}
	}
	return nil
}

// SetLinger sets the behavior of Close on a connection which still
// has data waiting to be sent or to be acknowledged.
//
// If sec < 0 (the default), the operating system finishes sending the
// data in the background.
//
// If sec == 0, the operating system discards any unsent or
// unacknowledged data.
//
// If sec > 0, the data is sent in the background as with sec < 0. On
// some operating systems after sec seconds have elapsed any remaining
// unsent data may be discarded.
func (c *TCPConn) SetLinger(sec int) error {
	if !c.ok() {
		return syscall.EINVAL
	}
	if err := setLinger(c.fd, sec); err != nil {
		return &OpError{Op: "set", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err}
	}
	return nil
}

// SetKeepAlive sets whether the operating system should send
// keep-alive messages on the connection.
func (c *TCPConn) SetKeepAlive(keepalive bool) error {
	if !c.ok() {
		return syscall.EINVAL
	}
	if err := setKeepAlive(c.fd, keepalive); err != nil {
		return &OpError{Op: "set", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err}
	}
	return nil
}

// SetKeepAlivePeriod sets period between keep-alives.
func (c *TCPConn) SetKeepAlivePeriod(d time.Duration) error {
	if !c.ok() {
		return syscall.EINVAL
	}
	if err := setKeepAlivePeriod(c.fd, d); err != nil {
		return &OpError{Op: "set", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err}
	}
	return nil
}

// SetNoDelay controls whether the operating system should delay
// packet transmission in hopes of sending fewer packets (Nagle's
// algorithm).  The default is true (no delay), meaning that data is
// sent as soon as possible after a Write.
func (c *TCPConn) SetNoDelay(noDelay bool) error {
	if !c.ok() {
		return syscall.EINVAL
	}
	if err := setNoDelay(c.fd, noDelay); err != nil {
		return &OpError{Op: "set", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err}
	}
	return nil
}

func newTCPConn(fd *netFD) *TCPConn {
	c := &TCPConn{conn{fd}}
	setNoDelay(c.fd, true)
	return c
}

// DialTCP acts like Dial for TCP networks.
//
// The network must be a TCP network name; see func Dial for details.
//
// If laddr is nil, a local address is automatically chosen.
// If the IP field of raddr is nil or an unspecified IP address, the
// local system is assumed.
func DialTCP(network string, laddr, raddr *TCPAddr) (*TCPConn, error) {
	switch network {
	case "tcp", "tcp4", "tcp6":
	default:
		return nil, &OpError{Op: "dial", Net: network, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: UnknownNetworkError(network)}
	}
	if raddr == nil {
		return nil, &OpError{Op: "dial", Net: network, Source: laddr.opAddr(), Addr: nil, Err: errMissingAddress}
	}
	sd := &sysDialer{network: network, address: raddr.String()}
	c, err := sd.dialTCP(context.Background(), laddr, raddr)
	if err != nil {
		return nil, &OpError{Op: "dial", Net: network, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err}
	}
	return c, nil
}

// TCPListener is a TCP network listener. Clients should typically
// use variables of type Listener instead of assuming TCP.
type TCPListener struct {
	fd *netFD
	lc ListenConfig
}

// SyscallConn returns a raw network connection.
// This implements the syscall.Conn interface.
//
// The returned RawConn only supports calling Control. Read and
// Write return an error.
func (l *TCPListener) SyscallConn() (syscall.RawConn, error) {
	if !l.ok() {
		return nil, syscall.EINVAL
	}
	return newRawListener(l.fd)
}

// AcceptTCP accepts the next incoming call and returns the new
// connection.
func (l *TCPListener) AcceptTCP() (*TCPConn, error) {
	if !l.ok() {
		return nil, syscall.EINVAL
	}
	c, err := l.accept()
	if err != nil {
		return nil, &OpError{Op: "accept", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err}
	}
	return c, nil
}

// Accept implements the Accept method in the Listener interface; it
// waits for the next call and returns a generic Conn.
func (l *TCPListener) Accept() (Conn, error) {
	if !l.ok() {
		return nil, syscall.EINVAL
	}
	c, err := l.accept()
	if err != nil {
		return nil, &OpError{Op: "accept", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err}
	}
	return c, nil
}

// Close stops listening on the TCP address.
// Already Accepted connections are not closed.
func (l *TCPListener) Close() error {
	if !l.ok() {
		return syscall.EINVAL
	}
	if err := l.close(); err != nil {
		return &OpError{Op: "close", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err}
	}
	return nil
}

// Addr returns the listener's network address, a *TCPAddr.
// The Addr returned is shared by all invocations of Addr, so
// do not modify it.
func (l *TCPListener) Addr() Addr { return l.fd.laddr }

// SetDeadline sets the deadline associated with the listener.
// A zero time value disables the deadline.
func (l *TCPListener) SetDeadline(t time.Time) error {
	if !l.ok() {
		return syscall.EINVAL
	}
	if err := l.fd.pfd.SetDeadline(t); err != nil {
		return &OpError{Op: "set", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err}
	}
	return nil
}

// File returns a copy of the underlying os.File.
// It is the caller's responsibility to close f when finished.
// Closing l does not affect f, and closing f does not affect l.
//
// The returned os.File's file descriptor is different from the
// connection's. Attempting to change properties of the original
// using this duplicate may or may not have the desired effect.
func (l *TCPListener) File() (f *os.File, err error) {
	if !l.ok() {
		return nil, syscall.EINVAL
	}
	f, err = l.file()
	if err != nil {
		return nil, &OpError{Op: "file", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err}
	}
	return
}

// ListenTCP acts like Listen for TCP networks.
//
// The network must be a TCP network name; see func Dial for details.
//
// If the IP field of laddr is nil or an unspecified IP address,
// ListenTCP listens on all available unicast and anycast IP addresses
// of the local system.
// If the Port field of laddr is 0, a port number is automatically
// chosen.
func ListenTCP(network string, laddr *TCPAddr) (*TCPListener, error) {
	switch network {
	case "tcp", "tcp4", "tcp6":
	default:
		return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: laddr.opAddr(), Err: UnknownNetworkError(network)}
	}
	if laddr == nil {
		laddr = &TCPAddr{}
	}
	sl := &sysListener{network: network, address: laddr.String()}
	ln, err := sl.listenTCP(context.Background(), laddr)
	if err != nil {
		return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: laddr.opAddr(), Err: err}
	}
	return ln, nil
}

// roundDurationUp rounds d to the next multiple of to.
func roundDurationUp(d time.Duration, to time.Duration) time.Duration {
	return (d + to - 1) / to
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/gorilla/context"
	"github.com/gorilla/sessions"
	"github.com/rs/cors"
	"encoding/json"
	"log"
	"net/http"
)

type State struct {
	Participators []sessions.Session   // All participators at the student division meeting.
	SpeakerLists  [][]sessions.Session // A list of speakerLists where each index is a list of sessions in queue to speak
}

var store = sessions.NewCookieStore([]byte("this is the secret stuff"))
var state State

func listHandler(w http.ResponseWriter, req *http.Request) {
	log.Print("Listhandler begin")
	session, err := store.Get(req, "talarlista_session")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	if session.IsNew {
		state.Participators = append(state.Participators, *session)
	}

	session.Options = &sessions.Options{
		MaxAge:   86400,
		HttpOnly: true,
	}

	session.Save(req, w)

	w.Header().Set("Content-Type", "text/plain")
	w.Header().Set("Access-Control-Allow-Origin", "*")

	switch req.Method {
	case http.MethodGet:
		listGet(w, session)
	case http.MethodPost:
		listPost(w, session)
	case http.MethodDelete:
		listDelete(w, session)
	default:
		w.Write([]byte("List unsupported method.\n"))
		log.Print("Unsupported method")
	}
}

func listGet(w http.ResponseWriter, session *sessions.Session) {
	b, err := json.Marshal(state)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.Write(b)
}

func listPost(w http.ResponseWriter, session *sessions.Session) {
	w.Write([]byte("List post.\n"))
	session.Values["registered"] = true

	currentSpeakerList := state.SpeakerLists[len(state.SpeakerLists)-1]

	if isRegistered(session, currentSpeakerList) {
		http.Error(w, "Already registered", http.StatusUnprocessableEntity)
		return
	} else {
		state.SpeakerLists[len(state.SpeakerLists)-1] = append(currentSpeakerList, *session)
		w.Write([]byte("Added to speakerslist"))
	}
}

func listDelete(w http.ResponseWriter, session *sessions.Session) {
	session.Values["registered"] = false
	currentSpeakerList := state.SpeakerLists[len(state.SpeakerLists)-1]
	if isRegistered(session, currentSpeakerList) {
		state.SpeakerLists[len(state.SpeakerLists)-1] = removeSessionFromList(session, currentSpeakerList)
		w.Write([]byte("Removed from speakerslist\n"))
	} else {
		http.Error(w, "Not in list", http.StatusUnprocessableEntity)
		return
	}

	w.Write([]byte("List delete.\n"))
}

func isRegistered(currentSession *sessions.Session, speakersList []sessions.Session) bool {
	for _, session := range speakersList {
		if (*currentSession).ID == session.ID {
			return true
		}
	}
	return false
}

func removeSessionFromList(session *sessions.Session, sessionList []sessions.Session) []sessions.Session {
	sessionIndex := -1
	for i, s := range sessionList {
		if (*session).ID == s.ID {
			sessionIndex = i
			break
		}
	}

	if sessionIndex == -1 {
		return sessionList
	} else {
		return append(sessionList[:sessionIndex], sessionList[sessionIndex+1:]...)
	}
}

func adminHandler(w http.ResponseWriter, req *http.Request) {
	w.Header().Set("Content-Type", "text/plain")
	w.Write([]byte("helo\n"))
}

func main() {
	state.SpeakerLists = append(state.SpeakerLists, []sessions.Session{})

	mux := http.NewServeMux()
	mux.HandleFunc("/", listHandler)
	mux.HandleFunc("/list", listHandler)
	mux.HandleFunc("/admin", adminHandler)

	handler := cors.Default().Handler(mux)

	c := cors.New(cors.Options{
		//Debug: true,
		AllowedMethods: []string{"GET", "POST", "DELETE"},
	})
	handler = c.Handler(handler)

	handler = context.ClearHandler(handler)

	log.Printf("About to listen on 3001. Go to http://127.0.0.1:3001/")
	//err := http.ListenAndServeTLS(":3001", "cert.pem", "key.pem", nil)
	err := http.ListenAndServe(":3001", handler)
	log.Fatal(err)
}
<commit_msg>Host backend on port 3001<commit_after>package main

import (
	"github.com/gorilla/context"
	"github.com/gorilla/sessions"
	"github.com/rs/cors"
	"encoding/json"
	"log"
	"net/http"
)

type State struct {
	Participators []sessions.Session   // All participators at the student division meeting.
	SpeakerLists  [][]sessions.Session // A list of speakerLists where each index is a list of sessions in queue to speak
}

var store = sessions.NewCookieStore([]byte("this is the secret stuff"))
var state State

func listHandler(w http.ResponseWriter, req *http.Request) {
	log.Print("Listhandler begin")
	session, err := store.Get(req, "talarlista_session")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	if session.IsNew {
		state.Participators = append(state.Participators, *session)
	}

	session.Options = &sessions.Options{
		MaxAge:   86400,
		HttpOnly: true,
	}

	session.Save(req, w)

	w.Header().Set("Content-Type", "text/plain")
	w.Header().Set("Access-Control-Allow-Origin", "*")

	switch req.Method {
	case http.MethodGet:
		listGet(w, session)
	case http.MethodPost:
		listPost(w, session)
	case http.MethodDelete:
		listDelete(w, session)
	default:
		w.Write([]byte("List unsupported method.\n"))
		log.Print("Unsupported method")
	}
}

func listGet(w http.ResponseWriter, session *sessions.Session) {
	b, err := json.Marshal(state)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.Write(b)
}

func listPost(w http.ResponseWriter, session *sessions.Session) {
	w.Write([]byte("List post.\n"))
	session.Values["registered"] = true

	currentSpeakerList := state.SpeakerLists[len(state.SpeakerLists)-1]

	if isRegistered(session, currentSpeakerList) {
		http.Error(w, "Already registered", http.StatusUnprocessableEntity)
		return
	} else {
		state.SpeakerLists[len(state.SpeakerLists)-1] = append(currentSpeakerList, *session)
		w.Write([]byte("Added to speakerslist"))
	}
}

func listDelete(w http.ResponseWriter, session *sessions.Session) {
	session.Values["registered"] = false
	currentSpeakerList := state.SpeakerLists[len(state.SpeakerLists)-1]
	if isRegistered(session, currentSpeakerList) {
		state.SpeakerLists[len(state.SpeakerLists)-1] = removeSessionFromList(session, currentSpeakerList)
		w.Write([]byte("Removed from speakerslist\n"))
	} else {
		http.Error(w, "Not in list", http.StatusUnprocessableEntity)
		return
	}

	w.Write([]byte("List delete.\n"))
}

func isRegistered(currentSession *sessions.Session, speakersList []sessions.Session) bool {
	for _, session := range speakersList {
		if (*currentSession).ID == session.ID {
			return true
		}
	}
	return false
}

func removeSessionFromList(session *sessions.Session, sessionList []sessions.Session) []sessions.Session {
	sessionIndex := -1
	for i, s := range sessionList {
		if (*session).ID == s.ID {
			sessionIndex = i
			break
		}
	}

	if sessionIndex == -1 {
		return sessionList
	} else {
		return append(sessionList[:sessionIndex], sessionList[sessionIndex+1:]...)
	}
}

func adminHandler(w http.ResponseWriter, req *http.Request) {
	w.Header().Set("Content-Type", "text/plain")
	w.Write([]byte("helo\n"))
}

func main() {
	state.SpeakerLists = append(state.SpeakerLists, []sessions.Session{})

	mux := http.NewServeMux()
	mux.HandleFunc("/", listHandler)
	mux.HandleFunc("/list", listHandler)
	mux.HandleFunc("/admin", adminHandler)

	handler := cors.Default().Handler(mux)

	c := cors.New(cors.Options{
		//Debug: true,
		AllowedMethods: []string{"GET", "POST", "DELETE"},
	})
	handler = c.Handler(handler)

	handler = context.ClearHandler(handler)

	log.Print("About to listen on 3001. Go to http://127.0.0.1:3001/")
	//err := http.ListenAndServeTLS(":3001", "cert.pem", "key.pem", nil)
	err := http.ListenAndServe(":3001", handler)
	log.Fatal(err)
}
<|endoftext|>


<commit_before>package msgboard

import (
	"appengine"
	"appengine/datastore"
	"errors"
	"io"
	"time"

	"encoding/json"
	"github.com/russross/blackfriday"
)

// Page models a slide to be shown on the message board.
type Page struct {
	Title       string    `json:"title"`                        // Title of the Page
	Content     string    `json:"content" datastore:",noindex"` // Markdown text content.
	LastUpdated time.Time `json:"last_updated"`                 // Timestamp the content was last updated.
	ID          string    `json:"id" datastore:"-"`             // Encoded datastore key acting as an ID.
	Rendered    string    `json:"rendered" datastore:"-"`       // HTML for the page based rendered from Content.
}

// Error codes returned for invalid pages.
var (
	ErrMissingTitle = errors.New("page missing title") // Pages always need a title.
)

// ListPages returns all the pages.
// The view arg can be used to restrict the output level:
//   (blank) - Full results
//   ids - Results with only the ID attribute.
func ListPages(ctx appengine.Context, view string) ([]Page, error) {
	q := datastore.NewQuery("Page")
	var pages []Page

	if view == "ids" {
		q = q.KeysOnly()
	}
	keys, err := q.GetAll(ctx, &pages)
	if err != nil {
		return nil, err
	}

	if len(keys) > len(pages) {
		// Handle instances where not enough pages were created for each key.
		// This behavior happens when GetAll doesn't create pages like a KeyOnly query.
		diff := len(keys) - len(pages)
		pages = append(pages, make([]Page, diff)...)
	}

	for i, k := range keys {
		pages[i].ID = k.Encode()
	}

	return pages, nil
}

// CreatePage uses JSON data to create and save a new Page.
func CreatePage(ctx appengine.Context, jsonBody io.Reader) (*Page, error) {
	var p Page
	if err := jsonToPage(jsonBody, &p); err != nil {
		return nil, err
	}
	p.LastUpdated = time.Now()

	key, err := datastore.Put(ctx, datastore.NewIncompleteKey(ctx, "Page", nil), &p)
	if err != nil {
		return nil, err
	}
	p.ID = key.Encode()
	return &p, nil
}

// GetPage uses a Page ID from the URL to return a Page.
func GetPage(ctx appengine.Context, ID string) (*Page, error) {
	k, err := datastore.DecodeKey(ID)
	if err != nil {
		return nil, err
	}

	var page Page
	if err := datastore.Get(ctx, k, &page); err != nil {
		return nil, err
	}
	page.ID = k.Encode()
	page.Rendered = string(blackfriday.MarkdownCommon([]byte(page.Content)))
	return &page, nil
}

// UpdatePage uses a Page ID from the URL and JSON data to update a Page.
func UpdatePage(ctx appengine.Context, ID string, jsonBody io.Reader) (*Page, error) {
	key, err := datastore.DecodeKey(ID)
	if err != nil {
		return nil, err
	}

	var page Page
	if err := datastore.Get(ctx, key, &page); err != nil {
		return nil, err
	}
	if err := jsonToPage(jsonBody, &page); err != nil {
		return nil, err
	}
	page.ID = ID
	page.LastUpdated = time.Now()

	if _, err := datastore.Put(ctx, key, &page); err != nil {
		return nil, err
	}
	return &page, nil
}

// DeletePage uses a Page ID to remove a page from the datastore.
func DeletePage(ctx appengine.Context, ID string) error {
	key, err := datastore.DecodeKey(ID)
	if err != nil {
		return err
	}
	return datastore.Delete(ctx, key)
}

func jsonToPage(data io.Reader, page *Page) error {
	decoder := json.NewDecoder(data)
	if err := decoder.Decode(&page); err != nil {
		return err
	}
	if page.Title == "" {
		return ErrMissingTitle
	}
	return nil
}
<commit_msg>Toss pages into memcache to reduce datastore usage.<commit_after>package msgboard

import (
	"appengine"
	"appengine/datastore"
	"appengine/memcache"
	"errors"
	"io"
	"time"

	"encoding/json"
	"github.com/russross/blackfriday"
)

// Page models a slide to be shown on the message board.
type Page struct {
	Title       string    `json:"title"`                        // Title of the Page
	Content     string    `json:"content" datastore:",noindex"` // Markdown text content.
	LastUpdated time.Time `json:"last_updated"`                 // Timestamp the content was last updated.
	ID          string    `json:"id" datastore:"-"`             // Encoded datastore key acting as an ID.
	Rendered    string    `json:"rendered" datastore:"-"`       // HTML for the page based rendered from Content.
}

// Error codes returned for invalid pages.
var (
	ErrMissingTitle = errors.New("page missing title") // Pages always need a title.
)

// ListPages returns all the pages.
// The view arg can be used to restrict the output level:
//   (blank) - Full results
//   ids - Results with only the ID attribute.
func ListPages(ctx appengine.Context, view string) ([]Page, error) {
	q := datastore.NewQuery("Page")
	var pages []Page

	if view == "ids" {
		q = q.KeysOnly()
	}
	keys, err := q.GetAll(ctx, &pages)
	if err != nil {
		return nil, err
	}

	if len(keys) > len(pages) {
		// Handle instances where not enough pages were created for each key.
		// This behavior happens when GetAll doesn't create pages like a KeyOnly query.
		diff := len(keys) - len(pages)
		pages = append(pages, make([]Page, diff)...)
	}

	for i, k := range keys {
		pages[i].ID = k.Encode()
	}

	return pages, nil
}

// CreatePage uses JSON data to create and save a new Page.
func CreatePage(ctx appengine.Context, jsonBody io.Reader) (*Page, error) {
	var p Page
	if err := jsonToPage(jsonBody, &p); err != nil {
		return nil, err
	}
	p.LastUpdated = time.Now()

	key, err := datastore.Put(ctx, datastore.NewIncompleteKey(ctx, "Page", nil), &p)
	if err != nil {
		return nil, err
	}
	p.ID = key.Encode()
	return &p, nil
}

// GetPage uses a Page ID from the URL to return a Page.
func GetPage(ctx appengine.Context, ID string) (*Page, error) {
	page, _ := getPageMemcache(ctx, ID)
	if page == nil {
		ctx.Debugf("memcache miss for id: %s", ID)
		var err error
		page, err = getPageDatastore(ctx, ID)
		if err != nil {
			return nil, err
		}
		defer setPageMemcache(ctx, page)
	}

	page.Rendered = string(blackfriday.MarkdownCommon([]byte(page.Content)))
	return page, nil
}

func getPageDatastore(ctx appengine.Context, ID string) (*Page, error) {
	k, err := datastore.DecodeKey(ID)
	if err != nil {
		return nil, err
	}

	var page Page
	if err := datastore.Get(ctx, k, &page); err != nil {
		return nil, err
	}
	page.ID = k.Encode()
	return &page, nil
}

func getPageMemcache(ctx appengine.Context, ID string) (*Page, error) {
	var page Page
	if _, err := memcache.JSON.Get(ctx, ID, &page); err != nil {
		return nil, err
	}
	return &page, nil
}

func setPageMemcache(ctx appengine.Context, page *Page) {
	// Don't store the rendered HTML in memcache
	page.Rendered = ""

	i := &memcache.Item{
		Key:    page.ID,
		Object: page,
	}

	if err := memcache.JSON.Set(ctx, i); err != nil {
		ctx.Errorf("error setting memcache: %s", err)
	}
}

// UpdatePage uses a Page ID from the URL and JSON data to update a Page.
func UpdatePage(ctx appengine.Context, ID string, jsonBody io.Reader) (*Page, error) {
	key, err := datastore.DecodeKey(ID)
	if err != nil {
		return nil, err
	}

	var page Page
	if err := datastore.Get(ctx, key, &page); err != nil {
		return nil, err
	}
	if err := jsonToPage(jsonBody, &page); err != nil {
		return nil, err
	}
	page.ID = ID
	page.LastUpdated = time.Now()

	if _, err := datastore.Put(ctx, key, &page); err != nil {
		return nil, err
	}
	defer setPageMemcache(ctx, &page)
	return &page, nil
}

// DeletePage uses a Page ID to remove a page from the datastore.
func DeletePage(ctx appengine.Context, ID string) error {
	key, err := datastore.DecodeKey(ID)
	if err != nil {
		return err
	}
	return datastore.Delete(ctx, key)
}

func jsonToPage(data io.Reader, page *Page) error {
	decoder := json.NewDecoder(data)
	if err := decoder.Decode(&page); err != nil {
		return err
	}
	if page.Title == "" {
		return ErrMissingTitle
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2015, Ben Morgan. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.

package twikutil

import (
	"errors"
	"io/ioutil"
	"strconv"
	"strings"

	"github.com/goulash/pre"
	past "github.com/goulash/pre/ast"

	"gopkg.in/twik.v1"
	"gopkg.in/twik.v1/ast"
)

var ErrFuncExists = errors.New("cannot set variable with name of existing function")

type LoaderFunc func(*twik.Scope) FuncMap

type Executer struct {
	PreProcessor *pre.Processor

	fset  *ast.FileSet
	scope *twik.Scope
	funcs map[string]bool
}

func New(loader LoaderFunc) *Executer {
	fset := twik.NewFileSet()
	s := twik.NewScope(fset)
	fns := loader(s)
	keys := make(map[string]bool)
	for k := range fns {
		keys[k] = true
	}
	fns.Export(s)
	return &Executer{
		fset:  fset,
		scope: s,
		funcs: keys,
	}
}

func (e *Executer) Scope() *twik.Scope { return e.scope }

// It is an error to use a key that has already been used as a function.
func (e *Executer) Set(key string, value interface{}) error {
	if e.funcs[key] {
		return errors.New("function with that name already exists")
	}
	_, err := e.scope.Get(key)
	if err == nil {
		return e.scope.Set(key, value)
	}
	return e.scope.Create(key, value)
}

// It is an error to get a key that has already been used as a function.
func (e *Executer) Get(key string) (interface{}, error) {
	if e.funcs[key] {
		return nil, errors.New("functions cannot be gotten")
	}
	return e.scope.Get(key)
}

func (e *Executer) Has(key string) bool {
	if e.funcs[key] {
		return false
	}
	v, err := e.scope.Get(key)
	return err == nil && v != nil
}

func (e *Executer) Create(key string, fn interface{}) error {
	err := e.scope.Create(key, Func(key, fn))
	if err != nil {
		return err
	}
	e.funcs[key] = true
	return nil
}

func (e *Executer) Override(key string, fn interface{}) error {
	if !e.funcs[key] {
		return errors.New("no function by that name exists")
	}
	return e.scope.Set(key, Func(key, fn))
}

func (e *Executer) Exec(file string) (s *twik.Scope, err error) {
	bs, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, err
	}
	return e.ExecString(file, string(bs))
}

func (e *Executer) ExecString(name, code string) (s *twik.Scope, err error) {
	// When the preprocessor is active, we need to convert any error messages
	// we get from twik so that they correspond to the correct file name, line
	// and column. This we do in the deferred function.
	var root past.Node
	if e.PreProcessor != nil {
		root, err = e.PreProcessor.ParseString(name, code)
		code = root.String()
		if err != nil {
			return nil, err
		}

		defer func() {
			err = replaceError(name, root, err)
		}()
	}

	node, err := twik.ParseString(e.fset, name, code)
	if err != nil {
		return nil, err
	}
	_, err = e.scope.Eval(node)
	return e.scope, err
}

func replaceError(name string, root past.Node, err error) error {
	if e, ok := err.(*twik.Error); ok {
		epi := e.PosInfo
		pi := root.OffsetLC(epi.Line, epi.Column)
		epi.Name = pi.Name
		epi.Line = pi.Line
		epi.Column = pi.Column
		return e
	}

	// Some parse errors aren't returned as *twik.Error,
	// so we have to figure it out ourselves. They tend
	// to all have the format:
	//
	//     name:line:col: error msg
	//
	// So we can work with that. If anywhere along the
	// way we run into an error, we just return the
	// original error.
	s := err.Error()
	if !strings.HasPrefix(s, name) {
		return err
	}
	s = strings.TrimPrefix(s, name)
	xs := strings.Split(s, ":")
	if len(xs) < 4 {
		return err
	}
	if xs[0] != "" {
		return err
	}
	line, e := strconv.ParseInt(xs[1], 0, 0)
	if e != nil {
		return err
	}
	col, e := strconv.ParseInt(xs[2], 0, 0)
	if e != nil {
		return err
	}
	pi := root.OffsetLC(int(line), int(col))
	xs[0] = pi.Name
	xs[1] = strconv.FormatInt(int64(pi.Line), 10)
	xs[2] = strconv.FormatInt(int64(pi.Column), 10)
	return errors.New(strings.Join(xs, ":"))
}
<commit_msg>Bugfix: regression from last commit<commit_after>// Copyright (c) 2015, Ben Morgan. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.

package twikutil

import (
	"errors"
	"io/ioutil"
	"strconv"
	"strings"

	"github.com/goulash/pre"
	past "github.com/goulash/pre/ast"

	"gopkg.in/twik.v1"
	"gopkg.in/twik.v1/ast"
)

var ErrFuncExists = errors.New("cannot set variable with name of existing function")

type LoaderFunc func(*twik.Scope) FuncMap

type Executer struct {
	PreProcessor *pre.Processor

	fset  *ast.FileSet
	scope *twik.Scope
	funcs map[string]bool
}

func New(loader LoaderFunc) *Executer {
	fset := twik.NewFileSet()
	s := twik.NewScope(fset)
	fns := loader(s)
	keys := make(map[string]bool)
	for k := range fns {
		keys[k] = true
	}
	fns.Export(s)
	return &Executer{
		fset:  fset,
		scope: s,
		funcs: keys,
	}
}

func (e *Executer) Scope() *twik.Scope { return e.scope }

// It is an error to use a key that has already been used as a function.
func (e *Executer) Set(key string, value interface{}) error {
	if e.funcs[key] {
		return errors.New("function with that name already exists")
	}
	_, err := e.scope.Get(key)
	if err == nil {
		return e.scope.Set(key, value)
	}
	return e.scope.Create(key, value)
}

// It is an error to get a key that has already been used as a function.
func (e *Executer) Get(key string) (interface{}, error) {
	if e.funcs[key] {
		return nil, errors.New("functions cannot be gotten")
	}
	return e.scope.Get(key)
}

func (e *Executer) Has(key string) bool {
	if e.funcs[key] {
		return false
	}
	v, err := e.scope.Get(key)
	return err == nil && v != nil
}

func (e *Executer) Create(key string, fn interface{}) error {
	err := e.scope.Create(key, Func(key, fn))
	if err != nil {
		return err
	}
	e.funcs[key] = true
	return nil
}

func (e *Executer) Override(key string, fn interface{}) error {
	if !e.funcs[key] {
		return errors.New("no function by that name exists")
	}
	return e.scope.Set(key, Func(key, fn))
}

func (e *Executer) Exec(file string) (s *twik.Scope, err error) {
	bs, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, err
	}
	return e.ExecString(file, string(bs))
}

func (e *Executer) ExecString(name, code string) (s *twik.Scope, err error) {
	// When the preprocessor is active, we need to convert any error messages
	// we get from twik so that they correspond to the correct file name, line
	// and column. This we do in the deferred function.
	var root past.Node
	if e.PreProcessor != nil {
		root, err = e.PreProcessor.ParseString(name, code)
		code = root.String()
		if err != nil {
			return nil, err
		}

		defer func() {
			if err != nil {
				err = replaceError(name, root, err)
			}
		}()
	}

	node, err := twik.ParseString(e.fset, name, code)
	if err != nil {
		return nil, err
	}
	_, err = e.scope.Eval(node)
	return e.scope, err
}

func replaceError(name string, root past.Node, err error) error {
	if e, ok := err.(*twik.Error); ok {
		epi := e.PosInfo
		pi := root.OffsetLC(epi.Line, epi.Column)
		epi.Name = pi.Name
		epi.Line = pi.Line
		epi.Column = pi.Column
		return e
	}

	// Some parse errors aren't returned as *twik.Error,
	// so we have to figure it out ourselves. They tend
	// to all have the format:
	//
	//     name:line:col: error msg
	//
	// So we can work with that. If anywhere along the
	// way we run into an error, we just return the
	// original error.
	s := err.Error()
	if !strings.HasPrefix(s, name) {
		return err
	}
	s = strings.TrimPrefix(s, name)
	xs := strings.Split(s, ":")
	if len(xs) < 4 {
		return err
	}
	if xs[0] != "" {
		return err
	}
	line, e := strconv.ParseInt(xs[1], 0, 0)
	if e != nil {
		return err
	}
	col, e := strconv.ParseInt(xs[2], 0, 0)
	if e != nil {
		return err
	}
	pi := root.OffsetLC(int(line), int(col))
	xs[0] = pi.Name
	xs[1] = strconv.FormatInt(int64(pi.Line), 10)
	xs[2] = strconv.FormatInt(int64(pi.Column), 10)
	return errors.New(strings.Join(xs, ":"))
}
<|endoftext|>


<commit_before>package dht

import (
	"testing"

	crand "crypto/rand"

	"code.google.com/p/goprotobuf/proto"

	peer "github.com/jbenet/go-ipfs/peer"
	swarm "github.com/jbenet/go-ipfs/swarm"
	u "github.com/jbenet/go-ipfs/util"
	ma "github.com/jbenet/go-multiaddr"

	"time"
)

// fauxNet is a standin for a swarm.Network in order to more easily recreate
// different testing scenarios
type fauxNet struct {
	Chan     *swarm.Chan
	handlers []mesHandleFunc

	swarm.Network
}

// mesHandleFunc is a function that takes in outgoing messages
// and can respond to them, simulating other peers on the network.
// returning nil will chose not to respond and pass the message onto the
// next registered handler
type mesHandleFunc func(*swarm.Message) *swarm.Message

func newFauxNet() *fauxNet {
	fn := new(fauxNet)
	fn.Chan = swarm.NewChan(8)

	return fn
}

// Instead of 'Listening' Start up a goroutine that will check
// all outgoing messages against registered message handlers,
// and reply if needed
func (f *fauxNet) Listen() error {
	go func() {
		for {
			select {
			case in := <-f.Chan.Outgoing:
				for _, h := range f.handlers {
					reply := h(in)
					if reply != nil {
						f.Chan.Incoming <- reply
						break
					}
				}
			}
		}
	}()
	return nil
}

func (f *fauxNet) AddHandler(fn func(*swarm.Message) *swarm.Message) {
	f.handlers = append(f.handlers, fn)
}

func (f *fauxNet) Send(mes *swarm.Message) {
	f.Chan.Outgoing <- mes
}

func (f *fauxNet) GetChan() *swarm.Chan {
	return f.Chan
}

func (f *fauxNet) Connect(addr *ma.Multiaddr) (*peer.Peer, error) {
	return nil, nil
}

func (f *fauxNet) GetConnection(id peer.ID, addr *ma.Multiaddr) (*peer.Peer, error) {
	return &peer.Peer{ID: id, Addresses: []*ma.Multiaddr{addr}}, nil
}

func TestGetFailures(t *testing.T) {
	fn := newFauxNet()
	fn.Listen()

	local := new(peer.Peer)
	local.ID = peer.ID("test_peer")

	d := NewDHT(local, fn)

	other := &peer.Peer{ID: peer.ID("other_peer")}

	d.Start()

	d.Update(other)

	// This one should time out
	_, err := d.GetValue(u.Key("test"), time.Millisecond*10)
	if err != nil {
		if err != u.ErrTimeout {
			t.Fatal("Got different error than we expected.")
		}
	} else {
		t.Fatal("Did not get expected error!")
	}

	// Reply with failures to every message
	fn.AddHandler(func(mes *swarm.Message) *swarm.Message {
		pmes := new(PBDHTMessage)
		err := proto.Unmarshal(mes.Data, pmes)
		if err != nil {
			t.Fatal(err)
		}

		resp := Message{
			Type:     pmes.GetType(),
			ID:       pmes.GetId(),
			Response: true,
			Success:  false,
		}
		return swarm.NewMessage(mes.Peer, resp.ToProtobuf())
	})

	// This one should fail with NotFound
	_, err = d.GetValue(u.Key("test"), time.Millisecond*1000)
	if err != nil {
		if err != u.ErrNotFound {
			t.Fatalf("Expected ErrNotFound, got: %s", err)
		}
	} else {
		t.Fatal("expected error, got none.")
	}

	success := make(chan struct{})
	fn.handlers = nil
	fn.AddHandler(func(mes *swarm.Message) *swarm.Message {
		resp := new(PBDHTMessage)
		err := proto.Unmarshal(mes.Data, resp)
		if err != nil {
			t.Fatal(err)
		}
		if resp.GetSuccess() {
			t.Fatal("Get returned success when it shouldnt have.")
		}
		success <- struct{}{}
		return nil
	})

	// Now we test this DHT's handleGetValue failure
	req := Message{
		Type:  PBDHTMessage_GET_VALUE,
		Key:   "hello",
		ID:    GenerateMessageID(),
		Value: []byte{0},
	}
	fn.Chan.Incoming <- swarm.NewMessage(other, req.ToProtobuf())

	<-success
}

// TODO: Maybe put these in some sort of "ipfs_testutil" package
func _randPeer() *peer.Peer {
	p := new(peer.Peer)
	p.ID = make(peer.ID, 16)
	p.Addresses = []*ma.Multiaddr{nil}
	crand.Read(p.ID)
	return p
}

func TestNotFound(t *testing.T) {
	u.Debug = true
	fn := newFauxNet()
	fn.Listen()

	local := new(peer.Peer)
	local.ID = peer.ID("test_peer")

	d := NewDHT(local, fn)
	d.Start()

	var ps []*peer.Peer
	for i := 0; i < 5; i++ {
		ps = append(ps, _randPeer())
		d.Update(ps[i])
	}

	// Reply with random peers to every message
	fn.AddHandler(func(mes *swarm.Message) *swarm.Message {
		t.Log("Handling message...")
		pmes := new(PBDHTMessage)
		err := proto.Unmarshal(mes.Data, pmes)
		if err != nil {
			t.Fatal(err)
		}

		switch pmes.GetType() {
		case PBDHTMessage_GET_VALUE:
			resp := Message{
				Type:     pmes.GetType(),
				ID:       pmes.GetId(),
				Response: true,
				Success:  false,
			}

			for i := 0; i < 7; i++ {
				resp.Peers = append(resp.Peers, _randPeer())
			}
			return swarm.NewMessage(mes.Peer, resp.ToProtobuf())
		default:
			panic("Shouldnt recieve this.")
		}

	})

	_, err := d.GetValue(u.Key("hello"), time.Second*30)
	if err != nil {
		switch err {
		case u.ErrNotFound:
			t.Fail()
			//Success!
			return
		case u.ErrTimeout:
			t.Fatal("Should not have gotten timeout!")
		default:
			t.Fatalf("Got unexpected error: %s", err)
		}
	}
	t.Fatal("Expected to recieve an error.")
}
<commit_msg>removed failure call i forgot to remove<commit_after>package dht

import (
	"testing"

	crand "crypto/rand"

	"code.google.com/p/goprotobuf/proto"

	peer "github.com/jbenet/go-ipfs/peer"
	swarm "github.com/jbenet/go-ipfs/swarm"
	u "github.com/jbenet/go-ipfs/util"
	ma "github.com/jbenet/go-multiaddr"

	"time"
)

// fauxNet is a standin for a swarm.Network in order to more easily recreate
// different testing scenarios
type fauxNet struct {
	Chan     *swarm.Chan
	handlers []mesHandleFunc

	swarm.Network
}

// mesHandleFunc is a function that takes in outgoing messages
// and can respond to them, simulating other peers on the network.
// returning nil will chose not to respond and pass the message onto the
// next registered handler
type mesHandleFunc func(*swarm.Message) *swarm.Message

func newFauxNet() *fauxNet {
	fn := new(fauxNet)
	fn.Chan = swarm.NewChan(8)

	return fn
}

// Instead of 'Listening' Start up a goroutine that will check
// all outgoing messages against registered message handlers,
// and reply if needed
func (f *fauxNet) Listen() error {
	go func() {
		for {
			select {
			case in := <-f.Chan.Outgoing:
				for _, h := range f.handlers {
					reply := h(in)
					if reply != nil {
						f.Chan.Incoming <- reply
						break
					}
				}
			}
		}
	}()
	return nil
}

func (f *fauxNet) AddHandler(fn func(*swarm.Message) *swarm.Message) {
	f.handlers = append(f.handlers, fn)
}

func (f *fauxNet) Send(mes *swarm.Message) {
	f.Chan.Outgoing <- mes
}

func (f *fauxNet) GetChan() *swarm.Chan {
	return f.Chan
}

func (f *fauxNet) Connect(addr *ma.Multiaddr) (*peer.Peer, error) {
	return nil, nil
}

func (f *fauxNet) GetConnection(id peer.ID, addr *ma.Multiaddr) (*peer.Peer, error) {
	return &peer.Peer{ID: id, Addresses: []*ma.Multiaddr{addr}}, nil
}

func TestGetFailures(t *testing.T) {
	fn := newFauxNet()
	fn.Listen()

	local := new(peer.Peer)
	local.ID = peer.ID("test_peer")

	d := NewDHT(local, fn)

	other := &peer.Peer{ID: peer.ID("other_peer")}

	d.Start()

	d.Update(other)

	// This one should time out
	_, err := d.GetValue(u.Key("test"), time.Millisecond*10)
	if err != nil {
		if err != u.ErrTimeout {
			t.Fatal("Got different error than we expected.")
		}
	} else {
		t.Fatal("Did not get expected error!")
	}

	// Reply with failures to every message
	fn.AddHandler(func(mes *swarm.Message) *swarm.Message {
		pmes := new(PBDHTMessage)
		err := proto.Unmarshal(mes.Data, pmes)
		if err != nil {
			t.Fatal(err)
		}

		resp := Message{
			Type:     pmes.GetType(),
			ID:       pmes.GetId(),
			Response: true,
			Success:  false,
		}
		return swarm.NewMessage(mes.Peer, resp.ToProtobuf())
	})

	// This one should fail with NotFound
	_, err = d.GetValue(u.Key("test"), time.Millisecond*1000)
	if err != nil {
		if err != u.ErrNotFound {
			t.Fatalf("Expected ErrNotFound, got: %s", err)
		}
	} else {
		t.Fatal("expected error, got none.")
	}

	success := make(chan struct{})
	fn.handlers = nil
	fn.AddHandler(func(mes *swarm.Message) *swarm.Message {
		resp := new(PBDHTMessage)
		err := proto.Unmarshal(mes.Data, resp)
		if err != nil {
			t.Fatal(err)
		}
		if resp.GetSuccess() {
			t.Fatal("Get returned success when it shouldnt have.")
		}
		success <- struct{}{}
		return nil
	})

	// Now we test this DHT's handleGetValue failure
	req := Message{
		Type:  PBDHTMessage_GET_VALUE,
		Key:   "hello",
		ID:    GenerateMessageID(),
		Value: []byte{0},
	}
	fn.Chan.Incoming <- swarm.NewMessage(other, req.ToProtobuf())

	<-success
}

// TODO: Maybe put these in some sort of "ipfs_testutil" package
func _randPeer() *peer.Peer {
	p := new(peer.Peer)
	p.ID = make(peer.ID, 16)
	p.Addresses = []*ma.Multiaddr{nil}
	crand.Read(p.ID)
	return p
}

func TestNotFound(t *testing.T) {
	u.Debug = true
	fn := newFauxNet()
	fn.Listen()

	local := new(peer.Peer)
	local.ID = peer.ID("test_peer")

	d := NewDHT(local, fn)
	d.Start()

	var ps []*peer.Peer
	for i := 0; i < 5; i++ {
		ps = append(ps, _randPeer())
		d.Update(ps[i])
	}

	// Reply with random peers to every message
	fn.AddHandler(func(mes *swarm.Message) *swarm.Message {
		t.Log("Handling message...")
		pmes := new(PBDHTMessage)
		err := proto.Unmarshal(mes.Data, pmes)
		if err != nil {
			t.Fatal(err)
		}

		switch pmes.GetType() {
		case PBDHTMessage_GET_VALUE:
			resp := Message{
				Type:     pmes.GetType(),
				ID:       pmes.GetId(),
				Response: true,
				Success:  false,
			}

			for i := 0; i < 7; i++ {
				resp.Peers = append(resp.Peers, _randPeer())
			}
			return swarm.NewMessage(mes.Peer, resp.ToProtobuf())
		default:
			panic("Shouldnt recieve this.")
		}

	})

	_, err := d.GetValue(u.Key("hello"), time.Second*30)
	if err != nil {
		switch err {
		case u.ErrNotFound:
			//Success!
			return
		case u.ErrTimeout:
			t.Fatal("Should not have gotten timeout!")
		default:
			t.Fatalf("Got unexpected error: %s", err)
		}
	}
	t.Fatal("Expected to recieve an error.")
}
<|endoftext|>


<commit_before>/*
Package fasttext provides a simple wrapper for Facebook
fastText dataset (https://github.com/facebookresearch/fastText/blob/master/pretrained-vectors.md).
It allows fast look-up of word embeddings from persistent data store (Sqlite3).

Installation

	go get -u github.com/ekzhu/go-fasttext

After downloading a .vec data file from the fastText project,
you can initialize the Sqlite3 database (in your code):

	ft := NewFastText("/path/to/sqlite3/file")
	err := ft.BuilDB("/path/to/word/embedding/.vec/file")

This will create a new file on your disk for the Sqlite3 database.
Once the above step is finished, you can start looking up word embeddings
(in your code):

	emb, err := ft.GetEmb("king")
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(emb.Word, emb.Vec)

Each word embedding vector is a slice of float64.

Note that you only need to initialize the Sqlite3 database once.
The next time you use it you can skip the call to BuildDB.
*/
package fasttext

import (
	"bufio"
	"database/sql"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"strconv"
	"strings"

	_ "github.com/mattn/go-sqlite3"
)

const (
	// Table name used in Sqlite
	TableName = "fasttext"
)

var (
	ErrNoEmbFound = errors.New("No embedding found for the given word")
	// TODO: parametrize byte order
	ByteOrder = binary.BigEndian
)

type FastText struct {
	db        *sql.DB
	tablename string
	byteOrder binary.ByteOrder
}

// WordEmb is a pair of word and its embedding vector.
type WordEmb struct {
	Word string
	Vec  []float64
}

// Start a new FastText session given the location
// of the Sqlite3 database file.
func NewFastText(dbFilename string) *FastText {
	db, err := sql.Open("sqlite3", dbFilename)
	if err != nil {
		panic(err)
	}
	return &FastText{
		db:        db,
		tablename: TableName,
		byteOrder: ByteOrder,
	}
}

// Close must be called before finishing using FastText
func (ft *FastText) Close() error {
	return ft.db.Close()
}

// GetEmb returns the word embedding of the given word.
func (ft *FastText) GetEmb(word string) (*WordEmb, error) {
	var binVec []byte
	err := ft.db.QueryRow(fmt.Sprintf(`
	SELECT emb FROM %s WHERE word=?;
	`, ft.tablename), word).Scan(&binVec)
	if err == sql.ErrNoRows {
		return nil, ErrNoEmbFound
	}
	if err != nil {
		panic(err)
	}
	vec, err := bytesToVec(binVec, ft.byteOrder)
	if err != nil {
		return nil, err
	}
	return &WordEmb{
		Word: word,
		Vec:  vec,
	}, nil
}

// BuilDB initialize the Sqlite database by importing the word embeddings
// from the .vec file downloaded from
// https://github.com/facebookresearch/fastText/blob/master/pretrained-vectors.md
func (ft *FastText) BuildDB(wordEmbFile io.Reader) error {
	_, err := ft.db.Exec(fmt.Sprintf(`
	CREATE TABLE %s (
		word TEXT UNIQUE,
		emb BLOB
	);
	`, ft.tablename))
	if err != nil {
		return err
	}
	stmt, err := ft.db.Prepare(fmt.Sprintf(`
	INSERT INTO %s(word, emb) VALUES(?, ?);
	`, ft.tablename))
	if err != nil {
		return err
	}
	defer stmt.Close()
	for emb := range readWordEmbdFile(wordEmbFile) {
		binVec := vecToBytes(emb.Vec, ft.byteOrder)
		if _, err := stmt.Exec(emb.Word, binVec); err != nil {
			return err
		}
	}
	// Indexing on words
	_, err = ft.db.Exec(fmt.Sprintf(`
	CREATE INDEX ind_word ON %s(word);
	`, ft.tablename))
	if err != nil {
		return err
	}
	return nil
}

func readWordEmbdFile(wordEmbFile io.Reader) chan *WordEmb {
	out := make(chan *WordEmb)
	go func() {
		defer close(out)
		scanner := bufio.NewScanner(wordEmbFile)
		var embSize int
		for scanner.Scan() {
			data := strings.TrimSpace(scanner.Text())
			if embSize == 0 {
				var err error
				embSize, err = strconv.Atoi(strings.Split(data, " ")[1])
				if err != nil {
					panic(err)
				}
				continue
			}
			items := strings.Split(data, " ")
			word := items[0]
			if len(items)-1 != embSize {
				panic("Embedding vector size not the same")
			}
			vec := make([]float64, embSize)
			for i := 0; i < embSize; i++ {
				sf, err := strconv.ParseFloat(items[i+1], 64)
				if err != nil {
					panic(err)
				}
				vec[i] = sf
			}
			out <- &WordEmb{
				Word: word,
				Vec:  vec,
			}
		}
		if err := scanner.Err(); err != nil {
			panic(err)
		}
	}()
	return out
}
<commit_msg>handle case when the word is a spcae character<commit_after>/*
Package fasttext provides a simple wrapper for Facebook
fastText dataset (https://github.com/facebookresearch/fastText/blob/master/pretrained-vectors.md).
It allows fast look-up of word embeddings from persistent data store (Sqlite3).

Installation

	go get -u github.com/ekzhu/go-fasttext

After downloading a .vec data file from the fastText project,
you can initialize the Sqlite3 database (in your code):

	ft := NewFastText("/path/to/sqlite3/file")
	err := ft.BuilDB("/path/to/word/embedding/.vec/file")

This will create a new file on your disk for the Sqlite3 database.
Once the above step is finished, you can start looking up word embeddings
(in your code):

	emb, err := ft.GetEmb("king")
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(emb.Word, emb.Vec)

Each word embedding vector is a slice of float64.

Note that you only need to initialize the Sqlite3 database once.
The next time you use it you can skip the call to BuildDB.
*/
package fasttext

import (
	"bufio"
	"database/sql"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"strconv"
	"strings"

	_ "github.com/mattn/go-sqlite3"
)

const (
	// Table name used in Sqlite
	TableName = "fasttext"
)

var (
	ErrNoEmbFound = errors.New("No embedding found for the given word")
	// TODO: parametrize byte order
	ByteOrder = binary.BigEndian
)

type FastText struct {
	db        *sql.DB
	tablename string
	byteOrder binary.ByteOrder
}

// WordEmb is a pair of word and its embedding vector.
type WordEmb struct {
	Word string
	Vec  []float64
}

// Start a new FastText session given the location
// of the Sqlite3 database file.
func NewFastText(dbFilename string) *FastText {
	db, err := sql.Open("sqlite3", dbFilename)
	if err != nil {
		panic(err)
	}
	return &FastText{
		db:        db,
		tablename: TableName,
		byteOrder: ByteOrder,
	}
}

// Close must be called before finishing using FastText
func (ft *FastText) Close() error {
	return ft.db.Close()
}

// GetEmb returns the word embedding of the given word.
func (ft *FastText) GetEmb(word string) (*WordEmb, error) {
	var binVec []byte
	err := ft.db.QueryRow(fmt.Sprintf(`
	SELECT emb FROM %s WHERE word=?;
	`, ft.tablename), word).Scan(&binVec)
	if err == sql.ErrNoRows {
		return nil, ErrNoEmbFound
	}
	if err != nil {
		panic(err)
	}
	vec, err := bytesToVec(binVec, ft.byteOrder)
	if err != nil {
		return nil, err
	}
	return &WordEmb{
		Word: word,
		Vec:  vec,
	}, nil
}

// BuilDB initialize the Sqlite database by importing the word embeddings
// from the .vec file downloaded from
// https://github.com/facebookresearch/fastText/blob/master/pretrained-vectors.md
func (ft *FastText) BuildDB(wordEmbFile io.Reader) error {
	_, err := ft.db.Exec(fmt.Sprintf(`
	CREATE TABLE %s (
		word TEXT UNIQUE,
		emb BLOB
	);
	`, ft.tablename))
	if err != nil {
		return err
	}
	stmt, err := ft.db.Prepare(fmt.Sprintf(`
	INSERT INTO %s(word, emb) VALUES(?, ?);
	`, ft.tablename))
	if err != nil {
		return err
	}
	defer stmt.Close()
	for emb := range readWordEmbdFile(wordEmbFile) {
		binVec := vecToBytes(emb.Vec, ft.byteOrder)
		if _, err := stmt.Exec(emb.Word, binVec); err != nil {
			return err
		}
	}
	// Indexing on words
	_, err = ft.db.Exec(fmt.Sprintf(`
	CREATE INDEX ind_word ON %s(word);
	`, ft.tablename))
	if err != nil {
		return err
	}
	return nil
}

func readWordEmbdFile(wordEmbFile io.Reader) chan *WordEmb {
	out := make(chan *WordEmb)
	go func() {
		defer close(out)
		scanner := bufio.NewScanner(wordEmbFile)
		var embSize int
		var line int
		for scanner.Scan() {
			line++
			data := scanner.Text()
			if embSize == 0 {
				var err error
				embSize, err = strconv.Atoi(strings.Split(data, " ")[1])
				if err != nil {
					panic(err)
				}
				continue
			}
			// Get the word
			items := strings.SplitN(data, " ", 2)
			word := items[0]
			if word == "" {
				word = " "
			}
			// Get the vec
			vecStrs := strings.Split(strings.TrimSpace(items[1]), " ")
			if len(vecStrs) != embSize {
				msg := fmt.Sprintf("Embedding vec size not same: expected %d, got %d. Loc: line %d, word %s",
					embSize, len(vecStrs), line, word)
				panic(msg)
			}
			vec := make([]float64, embSize)
			for i := 0; i < embSize; i++ {
				sf, err := strconv.ParseFloat(vecStrs[i], 64)
				if err != nil {
					panic(err)
				}
				vec[i] = sf
			}
			out <- &WordEmb{
				Word: word,
				Vec:  vec,
			}
		}
		if err := scanner.Err(); err != nil {
			panic(err)
		}
	}()
	return out
}
<|endoftext|>