<commit_before>// +build linux

package nsinit

import (
	"fmt"
	"github.com/dotcloud/docker/pkg/system"
	"io/ioutil"
	"os"
	"path/filepath"
	"syscall"
)

// default mount point flags
const defaultMountFlags = syscall.MS_NOEXEC | syscall.MS_NOSUID | syscall.MS_NODEV

// setupNewMountNamespace is used to initialize a new mount namespace for an new
// container in the rootfs that is specified.
//
// There is no need to unmount the new mounts because as soon as the mount namespace
// is no longer in use, the mounts will be removed automatically
func setupNewMountNamespace(rootfs, console string, readonly bool) error {
	// mount as slave so that the new mounts do not propagate to the host
	if err := system.Mount("", "/", "", syscall.MS_SLAVE|syscall.MS_REC, ""); err != nil {
		return fmt.Errorf("mounting / as slave %s", err)
	}
	if err := system.Mount(rootfs, rootfs, "bind", syscall.MS_BIND|syscall.MS_REC, ""); err != nil {
		return fmt.Errorf("mouting %s as bind %s", rootfs, err)
	}
	if readonly {
		if err := system.Mount(rootfs, rootfs, "bind", syscall.MS_BIND|syscall.MS_REMOUNT|syscall.MS_RDONLY|syscall.MS_REC, ""); err != nil {
			return fmt.Errorf("mounting %s as readonly %s", rootfs, err)
		}
	}
	if err := mountSystem(rootfs); err != nil {
		return fmt.Errorf("mount system %s", err)
	}
	if err := copyDevNodes(rootfs); err != nil {
		return fmt.Errorf("copy dev nodes %s", err)
	}
	// In non-privileged mode, this fails. Discard the error.
	setupLoopbackDevices(rootfs)
	if err := setupDev(rootfs); err != nil {
		return err
	}
	if console != "" {
		if err := setupPtmx(rootfs, console); err != nil {
			return err
		}
	}
	if err := system.Chdir(rootfs); err != nil {
		return fmt.Errorf("chdir into %s %s", rootfs, err)
	}

	pivotDir, err := ioutil.TempDir(rootfs, ".pivot_root")
	if err != nil {
		return fmt.Errorf("can't create pivot_root dir %s", pivotDir, err)
	}
	if err := system.Pivotroot(rootfs, pivotDir); err != nil {
		return fmt.Errorf("pivot_root %s", err)
	}
	if err := system.Chdir("/"); err != nil {
		return fmt.Errorf("chdir / %s", err)
	}

	// path to pivot dir now changed, update
	pivotDir = filepath.Join("/", filepath.Base(pivotDir))

	if err := system.Unmount(pivotDir, syscall.MNT_DETACH); err != nil {
		return fmt.Errorf("unmount pivot_root dir %s", err)
	}

	if err := os.Remove(pivotDir); err != nil {
		return fmt.Errorf("remove pivot_root dir %s", err)
	}

	system.Umask(0022)

	return nil
}

// copyDevNodes mknods the hosts devices so the new container has access to them
func copyDevNodes(rootfs string) error {
	oldMask := system.Umask(0000)
	defer system.Umask(oldMask)

	for _, node := range []string{
		"null",
		"zero",
		"full",
		"random",
		"urandom",
		"tty",
	} {
		if err := copyDevNode(rootfs, node); err != nil {
			return err
		}
	}
	return nil
}

func setupLoopbackDevices(rootfs string) error {
	for i := 0; ; i++ {
		if err := copyDevNode(rootfs, fmt.Sprintf("loop%d", i)); err != nil {
			if !os.IsNotExist(err) {
				return err
			}
			break
		}

	}
	return nil
}

func copyDevNode(rootfs, node string) error {
	stat, err := os.Stat(filepath.Join("/dev", node))
	if err != nil {
		return err
	}
	var (
		dest = filepath.Join(rootfs, "dev", node)
		st   = stat.Sys().(*syscall.Stat_t)
	)
	if err := system.Mknod(dest, st.Mode, int(st.Rdev)); err != nil && !os.IsExist(err) {
		return fmt.Errorf("copy %s %s", node, err)
	}
	return nil
}

// setupDev symlinks the current processes pipes into the
// appropriate destination on the containers rootfs
func setupDev(rootfs string) error {
	for _, link := range []struct {
		from string
		to   string
	}{
		{"/proc/kcore", "/dev/core"},
		{"/proc/self/fd", "/dev/fd"},
		{"/proc/self/fd/0", "/dev/stdin"},
		{"/proc/self/fd/1", "/dev/stdout"},
		{"/proc/self/fd/2", "/dev/stderr"},
	} {
		dest := filepath.Join(rootfs, link.to)
		if err := os.Remove(dest); err != nil && !os.IsNotExist(err) {
			return fmt.Errorf("remove %s %s", dest, err)
		}
		if err := os.Symlink(link.from, dest); err != nil {
			return fmt.Errorf("symlink %s %s", dest, err)
		}
	}
	return nil
}

// setupConsole ensures that the container has a proper /dev/console setup
func setupConsole(rootfs, console string) error {
	oldMask := system.Umask(0000)
	defer system.Umask(oldMask)

	stat, err := os.Stat(console)
	if err != nil {
		return fmt.Errorf("stat console %s %s", console, err)
	}
	var (
		st   = stat.Sys().(*syscall.Stat_t)
		dest = filepath.Join(rootfs, "dev/console")
	)
	if err := os.Remove(dest); err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("remove %s %s", dest, err)
	}
	if err := os.Chmod(console, 0600); err != nil {
		return err
	}
	if err := os.Chown(console, 0, 0); err != nil {
		return err
	}
	if err := system.Mknod(dest, (st.Mode&^07777)|0600, int(st.Rdev)); err != nil {
		return fmt.Errorf("mknod %s %s", dest, err)
	}
	if err := system.Mount(console, dest, "bind", syscall.MS_BIND, ""); err != nil {
		return fmt.Errorf("bind %s to %s %s", console, dest, err)
	}
	return nil
}

// mountSystem sets up linux specific system mounts like sys, proc, shm, and devpts
// inside the mount namespace
func mountSystem(rootfs string) error {
	for _, m := range []struct {
		source string
		path   string
		device string
		flags  int
		data   string
	}{
		{source: "proc", path: filepath.Join(rootfs, "proc"), device: "proc", flags: defaultMountFlags},
		{source: "sysfs", path: filepath.Join(rootfs, "sys"), device: "sysfs", flags: defaultMountFlags},
		{source: "tmpfs", path: filepath.Join(rootfs, "dev"), device: "tmpfs", flags: syscall.MS_NOSUID | syscall.MS_STRICTATIME, data: "mode=755"},
		{source: "shm", path: filepath.Join(rootfs, "dev", "shm"), device: "tmpfs", flags: defaultMountFlags, data: "mode=1777"},
		{source: "devpts", path: filepath.Join(rootfs, "dev", "pts"), device: "devpts", flags: syscall.MS_NOSUID | syscall.MS_NOEXEC, data: "newinstance,ptmxmode=0666,mode=620,gid=5"},
	} {
		if err := os.MkdirAll(m.path, 0755); err != nil && !os.IsExist(err) {
			return fmt.Errorf("mkdirall %s %s", m.path, err)
		}
		if err := system.Mount(m.source, m.path, m.device, uintptr(m.flags), m.data); err != nil {
			return fmt.Errorf("mounting %s into %s %s", m.source, m.path, err)
		}
	}
	return nil
}

// setupPtmx adds a symlink to pts/ptmx for /dev/ptmx and
// finishes setting up /dev/console
func setupPtmx(rootfs, console string) error {
	ptmx := filepath.Join(rootfs, "dev/ptmx")
	if err := os.Remove(ptmx); err != nil && !os.IsNotExist(err) {
		return err
	}
	if err := os.Symlink("pts/ptmx", ptmx); err != nil {
		return fmt.Errorf("symlink dev ptmx %s", err)
	}
	if err := setupConsole(rootfs, console); err != nil {
		return err
	}
	return nil
}

// remountProc is used to detach and remount the proc filesystem
// commonly needed with running a new process inside an existing container
func remountProc() error {
	if err := system.Unmount("/proc", syscall.MNT_DETACH); err != nil {
		return err
	}
	if err := system.Mount("proc", "/proc", "proc", uintptr(defaultMountFlags), ""); err != nil {
		return err
	}
	return nil
}

func remountSys() error {
	if err := system.Unmount("/sys", syscall.MNT_DETACH); err != nil {
		if err != syscall.EINVAL {
			return err
		}
	} else {
		if err := system.Mount("sysfs", "/sys", "sysfs", uintptr(defaultMountFlags), ""); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>Remove /dev tmpfs mountpoint<commit_after>// +build linux

package nsinit

import (
	"fmt"
	"github.com/dotcloud/docker/pkg/system"
	"io/ioutil"
	"os"
	"path/filepath"
	"syscall"
)

// default mount point flags
const defaultMountFlags = syscall.MS_NOEXEC | syscall.MS_NOSUID | syscall.MS_NODEV

// setupNewMountNamespace is used to initialize a new mount namespace for an new
// container in the rootfs that is specified.
//
// There is no need to unmount the new mounts because as soon as the mount namespace
// is no longer in use, the mounts will be removed automatically
func setupNewMountNamespace(rootfs, console string, readonly bool) error {
	// mount as slave so that the new mounts do not propagate to the host
	if err := system.Mount("", "/", "", syscall.MS_SLAVE|syscall.MS_REC, ""); err != nil {
		return fmt.Errorf("mounting / as slave %s", err)
	}
	if err := system.Mount(rootfs, rootfs, "bind", syscall.MS_BIND|syscall.MS_REC, ""); err != nil {
		return fmt.Errorf("mouting %s as bind %s", rootfs, err)
	}
	if readonly {
		if err := system.Mount(rootfs, rootfs, "bind", syscall.MS_BIND|syscall.MS_REMOUNT|syscall.MS_RDONLY|syscall.MS_REC, ""); err != nil {
			return fmt.Errorf("mounting %s as readonly %s", rootfs, err)
		}
	}
	if err := mountSystem(rootfs); err != nil {
		return fmt.Errorf("mount system %s", err)
	}
	if err := copyDevNodes(rootfs); err != nil {
		return fmt.Errorf("copy dev nodes %s", err)
	}
	// In non-privileged mode, this fails. Discard the error.
	setupLoopbackDevices(rootfs)
	if err := setupDev(rootfs); err != nil {
		return err
	}
	if console != "" {
		if err := setupPtmx(rootfs, console); err != nil {
			return err
		}
	}
	if err := system.Chdir(rootfs); err != nil {
		return fmt.Errorf("chdir into %s %s", rootfs, err)
	}

	pivotDir, err := ioutil.TempDir(rootfs, ".pivot_root")
	if err != nil {
		return fmt.Errorf("can't create pivot_root dir %s", pivotDir, err)
	}
	if err := system.Pivotroot(rootfs, pivotDir); err != nil {
		return fmt.Errorf("pivot_root %s", err)
	}
	if err := system.Chdir("/"); err != nil {
		return fmt.Errorf("chdir / %s", err)
	}

	// path to pivot dir now changed, update
	pivotDir = filepath.Join("/", filepath.Base(pivotDir))

	if err := system.Unmount(pivotDir, syscall.MNT_DETACH); err != nil {
		return fmt.Errorf("unmount pivot_root dir %s", err)
	}

	if err := os.Remove(pivotDir); err != nil {
		return fmt.Errorf("remove pivot_root dir %s", err)
	}

	system.Umask(0022)

	return nil
}

// copyDevNodes mknods the hosts devices so the new container has access to them
func copyDevNodes(rootfs string) error {
	oldMask := system.Umask(0000)
	defer system.Umask(oldMask)

	for _, node := range []string{
		"null",
		"zero",
		"full",
		"random",
		"urandom",
		"tty",
	} {
		if err := copyDevNode(rootfs, node); err != nil {
			return err
		}
	}
	return nil
}

func setupLoopbackDevices(rootfs string) error {
	for i := 0; ; i++ {
		if err := copyDevNode(rootfs, fmt.Sprintf("loop%d", i)); err != nil {
			if !os.IsNotExist(err) {
				return err
			}
			break
		}

	}
	return nil
}

func copyDevNode(rootfs, node string) error {
	stat, err := os.Stat(filepath.Join("/dev", node))
	if err != nil {
		return err
	}
	var (
		dest = filepath.Join(rootfs, "dev", node)
		st   = stat.Sys().(*syscall.Stat_t)
	)
	if err := system.Mknod(dest, st.Mode, int(st.Rdev)); err != nil && !os.IsExist(err) {
		return fmt.Errorf("copy %s %s", node, err)
	}
	return nil
}

// setupDev symlinks the current processes pipes into the
// appropriate destination on the containers rootfs
func setupDev(rootfs string) error {
	for _, link := range []struct {
		from string
		to   string
	}{
		{"/proc/kcore", "/dev/core"},
		{"/proc/self/fd", "/dev/fd"},
		{"/proc/self/fd/0", "/dev/stdin"},
		{"/proc/self/fd/1", "/dev/stdout"},
		{"/proc/self/fd/2", "/dev/stderr"},
	} {
		dest := filepath.Join(rootfs, link.to)
		if err := os.Remove(dest); err != nil && !os.IsNotExist(err) {
			return fmt.Errorf("remove %s %s", dest, err)
		}
		if err := os.Symlink(link.from, dest); err != nil {
			return fmt.Errorf("symlink %s %s", dest, err)
		}
	}
	return nil
}

// setupConsole ensures that the container has a proper /dev/console setup
func setupConsole(rootfs, console string) error {
	oldMask := system.Umask(0000)
	defer system.Umask(oldMask)

	stat, err := os.Stat(console)
	if err != nil {
		return fmt.Errorf("stat console %s %s", console, err)
	}
	var (
		st   = stat.Sys().(*syscall.Stat_t)
		dest = filepath.Join(rootfs, "dev/console")
	)
	if err := os.Remove(dest); err != nil && !os.IsNotExist(err) {
		return fmt.Errorf("remove %s %s", dest, err)
	}
	if err := os.Chmod(console, 0600); err != nil {
		return err
	}
	if err := os.Chown(console, 0, 0); err != nil {
		return err
	}
	if err := system.Mknod(dest, (st.Mode&^07777)|0600, int(st.Rdev)); err != nil {
		return fmt.Errorf("mknod %s %s", dest, err)
	}
	if err := system.Mount(console, dest, "bind", syscall.MS_BIND, ""); err != nil {
		return fmt.Errorf("bind %s to %s %s", console, dest, err)
	}
	return nil
}

// mountSystem sets up linux specific system mounts like sys, proc, shm, and devpts
// inside the mount namespace
func mountSystem(rootfs string) error {
	for _, m := range []struct {
		source string
		path   string
		device string
		flags  int
		data   string
	}{
		{source: "proc", path: filepath.Join(rootfs, "proc"), device: "proc", flags: defaultMountFlags},
		{source: "sysfs", path: filepath.Join(rootfs, "sys"), device: "sysfs", flags: defaultMountFlags},
		{source: "shm", path: filepath.Join(rootfs, "dev", "shm"), device: "tmpfs", flags: defaultMountFlags, data: "mode=1777"},
		{source: "devpts", path: filepath.Join(rootfs, "dev", "pts"), device: "devpts", flags: syscall.MS_NOSUID | syscall.MS_NOEXEC, data: "newinstance,ptmxmode=0666,mode=620,gid=5"},
	} {
		if err := os.MkdirAll(m.path, 0755); err != nil && !os.IsExist(err) {
			return fmt.Errorf("mkdirall %s %s", m.path, err)
		}
		if err := system.Mount(m.source, m.path, m.device, uintptr(m.flags), m.data); err != nil {
			return fmt.Errorf("mounting %s into %s %s", m.source, m.path, err)
		}
	}
	return nil
}

// setupPtmx adds a symlink to pts/ptmx for /dev/ptmx and
// finishes setting up /dev/console
func setupPtmx(rootfs, console string) error {
	ptmx := filepath.Join(rootfs, "dev/ptmx")
	if err := os.Remove(ptmx); err != nil && !os.IsNotExist(err) {
		return err
	}
	if err := os.Symlink("pts/ptmx", ptmx); err != nil {
		return fmt.Errorf("symlink dev ptmx %s", err)
	}
	if err := setupConsole(rootfs, console); err != nil {
		return err
	}
	return nil
}

// remountProc is used to detach and remount the proc filesystem
// commonly needed with running a new process inside an existing container
func remountProc() error {
	if err := system.Unmount("/proc", syscall.MNT_DETACH); err != nil {
		return err
	}
	if err := system.Mount("proc", "/proc", "proc", uintptr(defaultMountFlags), ""); err != nil {
		return err
	}
	return nil
}

func remountSys() error {
	if err := system.Unmount("/sys", syscall.MNT_DETACH); err != nil {
		if err != syscall.EINVAL {
			return err
		}
	} else {
		if err := system.Mount("sysfs", "/sys", "sysfs", uintptr(defaultMountFlags), ""); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2016 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Irfan Sharif (irfansharif@cockroachlabs.com)

package distsqlrun

import (
	"golang.org/x/net/context"

	"github.com/cockroachdb/cockroach/pkg/sql/parser"
	"github.com/cockroachdb/cockroach/pkg/sql/sqlbase"
	"github.com/cockroachdb/cockroach/pkg/util/log"
)

type joinerBase struct {
	inputs []RowSource
	output RowReceiver
	ctx    context.Context

	joinType    joinType
	outputCols  columns
	filter      exprHelper
	rowAlloc    sqlbase.EncDatumRowAlloc
	emptyLeft   sqlbase.EncDatumRow
	emptyRight  sqlbase.EncDatumRow
	combinedRow sqlbase.EncDatumRow
}

func (jb *joinerBase) init(
	flowCtx *FlowCtx,
	inputs []RowSource,
	output RowReceiver,
	outputCols []uint32,
	jType JoinType,
	leftTypes []sqlbase.ColumnType,
	rightTypes []sqlbase.ColumnType,
	expr Expression,
) error {
	jb.inputs = inputs
	jb.output = output
	jb.ctx = log.WithLogTag(flowCtx.Context, "Joiner", nil)
	jb.outputCols = columns(outputCols)
	jb.joinType = joinType(jType)
	jb.emptyLeft = make(sqlbase.EncDatumRow, len(leftTypes))
	for i := range jb.emptyLeft {
		jb.emptyLeft[i].Datum = parser.DNull
	}

	jb.emptyRight = make(sqlbase.EncDatumRow, len(rightTypes))
	for i := range jb.emptyRight {
		jb.emptyRight[i].Datum = parser.DNull
	}

	types := make([]sqlbase.ColumnType, 0, len(leftTypes)+len(rightTypes))
	types = append(types, leftTypes...)
	types = append(types, rightTypes...)

	return jb.filter.init(expr, types, flowCtx.evalCtx)
}

// render evaluates the provided filter and constructs a row with columns from
// both rows as specified by the provided output columns. We expect left or
// right to be nil if there was no explicit "join" match, the filter is then
// evaluated on a combinedRow with null values for the columns of the nil row.
// render returns a nil row if no row is to be emitted (eg. if join type is
// inner join and one of the given rows is nil).
func (jb *joinerBase) render(lrow, rrow sqlbase.EncDatumRow) (sqlbase.EncDatumRow, error) {
	switch jb.joinType {
	case innerJoin:
		if lrow == nil || rrow == nil {
			return nil, nil
		}
	case fullOuter:
		if lrow == nil {
			lrow = jb.emptyLeft
		} else if rrow == nil {
			rrow = jb.emptyRight
		}
	case leftOuter:
		if rrow == nil {
			rrow = jb.emptyRight
		}
	case rightOuter:
		if lrow == nil {
			lrow = jb.emptyLeft
		}
	}
	jb.combinedRow = append(jb.combinedRow[:0], lrow...)
	jb.combinedRow = append(jb.combinedRow, rrow...)
	res, err := jb.filter.evalFilter(jb.combinedRow)
	if !res || err != nil {
		return nil, err
	}

	row := jb.rowAlloc.AllocRow(len(jb.outputCols))
	for i, col := range jb.outputCols {
		row[i] = jb.combinedRow[col]
	}
	return row, nil
}
<commit_msg>sql: bugfix: early return in joinerbase.render<commit_after>// Copyright 2016 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Irfan Sharif (irfansharif@cockroachlabs.com)

package distsqlrun

import (
	"golang.org/x/net/context"

	"github.com/cockroachdb/cockroach/pkg/sql/parser"
	"github.com/cockroachdb/cockroach/pkg/sql/sqlbase"
	"github.com/cockroachdb/cockroach/pkg/util/log"
)

type joinerBase struct {
	inputs []RowSource
	output RowReceiver
	ctx    context.Context

	joinType    joinType
	outputCols  columns
	filter      exprHelper
	rowAlloc    sqlbase.EncDatumRowAlloc
	emptyLeft   sqlbase.EncDatumRow
	emptyRight  sqlbase.EncDatumRow
	combinedRow sqlbase.EncDatumRow
}

func (jb *joinerBase) init(
	flowCtx *FlowCtx,
	inputs []RowSource,
	output RowReceiver,
	outputCols []uint32,
	jType JoinType,
	leftTypes []sqlbase.ColumnType,
	rightTypes []sqlbase.ColumnType,
	expr Expression,
) error {
	jb.inputs = inputs
	jb.output = output
	jb.ctx = log.WithLogTag(flowCtx.Context, "Joiner", nil)
	jb.outputCols = columns(outputCols)
	jb.joinType = joinType(jType)
	jb.emptyLeft = make(sqlbase.EncDatumRow, len(leftTypes))
	for i := range jb.emptyLeft {
		jb.emptyLeft[i].Datum = parser.DNull
	}

	jb.emptyRight = make(sqlbase.EncDatumRow, len(rightTypes))
	for i := range jb.emptyRight {
		jb.emptyRight[i].Datum = parser.DNull
	}

	types := make([]sqlbase.ColumnType, 0, len(leftTypes)+len(rightTypes))
	types = append(types, leftTypes...)
	types = append(types, rightTypes...)

	return jb.filter.init(expr, types, flowCtx.evalCtx)
}

// render evaluates the provided filter and constructs a row with columns from
// both rows as specified by the provided output columns. We expect left or
// right to be nil if there was no explicit "join" match, the filter is then
// evaluated on a combinedRow with null values for the columns of the nil row.
// render returns a nil row if no row is to be emitted (eg. if join type is
// inner join and one of the given rows is nil).
func (jb *joinerBase) render(lrow, rrow sqlbase.EncDatumRow) (sqlbase.EncDatumRow, error) {
	lnil := lrow == nil
	rnil := rrow == nil
	switch jb.joinType {
	case innerJoin:
		if lnil || rnil {
			return nil, nil
		}
	case fullOuter:
		if lnil {
			lrow = jb.emptyLeft
		} else if rnil {
			rrow = jb.emptyRight
		}
	case leftOuter:
		if lnil {
			return nil, nil
		}
		if rnil {
			rrow = jb.emptyRight
		}
	case rightOuter:
		if rnil {
			return nil, nil
		}
		if lnil {
			lrow = jb.emptyLeft
		}
	}
	jb.combinedRow = append(jb.combinedRow[:0], lrow...)
	jb.combinedRow = append(jb.combinedRow, rrow...)
	res, err := jb.filter.evalFilter(jb.combinedRow)
	if !res || err != nil {
		return nil, err
	}

	row := jb.rowAlloc.AllocRow(len(jb.outputCols))
	for i, col := range jb.outputCols {
		row[i] = jb.combinedRow[col]
	}
	return row, nil
}
<|endoftext|>


<commit_before>// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// An example implementation of a client.

package main

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"time"

	"github.com/spf13/cobra"

	"istio.io/istio/pkg/cmd"
	"istio.io/istio/pkg/test/echo/common"
	"istio.io/istio/pkg/test/echo/proto"
	"istio.io/istio/pkg/test/echo/server/forwarder"
	"istio.io/pkg/log"
)

var (
	count       int
	timeout     time.Duration
	qps         int
	url         string
	uds         string
	headerKey   string
	headerVal   string
	headers     string
	msg         string
	method      string
	http2       bool
	serverFirst bool
	clientCert  string
	clientKey   string

	caFile string

	loggingOptions = log.DefaultOptions()

	rootCmd = &cobra.Command{
		Use:          "client",
		Short:        "Istio test Echo client.",
		SilenceUsage: true,
		Long: `Istio test Echo client used for generating traffic between instances of the Echo service.
For Kubernetes, this can be run from a source pod via "kubectl exec" with the desired flags.
In general, Echo's gRPC interface (ForwardEcho) should be preferred. This client is only needed in cases
where the network configuration doesn't support gRPC to the source pod.'
`,
		PersistentPreRunE: configureLogging,
		Run: func(cmd *cobra.Command, args []string) {
			// Create a request from the flags.
			request, err := getRequest()
			if err != nil {
				log.Fatala(err)
				os.Exit(-1)
			}

			// Create a forwarder.
			f, err := forwarder.New(forwarder.Config{
				Request: request,
				UDS:     uds,
				TLSCert: caFile,
			})
			if err != nil {
				log.Fatalf("Failed to create forwarder: %v", err)
				os.Exit(-1)
			}

			// Run the forwarder.
			defer func() {
				_ = f.Close()
			}()
			response, err := f.Run(context.Background())
			if err != nil {
				log.Fatalf("Error %s\n", err)
				os.Exit(-1)
			}

			// Log the output to stdout.
			for _, line := range response.Output {
				fmt.Println(line)
			}

			log.Infof("All requests succeeded")
		},
	}
)

func configureLogging(_ *cobra.Command, _ []string) error {
	if err := log.Configure(loggingOptions); err != nil {
		return err
	}
	return nil
}

func init() {
	rootCmd.PersistentFlags().IntVar(&count, "count", common.DefaultCount, "Number of times to make the request")
	rootCmd.PersistentFlags().IntVar(&qps, "qps", 0, "Queries per second")
	rootCmd.PersistentFlags().DurationVar(&timeout, "timeout", common.DefaultRequestTimeout, "Request timeout")
	rootCmd.PersistentFlags().StringVar(&url, "url", "", "Specify URL")
	rootCmd.PersistentFlags().StringVar(&uds, "uds", "",
		"Specify the Unix Domain Socket to connect to")
	rootCmd.PersistentFlags().StringVar(&headerKey, "key", "",
		"Header key (use Host for authority) - deprecated user headers instead")
	rootCmd.PersistentFlags().StringVar(&headerVal, "val", "", "Header value - deprecated")
	rootCmd.PersistentFlags().StringVar(&headers, "headers", "",
		"A list of http headers (use Host for authority) - name:value[,name:value]*")
	rootCmd.PersistentFlags().StringVar(&caFile, "ca", "/cert.crt", "CA root cert file")
	rootCmd.PersistentFlags().StringVar(&msg, "msg", "HelloWorld",
		"message to send (for websockets)")
	rootCmd.PersistentFlags().StringVar(&method, "method", "", "method to use (for HTTP)")
	rootCmd.PersistentFlags().BoolVar(&http2, "http2", false,
		"send http requests as HTTP with prior knowledge")
	rootCmd.PersistentFlags().BoolVar(&serverFirst, "server-first", false,
		"Treat as a server first protocol; do not send request until magic string is received")
	rootCmd.PersistentFlags().StringVar(&clientCert, "client-cert", "", "client certificate file to use for request")
	rootCmd.PersistentFlags().StringVar(&clientKey, "client-key", "", "client certificate key file to use for request")

	loggingOptions.AttachCobraFlags(rootCmd)

	cmd.AddFlags(rootCmd)
}

func getRequest() (*proto.ForwardEchoRequest, error) {
	request := &proto.ForwardEchoRequest{
		Url:           url,
		TimeoutMicros: common.DurationToMicros(timeout),
		Count:         int32(count),
		Qps:           int32(qps),
		Message:       msg,
		Http2:         http2,
		ServerFirst:   serverFirst,
		Method:        method,
	}

	// Old http add header - deprecated
	if headerKey != "" {
		request.Headers = append(request.Headers, &proto.Header{
			Key:   headerKey,
			Value: headerVal,
		})
	}

	if headers != "" {
		headersList := strings.Split(headers, ",")
		for _, header := range headersList {
			parts := strings.Split(header, ":")

			// require name:value format
			if len(parts) != 2 {
				return nil, fmt.Errorf("invalid header format: %q (want name:value)", header)
			}

			request.Headers = append(request.Headers, &proto.Header{
				Key:   parts[0],
				Value: parts[1],
			})
		}
	}

	if clientCert != "" && clientKey != "" {
		certData, err := ioutil.ReadFile(clientCert)
		if err != nil {
			return nil, fmt.Errorf("failed to load client certificate: %v", err)
		}
		request.Cert = string(certData)
		keyData, err := ioutil.ReadFile(clientKey)
		if err != nil {
			return nil, fmt.Errorf("failed to load client certificate key: %v", err)
		}
		request.Key = string(keyData)
	}
	return request, nil
}

func main() {
	if err := rootCmd.Execute(); err != nil {
		os.Exit(-1)
	}
}
<commit_msg>Make test client more like curl (#27401)<commit_after>// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// An example implementation of a client.

package main

import (
	"context"
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"strings"
	"time"

	"github.com/spf13/cobra"

	"istio.io/istio/pkg/cmd"
	"istio.io/istio/pkg/test/echo/common"
	"istio.io/istio/pkg/test/echo/proto"
	"istio.io/istio/pkg/test/echo/server/forwarder"
	"istio.io/pkg/log"
)

var (
	count       int
	timeout     time.Duration
	qps         int
	uds         string
	headers     []string
	msg         string
	method      string
	http2       bool
	serverFirst bool
	clientCert  string
	clientKey   string

	caFile string

	loggingOptions = log.DefaultOptions()

	rootCmd = &cobra.Command{
		Use:          "client",
		Short:        "Istio test Echo client.",
		SilenceUsage: true,
		Long: `Istio test Echo client used for generating traffic between instances of the Echo service.
For Kubernetes, this can be run from a source pod via "kubectl exec" with the desired flags.
In general, Echo's gRPC interface (ForwardEcho) should be preferred. This client is only needed in cases
where the network configuration doesn't support gRPC to the source pod.'
`,
		Args:              cobra.ExactArgs(1),
		PersistentPreRunE: configureLogging,
		Run: func(cmd *cobra.Command, args []string) {
			// Create a request from the flags.
			request, err := getRequest(args[0])
			if err != nil {
				log.Fatala(err)
				os.Exit(-1)
			}

			// Create a forwarder.
			f, err := forwarder.New(forwarder.Config{
				Request: request,
				UDS:     uds,
				TLSCert: caFile,
			})
			if err != nil {
				log.Fatalf("Failed to create forwarder: %v", err)
				os.Exit(-1)
			}

			// Run the forwarder.
			defer func() {
				_ = f.Close()
			}()
			response, err := f.Run(context.Background())
			if err != nil {
				log.Fatalf("Error %s\n", err)
				os.Exit(-1)
			}

			// Log the output to stdout.
			for _, line := range response.Output {
				fmt.Println(line)
			}

			log.Infof("All requests succeeded")
		},
	}
)

func configureLogging(_ *cobra.Command, _ []string) error {
	if err := log.Configure(loggingOptions); err != nil {
		return err
	}
	return nil
}

func init() {
	rootCmd.PersistentFlags().IntVar(&count, "count", common.DefaultCount, "Number of times to make the request")
	rootCmd.PersistentFlags().IntVar(&qps, "qps", 0, "Queries per second")
	rootCmd.PersistentFlags().DurationVar(&timeout, "timeout", common.DefaultRequestTimeout, "Request timeout")
	rootCmd.PersistentFlags().StringVar(&uds, "uds", "",
		"Specify the Unix Domain Socket to connect to")
	rootCmd.PersistentFlags().StringSliceVarP(&headers, "header", "H", headers,
		"A list of http headers (use Host for authority) - 'name: value', following curl syntax")
	rootCmd.PersistentFlags().StringVar(&caFile, "ca", "/cert.crt", "CA root cert file")
	rootCmd.PersistentFlags().StringVar(&msg, "msg", "HelloWorld",
		"message to send (for websockets)")
	rootCmd.PersistentFlags().StringVar(&method, "method", "", "method to use (for HTTP)")
	rootCmd.PersistentFlags().BoolVar(&http2, "http2", false,
		"send http requests as HTTP with prior knowledge")
	rootCmd.PersistentFlags().BoolVar(&serverFirst, "server-first", false,
		"Treat as a server first protocol; do not send request until magic string is received")
	rootCmd.PersistentFlags().StringVar(&clientCert, "client-cert", "", "client certificate file to use for request")
	rootCmd.PersistentFlags().StringVar(&clientKey, "client-key", "", "client certificate key file to use for request")

	loggingOptions.AttachCobraFlags(rootCmd)

	cmd.AddFlags(rootCmd)
}

// Adds http scheme to and url if not set. This matches curl logic
func defaultScheme(u string) string {
	p, err := url.Parse(u)
	if err != nil {
		return u
	}
	if p.Scheme == "" {
		return "http://" + u
	}
	return u
}

func getRequest(url string) (*proto.ForwardEchoRequest, error) {
	request := &proto.ForwardEchoRequest{
		Url:           defaultScheme(url),
		TimeoutMicros: common.DurationToMicros(timeout),
		Count:         int32(count),
		Qps:           int32(qps),
		Message:       msg,
		Http2:         http2,
		ServerFirst:   serverFirst,
		Method:        method,
	}

	for _, header := range headers {
		parts := strings.Split(header, ":")

		// require name:value format
		if len(parts) != 2 {
			return nil, fmt.Errorf("invalid header format: %q (want name:value)", header)
		}

		request.Headers = append(request.Headers, &proto.Header{
			Key:   parts[0],
			Value: strings.Trim(parts[1], " "),
		})
	}

	if clientCert != "" && clientKey != "" {
		certData, err := ioutil.ReadFile(clientCert)
		if err != nil {
			return nil, fmt.Errorf("failed to load client certificate: %v", err)
		}
		request.Cert = string(certData)
		keyData, err := ioutil.ReadFile(clientKey)
		if err != nil {
			return nil, fmt.Errorf("failed to load client certificate key: %v", err)
		}
		request.Key = string(keyData)
	}
	return request, nil
}

func main() {
	if err := rootCmd.Execute(); err != nil {
		os.Exit(-1)
	}
}
<|endoftext|>


<commit_before>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2017, 2018 Red Hat, Inc.
 *
 */

package virtconfig

/*
 This module is intended for determining whether an optional feature is enabled or not at the cluster-level.
*/

const (
	CPUManager            = "CPUManager"
	IgnitionGate          = "ExperimentalIgnitionSupport"
	LiveMigrationGate     = "LiveMigration"
	CPUNodeDiscoveryGate  = "CPUNodeDiscovery"
	HypervStrictCheckGate = "HypervStrictCheck"
	SidecarGate           = "Sidecar"
	GPUGate               = "GPU"
	SnapshotGate          = "Snapshot"
	DataVolumesGate       = "DataVolumes"
	SRIOVGate             = "SRIOVGate"
	HostDiskGate          = "HostDisk"
)

func (c *ClusterConfig) isFeatureGateEnabled(featureGate string) bool {
	for _, fg := range c.GetConfig().DeveloperConfiguration.FeatureGates {
		if fg == featureGate {
			return true
		}
	}
	return false
}

func (config *ClusterConfig) CPUManagerEnabled() bool {
	return config.isFeatureGateEnabled(CPUManager)
}

func (config *ClusterConfig) IgnitionEnabled() bool {
	return config.isFeatureGateEnabled(IgnitionGate)
}

func (config *ClusterConfig) LiveMigrationEnabled() bool {
	return config.isFeatureGateEnabled(LiveMigrationGate)
}

func (config *ClusterConfig) HypervStrictCheckEnabled() bool {
	return config.isFeatureGateEnabled(HypervStrictCheckGate)
}

func (config *ClusterConfig) CPUNodeDiscoveryEnabled() bool {
	return config.isFeatureGateEnabled(CPUNodeDiscoveryGate)
}

func (config *ClusterConfig) SidecarEnabled() bool {
	return config.isFeatureGateEnabled(SidecarGate)
}

func (config *ClusterConfig) GPUPassthroughEnabled() bool {
	return config.isFeatureGateEnabled(GPUGate)
}

func (config *ClusterConfig) SnapshotEnabled() bool {
	return config.isFeatureGateEnabled(SnapshotGate)
}

func (config *ClusterConfig) DataVolumesEnabled() bool {
	return config.isFeatureGateEnabled(DataVolumesGate)
}

func (config *ClusterConfig) SriovEnabled() bool {
	return config.isFeatureGateEnabled(SRIOVGate)
}

func (config *ClusterConfig) HostDiskEnabled() bool {
	return config.isFeatureGateEnabled(HostDiskGate)
}
<commit_msg>remove DataVolume feature gate<commit_after>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2017, 2018 Red Hat, Inc.
 *
 */

package virtconfig

/*
 This module is intended for determining whether an optional feature is enabled or not at the cluster-level.
*/

const (
	CPUManager            = "CPUManager"
	IgnitionGate          = "ExperimentalIgnitionSupport"
	LiveMigrationGate     = "LiveMigration"
	CPUNodeDiscoveryGate  = "CPUNodeDiscovery"
	HypervStrictCheckGate = "HypervStrictCheck"
	SidecarGate           = "Sidecar"
	GPUGate               = "GPU"
	SnapshotGate          = "Snapshot"
	SRIOVGate             = "SRIOVGate"
	HostDiskGate          = "HostDisk"
)

func (c *ClusterConfig) isFeatureGateEnabled(featureGate string) bool {
	for _, fg := range c.GetConfig().DeveloperConfiguration.FeatureGates {
		if fg == featureGate {
			return true
		}
	}
	return false
}

func (config *ClusterConfig) CPUManagerEnabled() bool {
	return config.isFeatureGateEnabled(CPUManager)
}

func (config *ClusterConfig) IgnitionEnabled() bool {
	return config.isFeatureGateEnabled(IgnitionGate)
}

func (config *ClusterConfig) LiveMigrationEnabled() bool {
	return config.isFeatureGateEnabled(LiveMigrationGate)
}

func (config *ClusterConfig) HypervStrictCheckEnabled() bool {
	return config.isFeatureGateEnabled(HypervStrictCheckGate)
}

func (config *ClusterConfig) CPUNodeDiscoveryEnabled() bool {
	return config.isFeatureGateEnabled(CPUNodeDiscoveryGate)
}

func (config *ClusterConfig) SidecarEnabled() bool {
	return config.isFeatureGateEnabled(SidecarGate)
}

func (config *ClusterConfig) GPUPassthroughEnabled() bool {
	return config.isFeatureGateEnabled(GPUGate)
}

func (config *ClusterConfig) SnapshotEnabled() bool {
	return config.isFeatureGateEnabled(SnapshotGate)
}

func (config *ClusterConfig) SriovEnabled() bool {
	return config.isFeatureGateEnabled(SRIOVGate)
}

func (config *ClusterConfig) HostDiskEnabled() bool {
	return config.isFeatureGateEnabled(HostDiskGate)
}
<|endoftext|>


<commit_before>package benchmarks

import (
	"fmt"
	"runtime"
	"testing"

	"github.com/smartystreets/go-disruptor"
)

const (
	singleWriterRingBufferSize = 1024 * 64
	singleWriterRingBufferMask = singleWriterRingBufferSize - 1
	reserveOne                 = 1
	reserveMany                = 16
	reserveManyDelta           = reserveMany - 1
)

func BenchmarkDisruptorWriterReserveSingle(b *testing.B) {
	ringBuffer := [singleWriterRingBufferSize]int64{}
	written, read := disruptor.NewCursor(), disruptor.NewCursor()
	reader := disruptor.NewReader(read, written, written, singleWriterConsumer{&ringBuffer})
	writer := disruptor.NewWriter(written, read, singleWriterRingBufferSize)
	reader.Start()

	iterations := int64(b.N)
	b.ReportAllocs()
	b.ResetTimer()

	sequence := disruptor.InitialSequenceValue
	for sequence < iterations {
		sequence = writer.Reserve(reserveOne)
		ringBuffer[sequence&singleWriterRingBufferMask] = sequence
		writer.Commit(sequence, sequence)
	}

	reader.Stop()
}
func BenchmarkDisruptorWriterReserveMultiple(b *testing.B) {
	ringBuffer := [singleWriterRingBufferSize]int64{}
	written, read := disruptor.NewCursor(), disruptor.NewCursor()
	reader := disruptor.NewReader(read, written, written, singleWriterConsumer{&ringBuffer})
	writer := disruptor.NewWriter(written, read, singleWriterRingBufferSize)
	reader.Start()

	iterations := int64(b.N)
	b.ReportAllocs()
	b.ResetTimer()

	sequence := disruptor.InitialSequenceValue
	for sequence < iterations {
		sequence = writer.Reserve(reserveMany)

		for i := sequence - reserveManyDelta; i <= sequence; i++ {
			ringBuffer[i&singleWriterRingBufferMask] = i
		}

		writer.Commit(sequence, sequence)
	}

	reader.Stop()
}

func benchmarkSingleWriter(b *testing.B, maxClaim int64) {

}

type singleWriterConsumer struct {
	ringBuffer *[singleWriterRingBufferSize]int64
}

func (this singleWriterConsumer) Consume(lower, upper int64) {
	for lower <= upper {
		message := this.ringBuffer[lower&singleWriterRingBufferMask]
		if message != lower {
			panic(fmt.Sprintf("\nRace condition %d %d\n", lower, message))
		}
		lower++
	}
}

func init() {
	runtime.GOMAXPROCS(2)
}
<commit_msg>Removed empty test.<commit_after>package benchmarks

import (
	"fmt"
	"runtime"
	"testing"

	"github.com/smartystreets/go-disruptor"
)

const (
	singleWriterRingBufferSize = 1024 * 64
	singleWriterRingBufferMask = singleWriterRingBufferSize - 1
	reserveOne                 = 1
	reserveMany                = 16
	reserveManyDelta           = reserveMany - 1
)

func BenchmarkDisruptorWriterReserveSingle(b *testing.B) {
	ringBuffer := [singleWriterRingBufferSize]int64{}
	written, read := disruptor.NewCursor(), disruptor.NewCursor()
	reader := disruptor.NewReader(read, written, written, singleWriterConsumer{&ringBuffer})
	writer := disruptor.NewWriter(written, read, singleWriterRingBufferSize)
	reader.Start()

	iterations := int64(b.N)
	b.ReportAllocs()
	b.ResetTimer()

	sequence := disruptor.InitialSequenceValue
	for sequence < iterations {
		sequence = writer.Reserve(reserveOne)
		ringBuffer[sequence&singleWriterRingBufferMask] = sequence
		writer.Commit(sequence, sequence)
	}

	reader.Stop()
}
func BenchmarkDisruptorWriterReserveMultiple(b *testing.B) {
	ringBuffer := [singleWriterRingBufferSize]int64{}
	written, read := disruptor.NewCursor(), disruptor.NewCursor()
	reader := disruptor.NewReader(read, written, written, singleWriterConsumer{&ringBuffer})
	writer := disruptor.NewWriter(written, read, singleWriterRingBufferSize)
	reader.Start()

	iterations := int64(b.N)
	b.ReportAllocs()
	b.ResetTimer()

	sequence := disruptor.InitialSequenceValue
	for sequence < iterations {
		sequence = writer.Reserve(reserveMany)

		for i := sequence - reserveManyDelta; i <= sequence; i++ {
			ringBuffer[i&singleWriterRingBufferMask] = i
		}

		writer.Commit(sequence, sequence)
	}

	reader.Stop()
}

type singleWriterConsumer struct {
	ringBuffer *[singleWriterRingBufferSize]int64
}

func (this singleWriterConsumer) Consume(lower, upper int64) {
	for lower <= upper {
		message := this.ringBuffer[lower&singleWriterRingBufferMask]
		if message != lower {
			panic(fmt.Sprintf("\nRace condition %d %d\n", lower, message))
		}
		lower++
	}
}

func init() {
	runtime.GOMAXPROCS(2)
}
<|endoftext|>


<commit_before>package packager

import(
	tp "tritium/proto"
	proto "goprotobuf.googlecode.com/hg/proto"
	yaml "launchpad.net/goyaml"
	"io/ioutil"
	"log"
	"strings"
	"fmt"
	"exec"
	linker "tritium/linker"
	"path/filepath"
)

type Package struct {
	loaded []*PackageInfo
	location string
	LoadPath string
	*tp.Package
}

type PackageInfo struct {
	Name string
	Dependencies []string
	Types []string
}

func BuildDefaultPackage(dir string) (*Package) {
	// Terrible directory handling here... has to be executed from Tritium root
	pkg := NewPackage(dir)

	//pkg.Load("base")
	//pkg.Load("node")
	pkg.Load("libxml")
	println("Packages all loaded")

	return pkg
}

func NewPackage(loadPath string) (*Package){
	return &Package{
		Package: &tp.Package{
			Name: proto.String("combined"),
			Functions: make([]*tp.Function, 0),
			Types: make([]*tp.Type, 0),
		},
		loaded: make([]*PackageInfo, 0),
		LoadPath: loadPath,
	}
}

func (pkg *Package)Load(packageName string) {

	location := filepath.Join(pkg.LoadPath, packageName)
	pkg.location = location

	info := readPackageInfoFile(location)

	if len(info.Dependencies) > 0 {

		println("==========\nLoading dependencies:")

		for _, dependency := range(info.Dependencies) {
			pkg.loadPackageDependency(dependency)
		}

		println("done.\n==========")

	}

	fmt.Printf("%v\n", location)

	for _, typeName := range(info.Types) {
		split := strings.Split(typeName, " < ")
		typeObj := &tp.Type{}
		if len(split) == 2 {
			typeName = split[0]
			index := pkg.findTypeIndex(split[1])

			typeObj.Implements = proto.Int32(int32(index))
		}
		typeObj.Name = proto.String(typeName)
		pkg.Types = append(pkg.Types, typeObj)
	}

	pkg.readHeaderFile(location)

	pkg.readPackageDefinitions(location)

	pkg.inheritFunctions()

	println(" -- done\n\n\n\n\n")
}

func (pkg *Package)resolveFunction(fun *tp.Function) {
	linkingContext := linker.NewLinkingContext(pkg.Package)

//	pkg.resolveFunctionDescendants(fun)

	fmt.Printf("\t -- Resolving --\n")
	fmt.Printf("\t\t -- function: %v\n", fun)

	// Re-uses linker's logic to resolve function definitions
	if ( proto.GetBool( fun.BuiltIn ) == false) {
		typeName := proto.GetString(fun.ScopeType)

		if len(typeName) != 0 {
			// When I pass in functions from the inheritance resolver, they're typeId is already set
			fun.ScopeTypeId = pkg.GetProtoTypeId(fun.ScopeType)
			fun.ScopeType = nil
		}

		localScope := make(linker.LocalDef, len(fun.Args))

		//		fun.ReturnTypeId = pkg.GetProtoTypeId(fun.ReturnType)
		for _, arg := range(fun.Args) {
			argTypeName := arg.TypeString
			var argTypeId int

			if argTypeName != nil {
				// Similar deal. Input functions from inheritance resolution already have ids set

				arg.TypeId = pkg.GetProtoTypeId(arg.TypeString)
				//println("Processing %", proto.GetString(arg.Name))
				argTypeId = pkg.GetTypeId(proto.GetString(arg.TypeString))
				arg.TypeString = nil
			} else {
				argTypeId = int( proto.GetInt32(arg.TypeId) )
			}

			localScope[proto.GetString(arg.Name)] = argTypeId
		}

		//fmt.Printf("Some insitruction: %v, %s", fun.Instruction, proto.GetString(fun.Name) )
		scopeTypeId := int(proto.GetInt32(fun.ScopeTypeId))
		fmt.Printf("\t\t -- opening scope type : %v\n", scopeTypeId)
		returnType := linkingContext.ProcessInstructionWithLocalScope(fun.Instruction, scopeTypeId, localScope)
		fun.ReturnTypeId = proto.Int32(int32(returnType))
	}
	pkg.Package.Functions = append(pkg.Package.Functions, fun)
	fmt.Printf("\t\t -- done --\n")
}

func (pkg *Package)inheritFunctions() {
	fmt.Printf("pkg types: %v", pkg.Types)
	for _, function := range(pkg.Functions) {
		pkg.resolveFunctionDescendants(function)
	}
}

// TODO : Make a hash for this
func (pkg *Package)findDescendentType(thisType int32) int {
	for index, someType := range(pkg.Types) {
		implements := proto.GetInt32(someType.Implements)
		if implements == thisType && implements != 0 {
			// The implements field defaults to 0. Base doesn't implement Base. Text doesn't implement Base
			// TODO(SJ): make the default value -1 so I know if its not set versus something is inheriting from base
			fmt.Printf("=== %v is ancestor of %v === (%v is of type %v and implements : %v)\n", thisType, someType, proto.GetString(someType.Name), index, implements)
			return index
		}
	}
	return -1
}

// TODO(SJ) : Make this not suck. I think I could make this 50% shorter if I use reflection
// - Also, I'm assuming a single depth level of inheritance. I'd have to run this function n times for n levels
// - Well that should be fine as long as I run it at the end of every package load

func (pkg *Package)resolveFunctionDescendants(fun *tp.Function) {

	// Check if this function contains any types that have descendants
	name := fun.Stub(pkg.Package)
	println("Checking for inheritance on function:", name )

	newFun := &tp.Function{}
	inherit := false

	// Iterate over ScopeType, Arg types, return Type, opens Type

	// ScopeType

	thisTypeId := proto.GetInt32(fun.ScopeTypeId)
	newType := pkg.findDescendentType(thisTypeId)

	if newType != -1 {
		if !inherit {
			fmt.Printf("\t -- ScopeType : Found ancestral type. Cloning function %v\n", proto.GetString( fun.Name ) )
			newFun = fun.Clone()
			// fmt.Printf("\t -- New fun: %v", newFun)
			inherit = true
		}
		println("\t -- Resetting scopeId")
		newFun.ScopeTypeId = proto.Int32( int32( newType ) )
	}

	// ReturnType

	thisTypeId = proto.GetInt32(fun.ReturnTypeId)
	newType = pkg.findDescendentType(thisTypeId)

	if newType != -1 {
		if !inherit {
			fmt.Printf("\t -- ReturnType : Found ancestral type. Cloning function %v\n", proto.GetString( fun.Name ) )
			newFun = fun.Clone()
			// fmt.Printf("\t -- New fun: %v", newFun)
			inherit = true
		}
		println("\t -- Resetting returnId")
		newFun.ReturnTypeId = proto.Int32( int32( newType ) )
	}

	// OpensType

	thisTypeId = proto.GetInt32(fun.OpensTypeId)
	newType = pkg.findDescendentType(thisTypeId)

	if newType != -1 {

		if !inherit {
			fmt.Printf("\t -- OpensType : Found ancestral type. Cloning function %v\n", proto.GetString( fun.Name ) )
			newFun = fun.Clone()
			// fmt.Printf("\t -- New fun: %v", newFun)
			inherit = true
		}
		println("\t -- Resetting openTypeId")
		newFun.OpensTypeId = proto.Int32( int32( newType ) )
	}

	// Arguments

	for index, arg := range( fun.Args) {
		thisTypeId = proto.GetInt32(arg.TypeId)
		newType = pkg.findDescendentType(thisTypeId)

		if newType != -1 {

			if !inherit {
				fmt.Printf("\t -- ArgType : Found ancestral type. Cloning function %v\n", proto.GetString( fun.Name ) )
				newFun = fun.Clone()
				// fmt.Printf("\t -- New fun: %v", newFun)
				inherit = true
			}
			println("\t -- Resetting argument")
			newFun.Args[index].TypeId = proto.Int32( int32( newType ) )
		}

	}

	fmt.Printf("\t -- Old function: %v\n\t -- New function: %v\n", fun, newFun)

	if inherit {
		pkg.resolveFunction(newFun)
	}

}

func (pkg *Package)readPackageDefinitions(location string) {

	println(" -- reading definitions")

	// Execute the ts2func-ruby script

	package_name := strings.Split(location,"/")[1]
	input_file := filepath.Join(location, "functions.ts")
	output_file := filepath.Join(location, package_name + ".tf")

	println(input_file)

	// Assume that tritium/bin is in $PATH (it will be when you install the gem)
	// -- if you're developing, add $REPOS/tritium/bin to $PATH

	command := exec.Command("ts2func-ruby", "-s", input_file, output_file)

	//fmt.Printf("\n\nExecuting command: \n %v\n", command)

	output, err := command.CombinedOutput()

	if err != nil {
		fmt.Printf("\tFunction conversion output:\n\t %s", output)
		log.Panic(err)
	}

	functions := &tp.FunctionArray{}
	data, err := ioutil.ReadFile(output_file)

	if err != nil {
		println("Failed to read output file.")
		log.Panic(err)
	}

	err = proto.Unmarshal(data, functions)

	if err != nil {
		println("Failed while loading output from ts2func.")
		println(string(output))
		log.Panic(err)
	}

	//fmt.Printf("functions : %v", functions)
	//fmt.Printf("\n\n prelim pkg functions : %v\n", pkg.Package.Functions) */

	//println("Function count before ", len(pkg.Package.Functions))
	for _, function := range(functions.Functions) {
		fmt.Printf("\t -- function: %v", function)
		pkg.resolveFunction(function)
	}
	//fmt.Printf("\n\npkg functions : %v\n", pkg.Package.Functions)
	//println("Function count after ", len(pkg.Package.Functions))
	//pkg.Package.Functions = functions.Functions

	//fmt.Printf("\n\npkg functions : %v\n", pkg.Package.Functions)

}

func (pkg *Package)Marshal() []byte {
	bytes, err := proto.Marshal(pkg.Package)
	if err != nil {
		log.Panic(err)
	}
	return bytes
}

func (pkg *Package)findTypeIndex(name string) int {
	for index, typeObj := range(pkg.Types) {
		if name == proto.GetString(typeObj.Name) {
			return index
		}
	}

	log.Panic("Bad type load order, type", name, "unknown")
	return -1
}

func (pkg *Package)loadPackageDependency(name string) {

	// Try and load the dependency
	// TODO : remove passing location around since I added it to the Package struct

	// TODO : Check for a pre-built package (pre-req is outputting a .tpkg file upon completion of a package load)

	newPath := filepath.Join(pkg.LoadPath, name)
	_, err := ioutil.ReadDir(newPath)

	if err == nil {
		// Directory exists
		pkg.Load(name)
	} else {
		println("Cannot find package at:", newPath)
		log.Panic(err)
	}

}

// Not fully functional. Dang it.
func readPackageInfoFile(location string) (*PackageInfo){
	packageInfo := &PackageInfo{}
	infoFile, err := ioutil.ReadFile(location + "/package.yml");
	if err != nil {
		log.Panic("No package info file found at " + location + "/package.yml")
	}
	yaml.Unmarshal([]byte(infoFile), &packageInfo)
	//fmt.Printf("--- m:\n%v\n\n", packageInfo)
	return packageInfo
}

// Not fully functional. Dang it.
func (pkg *Package)readHeaderFile(location string) {
	headerFile, err := ioutil.ReadFile(location + "/headers.tf");
	if err != nil {
		log.Panic("No header file found at " + location + "/headers.tf")
	}
	headerLines := strings.Split(string(headerFile), "\n")
	for _, line := range(headerLines) {
		if (len(line) > 2) && (line[:2] != "//") {
			function := &tp.Function{Args: make([]*tp.Function_Argument, 0)}
			line = line[6:]
			methodName := strings.Split(line, "(")[0]
			typeId := 0
			back := line[len(methodName)+1:]
			methodizer := strings.Split(methodName, ".")
			if len(methodizer) > 1 {
				typeStr := methodizer[0]
				typeId = pkg.GetTypeId(typeStr)
				methodName = methodizer[1]
			}
			function.Name = proto.String(methodName)
			function.ScopeTypeId = proto.Int32(int32(typeId))

			splat := strings.Split(back, ")")
			argString := splat[0]
			// Process the arguments
			if len(argString) > 0 {
				argList := strings.Split(argString, ",")
				for _, argLine := range(argList) {
					arg := &tp.Function_Argument{}
					typeStr := strings.Split(strings.Trim(argLine, " \t"), " ")[0]
					typeId = pkg.findTypeIndex(typeStr)
					arg.TypeId = proto.Int32(int32(typeId))
					function.Args = append(function.Args, arg)
				}
			}

			back = splat[1]

			hintStrs := strings.Split(back, "//")
			if len(hintStrs) == 2 {
				hints := strings.Split(hintStrs[1], ",")
				if len(hints) >= 1 {
					 function.ReturnTypeId = proto.Int32(int32(pkg.findTypeIndex(hints[0])))
				}
				if len(hints) >= 2 {
					 function.OpensTypeId = proto.Int32(int32(pkg.findTypeIndex(hints[1])))
				}
			}
			function.BuiltIn = proto.Bool(true)

			pkg.Functions = append(pkg.Functions, function)
		}
	}
}

func (pkg *Package)SerializedOutput() {

	bytes, err := proto.Marshal(pkg.Package)
	if err != nil {
		log.Panic(err)
	}
	println(string(bytes))
}
<commit_msg>Converted outputs to file log.<commit_after>package packager

import(
	tp "tritium/proto"
	proto "goprotobuf.googlecode.com/hg/proto"
	yaml "launchpad.net/goyaml"
	"io/ioutil"
	"log"
	"strings"
	"fmt"
	"exec"
	linker "tritium/linker"
	"path/filepath"
	"log4go"
)

type Package struct {
	loaded []*PackageInfo
	location string
	LoadPath string
	Log log4go.Logger
	*tp.Package
}

type PackageInfo struct {
	Name string
	Dependencies []string
	Types []string
}

func BuildDefaultPackage(dir string) (*Package) {
	// Terrible directory handling here... has to be executed from Tritium root
	pkg := NewPackage(dir)

	//pkg.Load("base")
	//pkg.Load("node")
	pkg.Load("libxml")
	println("Packages all loaded")

	return pkg
}

func NewPackage(loadPath string) (*Package){
	return &Package{
		Package: &tp.Package{
			Name: proto.String("combined"),
			Functions: make([]*tp.Function, 0),
			Types: make([]*tp.Type, 0),
		},
		loaded: make([]*PackageInfo, 0),
  	        Log: newLog(),
		LoadPath: loadPath,
	}
}

func newLog() (log4go.Logger) {
	log := log4go.NewLogger()
	log.AddFilter("file", log4go.FINE, log4go.NewFileLogWriter("debug.log", false))
	return log
}

func (pkg *Package)Load(packageName string) {

	location := filepath.Join(pkg.LoadPath, packageName)
	pkg.location = location

	println(location)

	info := readPackageInfoFile(location)

	if len(info.Dependencies) > 0 {

		//println("==========\nLoading dependencies:")

		for _, dependency := range(info.Dependencies) {
			pkg.loadPackageDependency(dependency)
		}

		//println("done.\n==========")

	}

	for _, typeName := range(info.Types) {
		split := strings.Split(typeName, " < ")
		typeObj := &tp.Type{}
		if len(split) == 2 {
			typeName = split[0]
			index := pkg.findTypeIndex(split[1])

			typeObj.Implements = proto.Int32(int32(index))
		}
		typeObj.Name = proto.String(typeName)
		pkg.Types = append(pkg.Types, typeObj)
	}

	pkg.readHeaderFile(location)

	pkg.readPackageDefinitions(location)

	pkg.inheritFunctions()

	println(" -- done")
	pkg.Log.Close()
}

func (pkg *Package)resolveFunction(fun *tp.Function) {
	linkingContext := linker.NewLinkingContext(pkg.Package)

//	pkg.resolveFunctionDescendants(fun)

	pkg.Log.Info("\t -- Resolving --\n")
	pkg.Log.Info("\t\t -- function: %v\n", fun)

	// Re-uses linker's logic to resolve function definitions
	if ( proto.GetBool( fun.BuiltIn ) == false) {
		typeName := proto.GetString(fun.ScopeType)

		if len(typeName) != 0 {
			// When I pass in functions from the inheritance resolver, they're typeId is already set
			fun.ScopeTypeId = pkg.GetProtoTypeId(fun.ScopeType)
			fun.ScopeType = nil
		}

		localScope := make(linker.LocalDef, len(fun.Args))

		//		fun.ReturnTypeId = pkg.GetProtoTypeId(fun.ReturnType)
		for _, arg := range(fun.Args) {
			argTypeName := arg.TypeString
			var argTypeId int

			if argTypeName != nil {
				// Similar deal. Input functions from inheritance resolution already have ids set

				arg.TypeId = pkg.GetProtoTypeId(arg.TypeString)
				//println("Processing %", proto.GetString(arg.Name))
				argTypeId = pkg.GetTypeId(proto.GetString(arg.TypeString))
				arg.TypeString = nil
			} else {
				argTypeId = int( proto.GetInt32(arg.TypeId) )
			}

			localScope[proto.GetString(arg.Name)] = argTypeId
		}

		//pkg.Log.Info("Some insitruction: %v, %s", fun.Instruction, proto.GetString(fun.Name) )
		scopeTypeId := int(proto.GetInt32(fun.ScopeTypeId))
		pkg.Log.Info("\t\t -- opening scope type : %v\n", scopeTypeId)
		returnType := linkingContext.ProcessInstructionWithLocalScope(fun.Instruction, scopeTypeId, localScope)
		fun.ReturnTypeId = proto.Int32(int32(returnType))
	}
	pkg.Package.Functions = append(pkg.Package.Functions, fun)
	pkg.Log.Info("\t\t -- done --\n")
}

func (pkg *Package)inheritFunctions() {
	pkg.Log.Info("pkg types: %v", pkg.Types)
	for _, function := range(pkg.Functions) {
		pkg.resolveFunctionDescendants(function)
	}
}

// TODO : Make a hash for this
func (pkg *Package)findDescendentType(thisType int32) int {
	for index, someType := range(pkg.Types) {
		implements := proto.GetInt32(someType.Implements)
		if implements == thisType && implements != 0 {
			// The implements field defaults to 0. Base doesn't implement Base. Text doesn't implement Base
			// TODO(SJ): make the default value -1 so I know if its not set versus something is inheriting from base
			pkg.Log.Info("=== %v is ancestor of %v === (%v is of type %v and implements : %v)\n", thisType, someType, proto.GetString(someType.Name), index, implements)
			return index
		}
	}
	return -1
}

// TODO(SJ) : Make this not suck. I think I could make this 50% shorter if I use reflection
// - Also, I'm assuming a single depth level of inheritance. I'd have to run this function n times for n levels
// - Well that should be fine as long as I run it at the end of every package load

func (pkg *Package)resolveFunctionDescendants(fun *tp.Function) {

	// Check if this function contains any types that have descendants
	name := fun.Stub(pkg.Package)
	pkg.Log.Info("Checking for inheritance on function: %v", name )

	newFun := &tp.Function{}
	inherit := false

	// Iterate over ScopeType, Arg types, return Type, opens Type

	// ScopeType

	thisTypeId := proto.GetInt32(fun.ScopeTypeId)
	newType := pkg.findDescendentType(thisTypeId)

	if newType != -1 {
		if !inherit {
			pkg.Log.Info("\t -- ScopeType : Found ancestral type. Cloning function %v\n", proto.GetString( fun.Name ) )
			newFun = fun.Clone()
			// pkg.Log.Info("\t -- New fun: %v", newFun)
			inherit = true
		}
		pkg.Log.Info("\t -- Resetting scopeId")
		newFun.ScopeTypeId = proto.Int32( int32( newType ) )
	}

	// ReturnType

	thisTypeId = proto.GetInt32(fun.ReturnTypeId)
	newType = pkg.findDescendentType(thisTypeId)

	if newType != -1 {
		if !inherit {
			pkg.Log.Info("\t -- ReturnType : Found ancestral type. Cloning function %v\n", proto.GetString( fun.Name ) )
			newFun = fun.Clone()
			// pkg.Log.Info("\t -- New fun: %v", newFun)
			inherit = true
		}
		pkg.Log.Info("\t -- Resetting returnId")
		newFun.ReturnTypeId = proto.Int32( int32( newType ) )
	}

	// OpensType

	thisTypeId = proto.GetInt32(fun.OpensTypeId)
	newType = pkg.findDescendentType(thisTypeId)

	if newType != -1 {

		if !inherit {
			pkg.Log.Info("\t -- OpensType : Found ancestral type. Cloning function %v\n", proto.GetString( fun.Name ) )
			newFun = fun.Clone()
			// pkg.Log.Info("\t -- New fun: %v", newFun)
			inherit = true
		}
		pkg.Log.Info("\t -- Resetting openTypeId")
		newFun.OpensTypeId = proto.Int32( int32( newType ) )
	}

	// Arguments

	for index, arg := range( fun.Args) {
		thisTypeId = proto.GetInt32(arg.TypeId)
		newType = pkg.findDescendentType(thisTypeId)

		if newType != -1 {

			if !inherit {
				pkg.Log.Info("\t -- ArgType : Found ancestral type. Cloning function %v\n", proto.GetString( fun.Name ) )
				newFun = fun.Clone()
				// pkg.Log.Info("\t -- New fun: %v", newFun)
				inherit = true
			}
			pkg.Log.Info("\t -- Resetting argument")
			newFun.Args[index].TypeId = proto.Int32( int32( newType ) )
		}

	}

	pkg.Log.Info("\t -- Old function: %v\n\t -- New function: %v\n", fun, newFun)

	if inherit {
		pkg.resolveFunction(newFun)
	}

}

func (pkg *Package)readPackageDefinitions(location string) {

	println(" -- reading definitions")

	// Execute the ts2func-ruby script

	package_name := strings.Split(location,"/")[1]
	input_file := filepath.Join(location, "functions.ts")
	output_file := filepath.Join(location, package_name + ".tf")

	// Assume that tritium/bin is in $PATH (it will be when you install the gem)
	// -- if you're developing, add $REPOS/tritium/bin to $PATH

	command := exec.Command("ts2func-ruby", "-s", input_file, output_file)

	//pkg.Log.Info("\n\nExecuting command: \n %v\n", command)

	output, err := command.CombinedOutput()

	if err != nil {
		fmt.Printf("\tFunction conversion output:\n\t %s", output)
		log.Panic(err)
	}

	functions := &tp.FunctionArray{}
	data, err := ioutil.ReadFile(output_file)

	if err != nil {
		println("Failed to read output file.")
		log.Panic(err)
	}

	err = proto.Unmarshal(data, functions)

	if err != nil {
		println("Failed while loading output from ts2func.")
		println(string(output))
		log.Panic(err)
	}

	//fmt.Printf("functions : %v", functions)
	//fmt.Printf("\n\n prelim pkg functions : %v\n", pkg.Package.Functions) */

	//println("Function count before ", len(pkg.Package.Functions))
	for _, function := range(functions.Functions) {
		pkg.Log.Info("\t -- function: %v", function)
		pkg.resolveFunction(function)
	}
	//fmt.Printf("\n\npkg functions : %v\n", pkg.Package.Functions)
	//println("Function count after ", len(pkg.Package.Functions))
	//pkg.Package.Functions = functions.Functions

	//fmt.Printf("\n\npkg functions : %v\n", pkg.Package.Functions)

}

func (pkg *Package)Marshal() []byte {
	bytes, err := proto.Marshal(pkg.Package)
	if err != nil {
		log.Panic(err)
	}
	return bytes
}

func (pkg *Package)findTypeIndex(name string) int {
	for index, typeObj := range(pkg.Types) {
		if name == proto.GetString(typeObj.Name) {
			return index
		}
	}

	log.Panic("Bad type load order, type", name, "unknown")
	return -1
}

func (pkg *Package)loadPackageDependency(name string) {

	// Try and load the dependency
	// TODO : remove passing location around since I added it to the Package struct

	// TODO : Check for a pre-built package (pre-req is outputting a .tpkg file upon completion of a package load)

	newPath := filepath.Join(pkg.LoadPath, name)
	_, err := ioutil.ReadDir(newPath)

	if err == nil {
		// Directory exists
		pkg.Load(name)
	} else {
		println("Cannot find package at:", newPath)
		log.Panic(err)
	}

}

// Not fully functional. Dang it.
func readPackageInfoFile(location string) (*PackageInfo){
	packageInfo := &PackageInfo{}
	infoFile, err := ioutil.ReadFile(location + "/package.yml");
	if err != nil {
		log.Panic("No package info file found at " + location + "/package.yml")
	}
	yaml.Unmarshal([]byte(infoFile), &packageInfo)
	//fmt.Printf("--- m:\n%v\n\n", packageInfo)
	return packageInfo
}

// Not fully functional. Dang it.
func (pkg *Package)readHeaderFile(location string) {
	headerFile, err := ioutil.ReadFile(location + "/headers.tf");
	if err != nil {
		log.Panic("No header file found at " + location + "/headers.tf")
	}
	headerLines := strings.Split(string(headerFile), "\n")
	for _, line := range(headerLines) {
		if (len(line) > 2) && (line[:2] != "//") {
			function := &tp.Function{Args: make([]*tp.Function_Argument, 0)}
			line = line[6:]
			methodName := strings.Split(line, "(")[0]
			typeId := 0
			back := line[len(methodName)+1:]
			methodizer := strings.Split(methodName, ".")
			if len(methodizer) > 1 {
				typeStr := methodizer[0]
				typeId = pkg.GetTypeId(typeStr)
				methodName = methodizer[1]
			}
			function.Name = proto.String(methodName)
			function.ScopeTypeId = proto.Int32(int32(typeId))

			splat := strings.Split(back, ")")
			argString := splat[0]
			// Process the arguments
			if len(argString) > 0 {
				argList := strings.Split(argString, ",")
				for _, argLine := range(argList) {
					arg := &tp.Function_Argument{}
					typeStr := strings.Split(strings.Trim(argLine, " \t"), " ")[0]
					typeId = pkg.findTypeIndex(typeStr)
					arg.TypeId = proto.Int32(int32(typeId))
					function.Args = append(function.Args, arg)
				}
			}

			back = splat[1]

			hintStrs := strings.Split(back, "//")
			if len(hintStrs) == 2 {
				hints := strings.Split(hintStrs[1], ",")
				if len(hints) >= 1 {
					 function.ReturnTypeId = proto.Int32(int32(pkg.findTypeIndex(hints[0])))
				}
				if len(hints) >= 2 {
					 function.OpensTypeId = proto.Int32(int32(pkg.findTypeIndex(hints[1])))
				}
			}
			function.BuiltIn = proto.Bool(true)

			pkg.Functions = append(pkg.Functions, function)
		}
	}
}

func (pkg *Package)SerializedOutput() {

	bytes, err := proto.Marshal(pkg.Package)
	if err != nil {
		log.Panic(err)
	}
	println(string(bytes))
}
<|endoftext|>


<commit_before>/*
 *    Copyright (C) 2014 Michael Wendland
 *
 *    This program is free software: you can redistribute it and/or modify
 *    it under the terms of the GNU Affero General Public License as published
 *    by the Free Software Foundation, either version 3 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *    Authors:
 *      Michael Wendland <michiwend@michiwend.com>
 */

package goefa

import (
	"encoding/xml"
	"errors"
	_ "fmt"
	"net/http"
	"net/url"
	"strconv"

	"code.google.com/p/go-charset/charset"
	_ "code.google.com/p/go-charset/data"
)

type EFAProvider struct {
	Name string

	BaseURL                  string
	DepartureMonitorEndpoint string
	StopFinderEndpoint       string
	TripEndpoint             string

	//FIXME: include general params for all requests (e.g. useRealtime, ...)
}

//FIXME: separate goefa structs (like Station) and XML structs
func (efa *EFAProvider) FindStop(name string) (*StopInfo, error) {
	// FindStop queries the stopfinder API and returns Stops matching 'name'

	params := url.Values{
		"type_sf":              {"stop"},
		"name_sf":              {name},
		"locationServerActive": {"1"},
		"outputFormat":         {"XML"},
		"stateless":            {"1"},
		// "limit":                {"5"},
		// "mode":                 {"direct"},
	}

	resp, err := http.PostForm(efa.BaseURL+efa.StopFinderEndpoint, params)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	var result StopResult
	decoder := xml.NewDecoder(resp.Body)
	decoder.CharsetReader = charset.NewReader
	if err = decoder.Decode(&result); err != nil {
		return nil, err
	}

	if result.Stop.State != "identified" {
		return nil, errors.New("Stop does not exist or name is not unique!")
	}

	return &result.Stop, nil
}

//FIXME: turn station_id into an int
func (efa *EFAProvider) Departures(station *StopInfo, results int) ([]Departure, error) {
	params := url.Values{
		"type_dm":              {"stop"},
		"name_dm":              {strconv.Itoa(station.IdfdStop.StopID)},
		"useRealtime":          {"1"},
		"locationServerActive": {"1"},
		"dmLineSelection":      {"all"},
		"limit":                {strconv.Itoa(results)},
		"mode":                 {"direct"},
	}

	resp, err := http.PostForm(efa.BaseURL+efa.DepartureMonitorEndpoint, params)
	if err != nil {
		return []Departure{}, err
	}
	defer resp.Body.Close()

	var result StopResult
	decoder := xml.NewDecoder(resp.Body)
	decoder.CharsetReader = charset.NewReader
	if err = decoder.Decode(&result); err != nil {
		return []Departure{}, err
	}
	//fmt.Printf("%+v", result)

	if result.Stop.State != "identified" {
		return []Departure{}, errors.New("Stop does not exist or name is not unique!")
	}

	return result.Departures, nil
}
<commit_msg>fix StopFinder request, etc<commit_after>/*
 *    Copyright (C) 2014 Michael Wendland
 *
 *    This program is free software: you can redistribute it and/or modify
 *    it under the terms of the GNU Affero General Public License as published
 *    by the Free Software Foundation, either version 3 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 *    Authors:
 *      Michael Wendland <michiwend@michiwend.com>
 */

package goefa

import (
	"encoding/xml"
	"errors"
	"net/http"
	"net/url"
	"time"

	"code.google.com/p/go-charset/charset"
	_ "code.google.com/p/go-charset/data"
)

type EFAProvider struct {
	Name string

	BaseURL                  string
	DepartureMonitorEndpoint string
	StopFinderEndpoint       string
	TripEndpoint             string

	//FIXME: include general params for all requests (e.g. useRealtime, ...)
}

// FindStop queries the EFA StopFinder API and returns a list of stops
// or a single identified stop that where matched by the given name
func (efa *EFAProvider) FindStop(name string) (bool, []*Stop, error) {

	// Struct for unmarshaling StopFinderRequest into
	type stopFinderRequest struct {
		Odv struct {
			OdvPlace struct {
			}
			OdvName struct {
				State string  `xml:"state,attr"`
				Stops []*Stop `xml:"odvNameElem"`
			} `xml:"itdOdvName"`
		} `xml:"itdStopFinderRequest>itdOdv"`
	}

	// To get a more detailed response from the StopFinder request we can use
	// EFAs LocationServer (locationServerActive=1, type_sf=any). To limit the
	// results to a specific type we can use anyObjFilter_sf=<bitmask> as
	// following:
	//	0 any type
	//	1 locations
	//	2 stations
	//	4 streets
	//	8 addresses
	//	16 crossroads
	//	32 POIs
	//	64 postal codes
	// "stations and streets" results in 2 + 4 = 6

	params := url.Values{
		"type_sf":              {"any"},
		"name_sf":              {name},
		"outputFormat":         {"XML"},
		"stateless":            {"1"},
		"locationServerActive": {"1"},
		"anyObjFilter_sf":      {"2"},
	}

	resp, err := http.PostForm(efa.BaseURL+efa.StopFinderEndpoint, params)
	if err != nil {
		return false, nil, err
	}
	defer resp.Body.Close()

	var result stopFinderRequest

	decoder := xml.NewDecoder(resp.Body)
	decoder.CharsetReader = charset.NewReader
	if err = decoder.Decode(&result); err != nil {
		return false, nil, err
	}

	for _, stop := range result.Odv.OdvName.Stops {
		stop.Provider = efa
	}

	switch result.Odv.OdvName.State {
	case "identified":
		return true, result.Odv.OdvName.Stops, nil
	case "list":
		return false, result.Odv.OdvName.Stops, nil
	default:
		return false, nil, errors.New("no matched stops")
	}

}

func (efa *EFAProvider) Trip(origin Stop, via Stop, destination Stop, time time.Time) error {
	return nil
}
<|endoftext|>


<commit_before>package golis

import (
	"log"
	"net"
	"sync"
	"time"
)

//系统变量定义
var (
	GolisHandler IoHandler            //事件处理
	GolisPackage Packager             //拆包封包处理接口
	w            WaitGroupWrapper     //等待退出
	runnable     bool                 //服务运行状态
	rwTimeout    time.Duration    = 0 //超时单位秒
)

//定义waitGroup
type WaitGroupWrapper struct {
	sync.WaitGroup
}

//定义session
type Iosession struct {
	SesionId   uint32   //session唯一表示
	Connection net.Conn //连接
	IsAuth     bool     //是否认证成功
	closed     bool     //是否已经关闭
}

//session写入数据
func (this *Iosession) Write(message interface{}) {
	//触发消息发送事件
	GolisHandler.MessageSent(this, message)
	this.Connection.Write(GolisPackage.Packet(message))
}

//关闭连接
func (this *Iosession) Close() {
	if !this.closed {
		GolisHandler.SessionClosed(this)
		this.closed = true
	}
	this.Connection.Close()
}

//设置读写超时
func SetTimeout(timeoutSec time.Duration) {
	rwTimeout = timeoutSec
}

//拆包封包接口定义
type Packager interface {
	//读取连接数据
	//packageChan 准备好包后交给该chan
	ReadConnData(buffer *Buffer, packageChan chan<- *[]byte)
	//拆包函数
	Unpacket(data []byte) interface{}
	//封包函数
	Packet(msg interface{}) []byte
}

//事件触发接口定义
type IoHandler interface {
	//session打开
	SessionOpened(session *Iosession)
	//session关闭
	SessionClosed(session *Iosession)
	//收到消息时触发
	MessageReceived(session *Iosession, message interface{})
	//消息发送时触发
	MessageSent(session *Iosession, message interface{})
}

//服务器端运行golis
//netPro：运行协议参数，tcp/udp
//laddr ：程序监听ip和端口，如127.0.0.1:8080
func Run(netPro, laddr string) {
	runnable = true
	log.Println("golis is listen port:", laddr)

	netLis, err := net.Listen(netPro, laddr)
	if err != nil {
		log.Fatalln(err)
	}
	defer netLis.Close()
	log.Println("waiting clients...")
	for runnable {
		conn, err := netLis.Accept()
		if err != nil {
			continue
		}
		go connectHandle(conn)
	}

	w.WaitGroup.Wait()
	log.Println("golis is safe exit")
}

//停止服务
func Stop() {
	runnable = false
}

//客户端程序连接服务器
func Dial(netPro, laddr string) {
	runnable = true
	conn, err := net.Dial(netPro, laddr)
	if err != nil {
		log.Fatalln(err)
	}
	go connectHandle(conn)
}

//处理新连接
func connectHandle(conn net.Conn) {
	w.Add(1)
	//声明一个临时缓冲区，用来存储被截断的数据
	ioBuffer := NewBuffer()

	buffer := make([]byte, 512)

	//声明一个管道用于接收解包的数据
	readerChannel := make(chan *[]byte, 16)
	//创建session
	session := Iosession{Connection: conn}
	//触发sessionCreated事件
	GolisHandler.SessionOpened(&session)

	exitChan := make(chan bool)
	go waitData(&session, readerChannel, exitChan)
	defer func() {
		conn.Close()
		ioBuffer = nil
		buffer = nil
		if err := recover(); err != nil {
			log.Println(err)
		}
		w.Done()
	}()

	for runnable && !session.closed {

		n, err := conn.Read(buffer)
		//设置超时
		resetTimeout(conn)
		ioBuffer.PutBytes(buffer[:n])
		if err == nil {
			GolisPackage.ReadConnData(ioBuffer, readerChannel)
		} else {
			//session已经关闭
			session.Close()
			exitChan <- true
			return
		}
		if !runnable || session.closed {
			//session关闭
			session.Close()
			exitChan <- true
			return
		}

	}

}

func resetTimeout(conn net.Conn) {
	if rwTimeout == 0 {
		conn.SetDeadline(time.Time{})
	} else {
		conn.SetDeadline(time.Now().Add(time.Duration(rwTimeout) * time.Second))
	}
}

//等待数据包
func waitData(session *Iosession, readerChannel chan *[]byte, exitChan chan bool) {
	defer func() {
		close(exitChan)
		close(readerChannel)
		if err := recover(); err != nil {
			log.Println(err)
		}
	}()
	for runnable && !session.closed {
		select {
		case data := <-readerChannel:
			readFromData(session, data)
		case <-exitChan:
			return
		}
	}
}

//从准备好的数据读取并拆包
func readFromData(session *Iosession, data *[]byte) {
	message := GolisPackage.Unpacket(*data) //拆包
	//收到消息到达时触发事件
	GolisHandler.MessageReceived(session, message)
}
<commit_msg>去掉超时操作<commit_after>package golis

import (
	"log"
	"net"
	"sync"
	"time"
)

//系统变量定义
var (
	GolisHandler IoHandler            //事件处理
	GolisPackage Packager             //拆包封包处理接口
	w            WaitGroupWrapper     //等待退出
	runnable     bool                 //服务运行状态
	rwTimeout    time.Duration    = 0 //超时单位秒
)

//定义waitGroup
type WaitGroupWrapper struct {
	sync.WaitGroup
}

//定义session
type Iosession struct {
	SesionId   uint32   //session唯一表示
	Connection net.Conn //连接
	IsAuth     bool     //是否认证成功
	closed     bool     //是否已经关闭
}

//session写入数据
func (this *Iosession) Write(message interface{}) {
	//触发消息发送事件
	GolisHandler.MessageSent(this, message)
	this.Connection.Write(GolisPackage.Packet(message))
}

//关闭连接
func (this *Iosession) Close() {
	if !this.closed {
		GolisHandler.SessionClosed(this)
		this.closed = true
	}
	this.Connection.Close()
}

//设置读写超时
func SetTimeout(timeoutSec time.Duration) {
	rwTimeout = timeoutSec
}

//拆包封包接口定义
type Packager interface {
	//读取连接数据
	//packageChan 准备好包后交给该chan
	ReadConnData(buffer *Buffer, packageChan chan<- *[]byte)
	//拆包函数
	Unpacket(data []byte) interface{}
	//封包函数
	Packet(msg interface{}) []byte
}

//事件触发接口定义
type IoHandler interface {
	//session打开
	SessionOpened(session *Iosession)
	//session关闭
	SessionClosed(session *Iosession)
	//收到消息时触发
	MessageReceived(session *Iosession, message interface{})
	//消息发送时触发
	MessageSent(session *Iosession, message interface{})
}

//服务器端运行golis
//netPro：运行协议参数，tcp/udp
//laddr ：程序监听ip和端口，如127.0.0.1:8080
func Run(netPro, laddr string) {
	runnable = true
	log.Println("golis is listen port:", laddr)

	netLis, err := net.Listen(netPro, laddr)
	if err != nil {
		log.Fatalln(err)
	}
	defer netLis.Close()
	log.Println("waiting clients...")
	for runnable {
		conn, err := netLis.Accept()
		if err != nil {
			continue
		}
		go connectHandle(conn)
	}

	w.WaitGroup.Wait()
	log.Println("golis is safe exit")
}

//停止服务
func Stop() {
	runnable = false
}

//客户端程序连接服务器
func Dial(netPro, laddr string) {
	runnable = true
	conn, err := net.Dial(netPro, laddr)
	if err != nil {
		log.Fatalln(err)
	}
	go connectHandle(conn)
}

//处理新连接
func connectHandle(conn net.Conn) {
	w.Add(1)
	//声明一个临时缓冲区，用来存储被截断的数据
	ioBuffer := NewBuffer()

	buffer := make([]byte, 512)

	//声明一个管道用于接收解包的数据
	readerChannel := make(chan *[]byte, 16)
	//创建session
	session := Iosession{Connection: conn}
	//触发sessionCreated事件
	GolisHandler.SessionOpened(&session)

	exitChan := make(chan bool)
	go waitData(&session, readerChannel, exitChan)
	defer func() {
		conn.Close()
		ioBuffer = nil
		buffer = nil
		if err := recover(); err != nil {
			log.Println(err)
		}
		w.Done()
	}()

	for runnable && !session.closed {

		n, err := conn.Read(buffer)
		//设置超时
		//resetTimeout(conn)
		ioBuffer.PutBytes(buffer[:n])
		if err == nil {
			GolisPackage.ReadConnData(ioBuffer, readerChannel)
		} else {
			//session已经关闭
			session.Close()
			exitChan <- true
			return
		}
		if !runnable || session.closed {
			//session关闭
			session.Close()
			exitChan <- true
			return
		}

	}

}

func resetTimeout(conn net.Conn) {
	if rwTimeout == 0 {
		conn.SetDeadline(time.Time{})
	} else {
		conn.SetDeadline(time.Now().Add(time.Duration(rwTimeout) * time.Second))
	}
}

//等待数据包
func waitData(session *Iosession, readerChannel chan *[]byte, exitChan chan bool) {
	defer func() {
		close(exitChan)
		close(readerChannel)
		if err := recover(); err != nil {
			log.Println(err)
		}
	}()
	for runnable && !session.closed {
		select {
		case data := <-readerChannel:
			readFromData(session, data)
		case <-exitChan:
			return
		}
	}
}

//从准备好的数据读取并拆包
func readFromData(session *Iosession, data *[]byte) {
	message := GolisPackage.Unpacket(*data) //拆包
	//收到消息到达时触发事件
	GolisHandler.MessageReceived(session, message)
}
<|endoftext|>