<commit_before>package pilosa

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"sync"
	"time"

	"github.com/gogo/protobuf/proto"
	"github.com/pilosa/pilosa/internal"
)

// Default server settings.
const (
	DefaultAntiEntropyInterval = 10 * time.Minute
	DefaultPollingInterval     = 60 * time.Second
)

// Server represents an index wrapped by a running HTTP server.
type Server struct {
	ln net.Listener

	// Close management.
	wg      sync.WaitGroup
	closing chan struct{}

	// Data storage and HTTP interface.
	Index             *Index
	Handler           *Handler
	Broadcaster       Broadcaster
	BroadcastReceiver BroadcastReceiver

	// Cluster configuration.
	// Host is replaced with actual host after opening if port is ":0".
	Host    string
	Cluster *Cluster

	// Background monitoring intervals.
	AntiEntropyInterval time.Duration
	PollingInterval     time.Duration

	LogOutput io.Writer
}

// NewServer returns a new instance of Server.
func NewServer() *Server {
	s := &Server{
		closing: make(chan struct{}),

		Index:             NewIndex(),
		Handler:           NewHandler(),
		Broadcaster:       NopBroadcaster,
		BroadcastReceiver: NopBroadcastReceiver,

		AntiEntropyInterval: DefaultAntiEntropyInterval,
		PollingInterval:     DefaultPollingInterval,

		LogOutput: os.Stderr,
	}

	s.Handler.Index = s.Index

	return s
}

// Open opens and initializes the server.
func (s *Server) Open() error {
	// Require a port in the hostname.
	host, port, err := net.SplitHostPort(s.Host)
	if err != nil {
		return err
	} else if port == "" {
		port = DefaultPort
	}

	// Open HTTP listener to determine port (if specified as :0).
	ln, err := net.Listen("tcp", ":"+port)
	if err != nil {
		return err
	}
	s.ln = ln

	// Determine hostname based on listening port.
	s.Host = net.JoinHostPort(host, strconv.Itoa(s.ln.Addr().(*net.TCPAddr).Port))

	// Create local node if no cluster is specified.
	if len(s.Cluster.Nodes) == 0 {
		s.Cluster.Nodes = []*Node{{Host: s.Host}}
	}

	// Open index.
	if err := s.Index.Open(); err != nil {
		return err
	}

	if err := s.BroadcastReceiver.Start(s); err != nil {
		return err
	}

	// Open NodeSet communication
	if err := s.Cluster.NodeSet.Open(); err != nil {
		return err
	}

	// Create executor for executing queries.
	e := NewExecutor()
	e.Index = s.Index
	e.Host = s.Host
	e.Cluster = s.Cluster

	// Initialize HTTP handler.
	s.Handler.Broadcaster = s.Broadcaster
	s.Handler.Host = s.Host
	s.Handler.Cluster = s.Cluster
	s.Handler.Executor = e
	s.Handler.LogOutput = s.LogOutput

	// Initialize Index.
	s.Index.Broadcaster = s.Broadcaster
	s.Index.LogOutput = s.LogOutput

	// Serve HTTP.
	go func() { http.Serve(ln, s.Handler) }()

	// Start background monitoring.
	s.wg.Add(2)
	go func() { defer s.wg.Done(); s.monitorAntiEntropy() }()
	go func() { defer s.wg.Done(); s.monitorMaxSlices() }()

	return nil
}

// Close closes the server and waits for it to shutdown.
func (s *Server) Close() error {
	// Notify goroutines to stop.
	close(s.closing)
	s.wg.Wait()

	if s.ln != nil {
		s.ln.Close()
	}
	if s.Index != nil {
		s.Index.Close()
	}

	return nil
}

// Addr returns the address of the listener.
func (s *Server) Addr() net.Addr {
	if s.ln == nil {
		return nil
	}
	return s.ln.Addr()
}

func (s *Server) logger() *log.Logger { return log.New(s.LogOutput, "", log.LstdFlags) }

func (s *Server) monitorAntiEntropy() {
	ticker := time.NewTicker(s.AntiEntropyInterval)
	defer ticker.Stop()

	s.logger().Printf("index sync monitor initializing (%s interval)", s.AntiEntropyInterval)

	for {
		// Wait for tick or a close.
		select {
		case <-s.closing:
			return
		case <-ticker.C:
		}

		s.logger().Printf("index sync beginning")

		// Initialize syncer with local index and remote client.
		var syncer IndexSyncer
		syncer.Index = s.Index
		syncer.Host = s.Host
		syncer.Cluster = s.Cluster
		syncer.Closing = s.closing

		// Sync indexes.
		if err := syncer.SyncIndex(); err != nil {
			s.logger().Printf("index sync error: err=%s", err)
			continue
		}

		// Record successful sync in log.
		s.logger().Printf("index sync complete")
	}
}

// monitorMaxSlices periodically pulls the highest slice from each node in the cluster.
func (s *Server) monitorMaxSlices() {
	// Ignore if only one node in the cluster.
	if len(s.Cluster.Nodes) <= 1 {
		return
	}

	ticker := time.NewTicker(s.PollingInterval)
	defer ticker.Stop()

	for {
		select {
		case <-s.closing:
			return
		case <-ticker.C:
		}

		oldmaxslices := s.Index.MaxSlices()
		for _, node := range s.Cluster.Nodes {
			if s.Host != node.Host {
				maxSlices, _ := checkMaxSlices(node.Host)
				for db, newmax := range maxSlices {
					// if we don't know about a db locally, log an error because
					// db's should be created and synced prior to slice creation
					if localdb := s.Index.DB(db); localdb != nil {
						if newmax > oldmaxslices[db] {
							oldmaxslices[db] = newmax
							localdb.SetRemoteMaxSlice(newmax)
						}
					} else {
						s.logger().Printf("Local DB not found: %s", db)
					}
				}
			}
		}
	}
}

// LocalState returns the state of the local node as well as the
// index (dbs/frames) according to the local node.
// In a gossip implementation, memberlist.Delegate.LocalState() uses this.
func (s *Server) LocalState() (proto.Message, error) {
	if s.Index == nil {
		return nil, errors.New("Server.Index is nil.")
	}
	return &internal.NodeState{
		Host:  s.Host,
		State: "OK", // TODO: make this work, pull from s.Cluster.Node
		DBs:   encodeDBs(s.Index.DBs()),
	}, nil
}

func (s *Server) ReceiveMessage(pb proto.Message) error {
	switch obj := pb.(type) {
	case *internal.CreateSliceMessage:
		d := s.Index.DB(obj.DB)
		if d == nil {
			return fmt.Errorf("Local DB not found: %s", obj.DB)
		}
		d.SetRemoteMaxSlice(obj.Slice)
	case *internal.CreateDBMessage:
		opt := DBOptions{ColumnLabel: obj.Meta.ColumnLabel}
		_, err := s.Index.CreateDB(obj.DB, opt)
		if err != nil {
			return err
		}
	case *internal.DeleteDBMessage:
		if err := s.Index.DeleteDB(obj.DB); err != nil {
			return err
		}
	case *internal.CreateFrameMessage:
		db := s.Index.DB(obj.DB)
		opt := FrameOptions{RowLabel: obj.Meta.RowLabel}
		_, err := db.CreateFrame(obj.Frame, opt)
		if err != nil {
			return err
		}
	case *internal.DeleteFrameMessage:
		db := s.Index.DB(obj.DB)
		if err := db.DeleteFrame(obj.Frame); err != nil {
			return err
		}
	}
	return nil
}

// HandleRemoteState receives incoming NodeState from remote nodes.
func (s *Server) HandleRemoteState(pb proto.Message) error {
	return s.mergeRemoteState(pb.(*internal.NodeState))
}

func (s *Server) mergeRemoteState(ns *internal.NodeState) error {
	// TODO: update some node state value in the cluster (it should be in cluster.node i guess)

	// Create databases that don't exist.
	for _, db := range ns.DBs {
		opt := DBOptions{
			ColumnLabel: db.Meta.ColumnLabel,
			TimeQuantum: TimeQuantum(db.Meta.TimeQuantum),
		}
		d, err := s.Index.CreateDBIfNotExists(db.Name, opt)
		if err != nil {
			return err
		}
		// Create frames that don't exist.
		for _, f := range db.Frames {
			opt := FrameOptions{
				RowLabel:    f.Meta.RowLabel,
				TimeQuantum: TimeQuantum(f.Meta.TimeQuantum),
				CacheSize:   f.Meta.CacheSize,
			}
			_, err := d.CreateFrameIfNotExists(f.Name, opt)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func checkMaxSlices(hostport string) (map[string]uint64, error) {
	// Create HTTP request.
	req, err := http.NewRequest("GET", (&url.URL{
		Scheme: "http",
		Host:   hostport,
		Path:   "/slices/max",
	}).String(), nil)

	if err != nil {
		return nil, err
	}

	// Require protobuf encoding.
	req.Header.Set("Accept", "application/x-protobuf")
	req.Header.Set("Content-Type", "application/x-protobuf")

	// Send request to remote node.
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	// Read response into buffer.
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	// Check status code.
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("invalid status: code=%d, err=%s", resp.StatusCode, body)
	}

	// Decode response object.
	pb := internal.MaxSlicesResponse{}

	if err = proto.Unmarshal(body, &pb); err != nil {
		return nil, err
	}

	return pb.MaxSlices, nil
}
<commit_msg>group Server interface implementation function together<commit_after>package pilosa

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"sync"
	"time"

	"github.com/gogo/protobuf/proto"
	"github.com/pilosa/pilosa/internal"
)

// Default server settings.
const (
	DefaultAntiEntropyInterval = 10 * time.Minute
	DefaultPollingInterval     = 60 * time.Second
)

// Server represents an index wrapped by a running HTTP server.
type Server struct {
	ln net.Listener

	// Close management.
	wg      sync.WaitGroup
	closing chan struct{}

	// Data storage and HTTP interface.
	Index             *Index
	Handler           *Handler
	Broadcaster       Broadcaster
	BroadcastReceiver BroadcastReceiver

	// Cluster configuration.
	// Host is replaced with actual host after opening if port is ":0".
	Host    string
	Cluster *Cluster

	// Background monitoring intervals.
	AntiEntropyInterval time.Duration
	PollingInterval     time.Duration

	LogOutput io.Writer
}

// NewServer returns a new instance of Server.
func NewServer() *Server {
	s := &Server{
		closing: make(chan struct{}),

		Index:             NewIndex(),
		Handler:           NewHandler(),
		Broadcaster:       NopBroadcaster,
		BroadcastReceiver: NopBroadcastReceiver,

		AntiEntropyInterval: DefaultAntiEntropyInterval,
		PollingInterval:     DefaultPollingInterval,

		LogOutput: os.Stderr,
	}

	s.Handler.Index = s.Index

	return s
}

// Open opens and initializes the server.
func (s *Server) Open() error {
	// Require a port in the hostname.
	host, port, err := net.SplitHostPort(s.Host)
	if err != nil {
		return err
	} else if port == "" {
		port = DefaultPort
	}

	// Open HTTP listener to determine port (if specified as :0).
	ln, err := net.Listen("tcp", ":"+port)
	if err != nil {
		return err
	}
	s.ln = ln

	// Determine hostname based on listening port.
	s.Host = net.JoinHostPort(host, strconv.Itoa(s.ln.Addr().(*net.TCPAddr).Port))

	// Create local node if no cluster is specified.
	if len(s.Cluster.Nodes) == 0 {
		s.Cluster.Nodes = []*Node{{Host: s.Host}}
	}

	// Open index.
	if err := s.Index.Open(); err != nil {
		return err
	}

	if err := s.BroadcastReceiver.Start(s); err != nil {
		return err
	}

	// Open NodeSet communication
	if err := s.Cluster.NodeSet.Open(); err != nil {
		return err
	}

	// Create executor for executing queries.
	e := NewExecutor()
	e.Index = s.Index
	e.Host = s.Host
	e.Cluster = s.Cluster

	// Initialize HTTP handler.
	s.Handler.Broadcaster = s.Broadcaster
	s.Handler.Host = s.Host
	s.Handler.Cluster = s.Cluster
	s.Handler.Executor = e
	s.Handler.LogOutput = s.LogOutput

	// Initialize Index.
	s.Index.Broadcaster = s.Broadcaster
	s.Index.LogOutput = s.LogOutput

	// Serve HTTP.
	go func() { http.Serve(ln, s.Handler) }()

	// Start background monitoring.
	s.wg.Add(2)
	go func() { defer s.wg.Done(); s.monitorAntiEntropy() }()
	go func() { defer s.wg.Done(); s.monitorMaxSlices() }()

	return nil
}

// Close closes the server and waits for it to shutdown.
func (s *Server) Close() error {
	// Notify goroutines to stop.
	close(s.closing)
	s.wg.Wait()

	if s.ln != nil {
		s.ln.Close()
	}
	if s.Index != nil {
		s.Index.Close()
	}

	return nil
}

// Addr returns the address of the listener.
func (s *Server) Addr() net.Addr {
	if s.ln == nil {
		return nil
	}
	return s.ln.Addr()
}

func (s *Server) logger() *log.Logger { return log.New(s.LogOutput, "", log.LstdFlags) }

func (s *Server) monitorAntiEntropy() {
	ticker := time.NewTicker(s.AntiEntropyInterval)
	defer ticker.Stop()

	s.logger().Printf("index sync monitor initializing (%s interval)", s.AntiEntropyInterval)

	for {
		// Wait for tick or a close.
		select {
		case <-s.closing:
			return
		case <-ticker.C:
		}

		s.logger().Printf("index sync beginning")

		// Initialize syncer with local index and remote client.
		var syncer IndexSyncer
		syncer.Index = s.Index
		syncer.Host = s.Host
		syncer.Cluster = s.Cluster
		syncer.Closing = s.closing

		// Sync indexes.
		if err := syncer.SyncIndex(); err != nil {
			s.logger().Printf("index sync error: err=%s", err)
			continue
		}

		// Record successful sync in log.
		s.logger().Printf("index sync complete")
	}
}

// monitorMaxSlices periodically pulls the highest slice from each node in the cluster.
func (s *Server) monitorMaxSlices() {
	// Ignore if only one node in the cluster.
	if len(s.Cluster.Nodes) <= 1 {
		return
	}

	ticker := time.NewTicker(s.PollingInterval)
	defer ticker.Stop()

	for {
		select {
		case <-s.closing:
			return
		case <-ticker.C:
		}

		oldmaxslices := s.Index.MaxSlices()
		for _, node := range s.Cluster.Nodes {
			if s.Host != node.Host {
				maxSlices, _ := checkMaxSlices(node.Host)
				for db, newmax := range maxSlices {
					// if we don't know about a db locally, log an error because
					// db's should be created and synced prior to slice creation
					if localdb := s.Index.DB(db); localdb != nil {
						if newmax > oldmaxslices[db] {
							oldmaxslices[db] = newmax
							localdb.SetRemoteMaxSlice(newmax)
						}
					} else {
						s.logger().Printf("Local DB not found: %s", db)
					}
				}
			}
		}
	}
}

// ReceiveMessage represents an implementation of BroadcastHandler.
func (s *Server) ReceiveMessage(pb proto.Message) error {
	switch obj := pb.(type) {
	case *internal.CreateSliceMessage:
		d := s.Index.DB(obj.DB)
		if d == nil {
			return fmt.Errorf("Local DB not found: %s", obj.DB)
		}
		d.SetRemoteMaxSlice(obj.Slice)
	case *internal.CreateDBMessage:
		opt := DBOptions{ColumnLabel: obj.Meta.ColumnLabel}
		_, err := s.Index.CreateDB(obj.DB, opt)
		if err != nil {
			return err
		}
	case *internal.DeleteDBMessage:
		if err := s.Index.DeleteDB(obj.DB); err != nil {
			return err
		}
	case *internal.CreateFrameMessage:
		db := s.Index.DB(obj.DB)
		opt := FrameOptions{RowLabel: obj.Meta.RowLabel}
		_, err := db.CreateFrame(obj.Frame, opt)
		if err != nil {
			return err
		}
	case *internal.DeleteFrameMessage:
		db := s.Index.DB(obj.DB)
		if err := db.DeleteFrame(obj.Frame); err != nil {
			return err
		}
	}
	return nil
}

// Server implements gossip.StateHandler.
// LocalState returns the state of the local node as well as the
// index (dbs/frames) according to the local node.
// In a gossip implementation, memberlist.Delegate.LocalState() uses this.
func (s *Server) LocalState() (proto.Message, error) {
	if s.Index == nil {
		return nil, errors.New("Server.Index is nil.")
	}
	return &internal.NodeState{
		Host:  s.Host,
		State: "OK", // TODO: make this work, pull from s.Cluster.Node
		DBs:   encodeDBs(s.Index.DBs()),
	}, nil
}

// HandleRemoteState receives incoming NodeState from remote nodes.
func (s *Server) HandleRemoteState(pb proto.Message) error {
	return s.mergeRemoteState(pb.(*internal.NodeState))
}

func (s *Server) mergeRemoteState(ns *internal.NodeState) error {
	// TODO: update some node state value in the cluster (it should be in cluster.node i guess)

	// Create databases that don't exist.
	for _, db := range ns.DBs {
		opt := DBOptions{
			ColumnLabel: db.Meta.ColumnLabel,
			TimeQuantum: TimeQuantum(db.Meta.TimeQuantum),
		}
		d, err := s.Index.CreateDBIfNotExists(db.Name, opt)
		if err != nil {
			return err
		}
		// Create frames that don't exist.
		for _, f := range db.Frames {
			opt := FrameOptions{
				RowLabel:    f.Meta.RowLabel,
				TimeQuantum: TimeQuantum(f.Meta.TimeQuantum),
				CacheSize:   f.Meta.CacheSize,
			}
			_, err := d.CreateFrameIfNotExists(f.Name, opt)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func checkMaxSlices(hostport string) (map[string]uint64, error) {
	// Create HTTP request.
	req, err := http.NewRequest("GET", (&url.URL{
		Scheme: "http",
		Host:   hostport,
		Path:   "/slices/max",
	}).String(), nil)

	if err != nil {
		return nil, err
	}

	// Require protobuf encoding.
	req.Header.Set("Accept", "application/x-protobuf")
	req.Header.Set("Content-Type", "application/x-protobuf")

	// Send request to remote node.
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	// Read response into buffer.
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	// Check status code.
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("invalid status: code=%d, err=%s", resp.StatusCode, body)
	}

	// Decode response object.
	pb := internal.MaxSlicesResponse{}

	if err = proto.Unmarshal(body, &pb); err != nil {
		return nil, err
	}

	return pb.MaxSlices, nil
}
<|endoftext|>


<commit_before>// Package main provides a simple http server that can store a file and serve it
// for download once before deleting it.
package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strconv"

	"github.com/nu7hatch/gouuid"
)

// main parses a single port flag, sets up the routes, and starts the server on
// the specified port or 1111 by default.
func main() {
	var port = flag.Int("p", 1111, "port on which the server should start")
	flag.Parse()

	http.HandleFunc("/new", HostFile)
	http.HandleFunc("/", ServeFile)
	http.ListenAndServe(":"+strconv.Itoa(*port), nil)
}

// HostFile expects a POST with a file. It grabs the file's contents, generates
// an id, saves the file locally as that id, then returns the id.
func HostFile(res http.ResponseWriter, req *http.Request) {
	if req.Method != "POST" {
		fourohfour(res)
		return
	}

	contents := getFileContents(req)

	id, err := uuid.NewV4()
	if err != nil {
		fourohfour(res)
		log.Fatal(err)
	}

	file, err := createFile(id.String())
	if err != nil {
		fourohfour(res)
		log.Fatal(err)
	}

	file.Write(contents)

	fmt.Fprint(res, id.String())
}

// ServeFile catches all other requests. It expects a "file" param in the
// request body, specifying an id. It searches for a file named with that id,
// and if it exists, serves that file then deletes it. If not, 404.
func ServeFile(res http.ResponseWriter, req *http.Request) {
	params, err := getFilename(req)
	if err != nil {
		fourohfour(res)
		return
	}

	dirname, err := dirname()
	if err != nil {
		fourohfour(res)
		log.Fatal(err)
	}

	fPath := filepath.Join(dirname, "files", params.File+".tar.gz")
	content, err := ioutil.ReadFile(fPath)
	if err != nil {
		fourohfour(res)
		return
	}

	fmt.Fprint(res, string(content))

	if err = os.Remove(fPath); err != nil {
		log.Fatal(err)
	}
}

// fourohfour writes a 404 response to a passed in http.ResponseWriter.
func fourohfour(res http.ResponseWriter) {
	res.WriteHeader(404)
	fmt.Fprint(res, "not found")
}

// getFileContents reads the given request body and returns it as a byte slice.
func getFileContents(req *http.Request) []byte {
	buf := new(bytes.Buffer)
	buf.ReadFrom(req.Body)
	return buf.Bytes()
}

// params is just a struct used to return from the getFilename function below.
type params struct {
	File string
}

// getFilename extracts the request body and pull the "file" param into a struct
// containing the file name requested as a string.
func getFilename(req *http.Request) (p params, err error) {
	content, err := ioutil.ReadAll(req.Body)
	if err != nil {
		return
	}

	err = json.Unmarshal(content, &p)
	return
}

// createFile creates a blank file using the given name at ./files/NAME.tar.gz.
func createFile(id string) (file *os.File, err error) {
	dirname, err := dirname()
	if err != nil {
		return
	}

	file, err = os.Create(filepath.Join(dirname, "files", id+".tar.gz"))

	return
}

// dirname gets the name of the directory this file is in.
func dirname() (dirname string, err error) {
	_, filename, _, _ := runtime.Caller(1)
	dirname, err = filepath.Abs(filepath.Dir(filename))
	return
}
<commit_msg>drop file param, request id directly<commit_after>// Package main provides a simple http server that can store a file and serve it
// for download once before deleting it.
package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strconv"

	"github.com/nu7hatch/gouuid"
)

// main parses a single port flag, sets up the routes, and starts the server on
// the specified port or 1111 by default.
func main() {
	var port = flag.Int("p", 1111, "port on which the server should start")
	flag.Parse()

	http.HandleFunc("/new", HostFile)
	http.HandleFunc("/", ServeFile)
	http.ListenAndServe(":"+strconv.Itoa(*port), nil)
}

// HostFile expects a POST with a file. It grabs the file's contents, generates
// an id, saves the file locally as that id, then returns the id.
func HostFile(res http.ResponseWriter, req *http.Request) {
	if req.Method != "POST" {
		fourohfour(res)
		return
	}

	contents := getFileContents(req)

	id, err := uuid.NewV4()
	if err != nil {
		fourohfour(res)
		log.Fatal(err)
	}

	file, err := createFile(id.String())
	if err != nil {
		fourohfour(res)
		log.Fatal(err)
	}

	file.Write(contents)

	fmt.Fprint(res, id.String())
}

// ServeFile catches all other requests. It expects a "file" param in the
// request body, specifying an id. It searches for a file named with that id,
// and if it exists, serves that file then deletes it. If not, 404.
func ServeFile(res http.ResponseWriter, req *http.Request) {
	dirname, err := dirname()
	if err != nil {
		fourohfour(res)
		log.Fatal(err)
	}

	fPath := filepath.Join(dirname, "files", req.URL.String()+".tar")
	content, err := ioutil.ReadFile(fPath)
	if err != nil {
		fourohfour(res)
		return
	}

	fmt.Fprint(res, string(content))

	if err = os.Remove(fPath); err != nil {
		log.Fatal(err)
	}
}

// fourohfour writes a 404 response to a passed in http.ResponseWriter.
func fourohfour(res http.ResponseWriter) {
	res.WriteHeader(404)
	fmt.Fprint(res, "not found")
}

// getFileContents reads the given request body and returns it as a byte slice.
func getFileContents(req *http.Request) []byte {
	buf := new(bytes.Buffer)
	buf.ReadFrom(req.Body)
	return buf.Bytes()
}

// params is just a struct used to return from the getFilename function below.
type params struct {
	File string
}

// createFile creates a blank file using the given name at ./files/NAME.tar.gz.
func createFile(id string) (file *os.File, err error) {
	dirname, err := dirname()
	if err != nil {
		return
	}

	file, err = os.Create(filepath.Join(dirname, "files", id+".tar"))

	return
}

// dirname gets the name of the directory this file is in.
func dirname() (dirname string, err error) {
	_, filename, _, _ := runtime.Caller(1)
	dirname, err = filepath.Abs(filepath.Dir(filename))
	return
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/clawio/service.auth/lib"
	pb "github.com/clawio/service.localstore.meta/proto"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"mime"
	"os"
	"path"
	"strings"
)

const (
	dirPerm = 0755
)

var (
	unauthenticatedError = grpc.Errorf(codes.Unauthenticated, "identity not found")
	permissionDenied     = grpc.Errorf(codes.PermissionDenied, "access denied")
)

type newServerParams struct {
	dataDir      string
	tmpDir       string
	sharedSecret string
}

func newServer(p *newServerParams) *server {
	return &server{p}
}

type server struct {
	p *newServerParams
}

func (s *server) Home(ctx context.Context, req *pb.HomeReq) (*pb.Void, error) {

	idt, err := lib.ParseToken(req.AccessToken, s.p.sharedSecret)
	if err != nil {
		log.Error(err)
		return &pb.Void{}, unauthenticatedError
	}

	log.Infof("%s", idt)

	home := getHome(idt)

	log.Infof("home is %s", home)

	pp := s.getPhysicalPath(home)

	log.Infof("physical path is %s", pp)

	_, err = os.Stat(pp)

	// Create home dir if not exists
	if os.IsNotExist(err) {

		log.Infof("home does not exist")

		err = os.MkdirAll(pp, dirPerm)
		if err != nil {
			log.Error(err)
			return nil, err
		}

		log.Infof("home created")

		return &pb.Void{}, nil
	}

	if err != nil {
		log.Error(err)
		return nil, err
	}

	log.Infof("home already created")

	return &pb.Void{}, nil
}

func (s *server) Mkdir(ctx context.Context, req *pb.MkdirReq) (*pb.Void, error) {

	idt, err := lib.ParseToken(req.AccessToken, s.p.sharedSecret)
	if err != nil {
		log.Error(err)
		return &pb.Void{}, unauthenticatedError
	}

	log.Infof("%s", idt)

	p := path.Clean(req.Path)

	log.Infof("path is %s", p)

	if !isUnderHome(p, idt) {
		log.Error(permissionDenied)
		return &pb.Void{}, permissionDenied
	}

	if p == getHome(idt) {
		return &pb.Void{}, grpc.Errorf(codes.PermissionDenied, "cannot create directory")
	}

	pp := s.getPhysicalPath(p)

	log.Infof("physical path is %s", pp)

	err = os.Mkdir(pp, dirPerm)
	if err != nil {
		log.Error(err)
		return &pb.Void{}, err
	}

	log.Infof("created dir %s", pp)

	return &pb.Void{}, nil
}

func (s *server) Stat(ctx context.Context, req *pb.StatReq) (*pb.Metadata, error) {

	idt, err := lib.ParseToken(req.AccessToken, s.p.sharedSecret)
	if err != nil {
		log.Error(err)
		return &pb.Metadata{}, unauthenticatedError
	}

	log.Infof("%s", s)

	p := path.Clean(req.Path)

	log.Infof("path is %s", p)

	if !isUnderHome(p, idt) {
		log.Error(permissionDenied)
		return &pb.Metadata{}, permissionDenied
	}

	pp := s.getPhysicalPath(p)

	log.Infof("physical path is %s", pp)

	parentMeta, err := s.getMeta(pp)
	if err != nil {
		log.Error(err)
		return &pb.Metadata{}, err
	}

	log.Infof("stated parent %s", pp)

	if !parentMeta.IsContainer || req.Children == false {
		return parentMeta, nil
	}

	dir, err := os.Open(pp)
	if err != nil {
		log.Error(err)
		return &pb.Metadata{}, err
	}

	log.Infof("opened dir %s", pp)

	defer dir.Close()

	names, err := dir.Readdirnames(0)
	if err != nil {
		log.Error(err)
		return &pb.Metadata{}, err
	}

	log.Infof("dir %s has %d entries", pp, len(names))

	for _, n := range names {
		cpp := s.getPhysicalPath(path.Join(parentMeta.Path, path.Clean(n)))
		m, err := s.getMeta(cpp)
		if err != nil {
			log.Error(err)
		} else {
			parentMeta.Children = append(parentMeta.Children, m)
		}

	}

	log.Infof("added %d entries to parent", len(parentMeta.Children))

	return parentMeta, nil
}

func (s *server) Cp(ctx context.Context, req *pb.CpReq) (*pb.Void, error) {

	idt, err := lib.ParseToken(req.AccessToken, s.p.sharedSecret)
	if err != nil {
		log.Error(err)
		return &pb.Void{}, unauthenticatedError
	}

	log.Infof("%s", idt)

	src := path.Clean(req.Src)
	dst := path.Clean(req.Dst)

	log.Infof("src is %s", src)
	log.Info("dst is %s", dst)

	if !isUnderHome(src, idt) {
		log.Error(permissionDenied)
		return &pb.Void{}, permissionDenied
	}

	if !isUnderHome(dst, idt) {
		log.Error(permissionDenied)
		return &pb.Void{}, permissionDenied
	}

	if src == getHome(idt) || dst == getHome(idt) {
		return &pb.Void{}, grpc.Errorf(codes.PermissionDenied, "cannot copy from/to home directory")
	}

	psrc := s.getPhysicalPath(src)
	pdst := s.getPhysicalPath(dst)

	log.Infof("physical src is %s", psrc)
	log.Infof("physical dst is %s", pdst)

	statReq := &pb.StatReq{}
	statReq.AccessToken = req.AccessToken
	statReq.Path = req.Src

	meta, err := s.Stat(ctx, statReq)
	if err != nil {
		log.Error(err)
		return &pb.Void{}, err
	}

	log.Infof("stated %s", src)

	if meta.IsContainer {
		err = copyDir(psrc, pdst)
		if err != nil {
			return &pb.Void{}, err
		}

		log.Infof("copied from dir %s to dir %s", psrc, pdst)
	}

	err = copyFile(psrc, pdst, int64(meta.Size))
	if err != nil {
		return &pb.Void{}, err
	}

	log.Infof("copied from file %s to file %s", psrc, pdst)

	return &pb.Void{}, nil
}

func (s *server) Mv(ctx context.Context, req *pb.MvReq) (*pb.Void, error) {

	idt, err := lib.ParseToken(req.AccessToken, s.p.sharedSecret)
	if err != nil {
		log.Error(err)
		return &pb.Void{}, unauthenticatedError
	}

	log.Infof("%s", idt)

	src := path.Clean(req.Src)
	dst := path.Clean(req.Dst)

	log.Infof("src is %s", src)
	log.Info("dst is %s", dst)

	if isUnderHome(src, idt) {
		log.Error(permissionDenied)
		return &pb.Void{}, permissionDenied
	}

	if !isUnderHome(dst, idt) {
		log.Error(permissionDenied)
		return &pb.Void{}, permissionDenied
	}

	if src == getHome(idt) || dst == getHome(idt) {
		return &pb.Void{}, grpc.Errorf(codes.PermissionDenied, "cannot rename from/to home directory")
	}

	psrc := s.getPhysicalPath(src)
	pdst := s.getPhysicalPath(dst)

	log.Infof("physical src is %s", psrc)
	log.Infof("physical dst is %s", pdst)

	err = os.Rename(psrc, pdst)
	if err != nil {
		log.Error(err)
		return &pb.Void{}, err
	}

	log.Infof("renamed from %s to %s", psrc, pdst)

	return &pb.Void{}, nil
}

func (s *server) Rm(ctx context.Context, req *pb.RmReq) (*pb.Void, error) {

	idt, err := lib.ParseToken(req.AccessToken, s.p.sharedSecret)
	if err != nil {
		log.Error(err)
		return &pb.Void{}, unauthenticatedError
	}

	log.Infof("%s", idt)

	p := path.Clean(req.Path)

	log.Infof("path is %s", p)

	if !isUnderHome(p, idt) {
		log.Error(permissionDenied)
		return &pb.Void{}, permissionDenied
	}

	if p == getHome(idt) {
		return &pb.Void{}, grpc.Errorf(codes.PermissionDenied, "cannot remove home directory")
	}

	pp := s.getPhysicalPath(p)

	log.Infof("physical path is %s", pp)

	err = os.Remove(pp)
	if err != nil {
		log.Error(err)
		return &pb.Void{}, err
	}

	log.Infof("removed %s", pp)

	return &pb.Void{}, nil
}

// getMeta return the metadata of path pp.
// p is the physical path and should never be exposed to clients.
func (s *server) getMeta(pp string) (*pb.Metadata, error) {

	finfo, err := os.Stat(pp)
	if err != nil {
		return &pb.Metadata{}, err
	}

	logicalPath := s.getLogicalPath(pp)

	m := &pb.Metadata{}
	m.Id = "TODO"
	m.Path = logicalPath
	m.Size = uint32(finfo.Size())
	m.IsContainer = finfo.IsDir()
	m.Modified = uint32(finfo.ModTime().Unix())
	m.Etag = "TODO"
	m.Permissions = 0
	m.MimeType = mime.TypeByExtension(path.Ext(m.Path))

	if m.MimeType == "" {
		m.MimeType = "application/octet-stream"
	}

	if m.IsContainer {
		m.MimeType = "inode/container"
	}

	return m, nil
}

func (s *server) getPhysicalPath(p string) string {
	return path.Join(s.p.dataDir, path.Clean(p))
}

func (s *server) getLogicalPath(pp string) string {
	return path.Clean(strings.TrimPrefix(pp, s.p.dataDir))
}
<commit_msg>Added more dev info<commit_after>package main

import (
	"github.com/clawio/service.auth/lib"
	pb "github.com/clawio/service.localstore.meta/proto"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"mime"
	"os"
	"path"
	"strings"
)

const (
	dirPerm = 0755
)

var (
	unauthenticatedError = grpc.Errorf(codes.Unauthenticated, "identity not found")
	permissionDenied     = grpc.Errorf(codes.PermissionDenied, "access denied")
)

type newServerParams struct {
	dataDir      string
	tmpDir       string
	sharedSecret string
}

func newServer(p *newServerParams) *server {
	return &server{p}
}

type server struct {
	p *newServerParams
}

// TODO(labkode) ask service.localstore.prop to propagate info.
func (s *server) Home(ctx context.Context, req *pb.HomeReq) (*pb.Void, error) {

	idt, err := lib.ParseToken(req.AccessToken, s.p.sharedSecret)
	if err != nil {
		log.Error(err)
		return &pb.Void{}, unauthenticatedError
	}

	log.Infof("%s", idt)

	home := getHome(idt)

	log.Infof("home is %s", home)

	pp := s.getPhysicalPath(home)

	log.Infof("physical path is %s", pp)

	_, err = os.Stat(pp)

	// Create home dir if not exists
	if os.IsNotExist(err) {

		log.Infof("home does not exist")

		err = os.MkdirAll(pp, dirPerm)
		if err != nil {
			log.Error(err)
			return nil, err
		}

		log.Infof("home created")

		return &pb.Void{}, nil
	}

	if err != nil {
		log.Error(err)
		return nil, err
	}

	log.Infof("home already created")

	return &pb.Void{}, nil
}

//TODO(labkode) ask service.localstore.prop for put dir info
func (s *server) Mkdir(ctx context.Context, req *pb.MkdirReq) (*pb.Void, error) {

	idt, err := lib.ParseToken(req.AccessToken, s.p.sharedSecret)
	if err != nil {
		log.Error(err)
		return &pb.Void{}, unauthenticatedError
	}

	log.Infof("%s", idt)

	p := path.Clean(req.Path)

	log.Infof("path is %s", p)

	if !isUnderHome(p, idt) {
		log.Error(permissionDenied)
		return &pb.Void{}, permissionDenied
	}

	if p == getHome(idt) {
		return &pb.Void{}, grpc.Errorf(codes.PermissionDenied, "cannot create directory")
	}

	pp := s.getPhysicalPath(p)

	log.Infof("physical path is %s", pp)

	err = os.Mkdir(pp, dirPerm)
	if err != nil {
		log.Error(err)
		return &pb.Void{}, err
	}

	log.Infof("created dir %s", pp)

	return &pb.Void{}, nil
}

// TODO(labkode) ask service.localstore.prop for propagation info
func (s *server) Stat(ctx context.Context, req *pb.StatReq) (*pb.Metadata, error) {

	idt, err := lib.ParseToken(req.AccessToken, s.p.sharedSecret)
	if err != nil {
		log.Error(err)
		return &pb.Metadata{}, unauthenticatedError
	}

	log.Infof("%s", idt)

	p := path.Clean(req.Path)

	log.Infof("path is %s", p)

	if !isUnderHome(p, idt) {
		log.Error(permissionDenied)
		return &pb.Metadata{}, permissionDenied
	}

	pp := s.getPhysicalPath(p)

	log.Infof("physical path is %s", pp)

	parentMeta, err := s.getMeta(pp)
	if err != nil {
		log.Error(err)
		return &pb.Metadata{}, err
	}

	log.Infof("stated parent %s", pp)

	if !parentMeta.IsContainer || req.Children == false {
		return parentMeta, nil
	}

	dir, err := os.Open(pp)
	if err != nil {
		log.Error(err)
		return &pb.Metadata{}, err
	}

	log.Infof("opened dir %s", pp)

	defer dir.Close()

	names, err := dir.Readdirnames(0)
	if err != nil {
		log.Error(err)
		return &pb.Metadata{}, err
	}

	log.Infof("dir %s has %d entries", pp, len(names))

	for _, n := range names {
		cpp := s.getPhysicalPath(path.Join(parentMeta.Path, path.Clean(n)))
		m, err := s.getMeta(cpp)
		if err != nil {
			log.Error(err)
		} else {
			parentMeta.Children = append(parentMeta.Children, m)
		}

	}

	log.Infof("added %d entries to parent", len(parentMeta.Children))

	return parentMeta, nil
}

// TODO(labkode) ask service.localstore.prop to update metadata of new resources
func (s *server) Cp(ctx context.Context, req *pb.CpReq) (*pb.Void, error) {

	idt, err := lib.ParseToken(req.AccessToken, s.p.sharedSecret)
	if err != nil {
		log.Error(err)
		return &pb.Void{}, unauthenticatedError
	}

	log.Infof("%s", idt)

	src := path.Clean(req.Src)
	dst := path.Clean(req.Dst)

	log.Infof("src is %s", src)
	log.Info("dst is %s", dst)

	if !isUnderHome(src, idt) {
		log.Error(permissionDenied)
		return &pb.Void{}, permissionDenied
	}

	if !isUnderHome(dst, idt) {
		log.Error(permissionDenied)
		return &pb.Void{}, permissionDenied
	}

	if src == getHome(idt) || dst == getHome(idt) {
		return &pb.Void{}, grpc.Errorf(codes.PermissionDenied, "cannot copy from/to home directory")
	}

	psrc := s.getPhysicalPath(src)
	pdst := s.getPhysicalPath(dst)

	log.Infof("physical src is %s", psrc)
	log.Infof("physical dst is %s", pdst)

	statReq := &pb.StatReq{}
	statReq.AccessToken = req.AccessToken
	statReq.Path = req.Src

	meta, err := s.Stat(ctx, statReq)
	if err != nil {
		log.Error(err)
		return &pb.Void{}, err
	}

	log.Infof("stated %s", src)

	if meta.IsContainer {
		err = copyDir(psrc, pdst)
		if err != nil {
			return &pb.Void{}, err
		}

		log.Infof("copied from dir %s to dir %s", psrc, pdst)
	}

	err = copyFile(psrc, pdst, int64(meta.Size))
	if err != nil {
		return &pb.Void{}, err
	}

	log.Infof("copied from file %s to file %s", psrc, pdst)

	return &pb.Void{}, nil
}

// TODO(labkode) ask service.localstore.prop to mv metadata.
func (s *server) Mv(ctx context.Context, req *pb.MvReq) (*pb.Void, error) {

	idt, err := lib.ParseToken(req.AccessToken, s.p.sharedSecret)
	if err != nil {
		log.Error(err)
		return &pb.Void{}, unauthenticatedError
	}

	log.Infof("%s", idt)

	src := path.Clean(req.Src)
	dst := path.Clean(req.Dst)

	log.Infof("src is %s", src)
	log.Info("dst is %s", dst)

	if isUnderHome(src, idt) {
		log.Error(permissionDenied)
		return &pb.Void{}, permissionDenied
	}

	if !isUnderHome(dst, idt) {
		log.Error(permissionDenied)
		return &pb.Void{}, permissionDenied
	}

	if src == getHome(idt) || dst == getHome(idt) {
		return &pb.Void{}, grpc.Errorf(codes.PermissionDenied, "cannot rename from/to home directory")
	}

	psrc := s.getPhysicalPath(src)
	pdst := s.getPhysicalPath(dst)

	log.Infof("physical src is %s", psrc)
	log.Infof("physical dst is %s", pdst)

	err = os.Rename(psrc, pdst)
	if err != nil {
		log.Error(err)
		return &pb.Void{}, err
	}

	log.Infof("renamed from %s to %s", psrc, pdst)

	return &pb.Void{}, nil
}

// TODO(labkode) ask service.localstore.prop to remove
func (s *server) Rm(ctx context.Context, req *pb.RmReq) (*pb.Void, error) {

	idt, err := lib.ParseToken(req.AccessToken, s.p.sharedSecret)
	if err != nil {
		log.Error(err)
		return &pb.Void{}, unauthenticatedError
	}

	log.Infof("%s", idt)

	p := path.Clean(req.Path)

	log.Infof("path is %s", p)

	if !isUnderHome(p, idt) {
		log.Error(permissionDenied)
		return &pb.Void{}, permissionDenied
	}

	if p == getHome(idt) {
		return &pb.Void{}, grpc.Errorf(codes.PermissionDenied, "cannot remove home directory")
	}

	pp := s.getPhysicalPath(p)

	log.Infof("physical path is %s", pp)

	err = os.Remove(pp)
	if err != nil {
		log.Error(err)
		return &pb.Void{}, err
	}

	log.Infof("removed %s", pp)

	return &pb.Void{}, nil
}

// getMeta return the metadata of path pp.
// p is the physical path and should never be exposed to clients.
// TODO(labkode) ask service.localstore.prop for id, mtime and etag.
func (s *server) getMeta(pp string) (*pb.Metadata, error) {

	finfo, err := os.Stat(pp)
	if err != nil {
		return &pb.Metadata{}, err
	}

	logicalPath := s.getLogicalPath(pp)

	m := &pb.Metadata{}
	m.Id = "TODO"
	m.Path = logicalPath
	m.Size = uint32(finfo.Size())
	m.IsContainer = finfo.IsDir()
	m.Modified = uint32(finfo.ModTime().Unix())
	m.Etag = "TODO"
	m.Permissions = 0
	m.MimeType = mime.TypeByExtension(path.Ext(m.Path))

	if m.MimeType == "" {
		m.MimeType = "application/octet-stream"
	}

	if m.IsContainer {
		m.MimeType = "inode/container"
	}

	return m, nil
}

func (s *server) getPhysicalPath(p string) string {
	return path.Join(s.p.dataDir, path.Clean(p))
}

func (s *server) getLogicalPath(pp string) string {
	return path.Clean(strings.TrimPrefix(pp, s.p.dataDir))
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"go/build"
	"log"
	"net"
	"net/rpc"
	"os"
	"path/filepath"
	"reflect"
	"runtime"
)

func do_server() int {
	g_config.read()
	if g_config.ForceDebugOutput != "" {
		// forcefully enable debugging and redirect logging into the
		// specified file
		*g_debug = true
		f, err := os.Create(g_config.ForceDebugOutput)
		if err != nil {
			panic(err)
		}
		log.SetOutput(f)
	}

	addr := *g_addr
	if *g_sock == "unix" {
		addr = get_socket_filename()
		if file_exists(addr) {
			log.Printf("unix socket: '%s' already exists\n", addr)
			return 1
		}
	}
	g_daemon = new_daemon(*g_sock, addr)
	if *g_sock == "unix" {
		// cleanup unix socket file
		defer os.Remove(addr)
	}

	rpc.Register(new(RPC))

	g_daemon.loop()
	return 0
}

//-------------------------------------------------------------------------
// daemon
//-------------------------------------------------------------------------

type daemon struct {
	listener     net.Listener
	cmd_in       chan int
	autocomplete *auto_complete_context
	pkgcache     package_cache
	declcache    *decl_cache
	context      package_lookup_context
}

func new_daemon(network, address string) *daemon {
	var err error

	d := new(daemon)
	d.listener, err = net.Listen(network, address)
	if err != nil {
		panic(err)
	}

	d.cmd_in = make(chan int, 1)
	d.pkgcache = new_package_cache()
	d.declcache = new_decl_cache(&d.context)
	d.autocomplete = new_auto_complete_context(d.pkgcache, d.declcache)
	return d
}

func (this *daemon) drop_cache() {
	this.pkgcache = new_package_cache()
	this.declcache = new_decl_cache(&this.context)
	this.autocomplete = new_auto_complete_context(this.pkgcache, this.declcache)
}

const (
	daemon_close = iota
)

func (this *daemon) loop() {
	conn_in := make(chan net.Conn)
	go func() {
		for {
			c, err := this.listener.Accept()
			if err != nil {
				panic(err)
			}
			conn_in <- c
		}
	}()
	for {
		// handle connections or server CMDs (currently one CMD)
		select {
		case c := <-conn_in:
			rpc.ServeConn(c)
			runtime.GC()
		case cmd := <-this.cmd_in:
			switch cmd {
			case daemon_close:
				return
			}
		}
	}
}

func (this *daemon) close() {
	this.cmd_in <- daemon_close
}

var g_daemon *daemon

//-------------------------------------------------------------------------
// server_* functions
//
// Corresponding client_* functions are autogenerated by goremote.
//-------------------------------------------------------------------------

func server_auto_complete(file []byte, filename string, cursor int, context_packed go_build_context) (c []candidate, d int) {
	context := unpack_build_context(&context_packed)
	defer func() {
		if err := recover(); err != nil {
			print_backtrace(err)
			c = []candidate{
				{"PANIC", "PANIC", decl_invalid},
			}

			// drop cache
			g_daemon.drop_cache()
		}
	}()
	// TODO: Probably we don't care about comparing all the fields, checking GOROOT and GOPATH
	// should be enough.
	if !reflect.DeepEqual(g_daemon.context.Context, context.Context) {
		g_daemon.context = context
		g_daemon.drop_cache()
	}
	switch g_config.PackageLookupMode {
	case "gb":
		// when package lookup mode is gb, we set GOPATH to "" explicitly and
		// GBProjectRoot becomes valid (or empty)
		var err error
		g_daemon.context.GOPATH = ""
		g_daemon.context.GBProjectRoot, err = find_gb_project_root(filename)
		if *g_debug && err != nil {
			log.Printf("Gb project root not found: %s", err)
		}
	case "go":
		// get current package path for GO15VENDOREXPERIMENT hack
		g_daemon.context.CurrentPackagePath = ""
		pkg, err := g_daemon.context.ImportDir(filepath.Dir(filename), build.FindOnly)
		if err == nil {
			if *g_debug {
				log.Printf("Go project path: %s", pkg.ImportPath)
			}
			g_daemon.context.CurrentPackagePath = pkg.ImportPath
		} else if *g_debug {
			log.Printf("Go project path not found: %s", err)
		}
	}
	if *g_debug {
		var buf bytes.Buffer
		log.Printf("Got autocompletion request for '%s'\n", filename)
		log.Printf("Cursor at: %d\n", cursor)
		buf.WriteString("-------------------------------------------------------\n")
		buf.Write(file[:cursor])
		buf.WriteString("#")
		buf.Write(file[cursor:])
		log.Print(buf.String())
		log.Println("-------------------------------------------------------")
	}
	candidates, d := g_daemon.autocomplete.apropos(file, filename, cursor)
	if *g_debug {
		log.Printf("Offset: %d\n", d)
		log.Printf("Number of candidates found: %d\n", len(candidates))
		log.Printf("Candidates are:\n")
		for _, c := range candidates {
			abbr := fmt.Sprintf("%s %s %s", c.Class, c.Name, c.Type)
			if c.Class == decl_func {
				abbr = fmt.Sprintf("%s %s%s", c.Class, c.Name, c.Type[len("func"):])
			}
			log.Printf("  %s\n", abbr)
		}
		log.Println("=======================================================")
	}
	return candidates, d
}

func server_close(notused int) int {
	g_daemon.close()
	return 0
}

func server_status(notused int) string {
	return g_daemon.autocomplete.status()
}

func server_drop_cache(notused int) int {
	// drop cache
	g_daemon.drop_cache()
	return 0
}

func server_set(key, value string) string {
	if key == "\x00" {
		return g_config.list()
	} else if value == "\x00" {
		return g_config.list_option(key)
	}
	// drop cache on settings changes
	g_daemon.drop_cache()
	return g_config.set_option(key, value)
}
<commit_msg>Implement new auto close feature, so tha the process doesn't stay in the background after your editor has been closed and/or is not using it anymore<commit_after>package main

import (
	"bytes"
	"fmt"
	"go/build"
	"log"
	"net"
	"net/rpc"
	"os"
	"path/filepath"
	"reflect"
	"runtime"
	"time"
)

func do_server() int {
	g_config.read()
	if g_config.ForceDebugOutput != "" {
		// forcefully enable debugging and redirect logging into the
		// specified file
		*g_debug = true
		f, err := os.Create(g_config.ForceDebugOutput)
		if err != nil {
			panic(err)
		}
		log.SetOutput(f)
	}

	addr := *g_addr
	if *g_sock == "unix" {
		addr = get_socket_filename()
		if file_exists(addr) {
			log.Printf("unix socket: '%s' already exists\n", addr)
			return 1
		}
	}
	g_daemon = new_daemon(*g_sock, addr)
	if *g_sock == "unix" {
		// cleanup unix socket file
		defer os.Remove(addr)
	}

	rpc.Register(new(RPC))

	g_daemon.loop()
	return 0
}

//-------------------------------------------------------------------------
// daemon
//-------------------------------------------------------------------------

type daemon struct {
	listener     net.Listener
	cmd_in       chan int
	autocomplete *auto_complete_context
	pkgcache     package_cache
	declcache    *decl_cache
	context      package_lookup_context
}

func new_daemon(network, address string) *daemon {
	var err error

	d := new(daemon)
	d.listener, err = net.Listen(network, address)
	if err != nil {
		panic(err)
	}

	d.cmd_in = make(chan int, 1)
	d.pkgcache = new_package_cache()
	d.declcache = new_decl_cache(&d.context)
	d.autocomplete = new_auto_complete_context(d.pkgcache, d.declcache)
	return d
}

func (this *daemon) drop_cache() {
	this.pkgcache = new_package_cache()
	this.declcache = new_decl_cache(&this.context)
	this.autocomplete = new_auto_complete_context(this.pkgcache, this.declcache)
}

const (
	daemon_close = iota
)

func (this *daemon) loop() {
	conn_in := make(chan net.Conn)
	go func() {
		for {
			c, err := this.listener.Accept()
			if err != nil {
				panic(err)
			}
			conn_in <- c
		}
	}()

	timeout := time.Duration(g_config.CloseTimeout) * time.Second
	countdown := time.NewTimer(timeout)

	for {
		// handle connections or server CMDs (currently one CMD)
		select {
		case c := <-conn_in:
			rpc.ServeConn(c)
			countdown.Reset(timeout)
			runtime.GC()
		case cmd := <-this.cmd_in:
			switch cmd {
			case daemon_close:
				return
			}
		case <-countdown.C:
			return
		}
	}
}

func (this *daemon) close() {
	this.cmd_in <- daemon_close
}

var g_daemon *daemon

//-------------------------------------------------------------------------
// server_* functions
//
// Corresponding client_* functions are autogenerated by goremote.
//-------------------------------------------------------------------------

func server_auto_complete(file []byte, filename string, cursor int, context_packed go_build_context) (c []candidate, d int) {
	context := unpack_build_context(&context_packed)
	defer func() {
		if err := recover(); err != nil {
			print_backtrace(err)
			c = []candidate{
				{"PANIC", "PANIC", decl_invalid},
			}

			// drop cache
			g_daemon.drop_cache()
		}
	}()
	// TODO: Probably we don't care about comparing all the fields, checking GOROOT and GOPATH
	// should be enough.
	if !reflect.DeepEqual(g_daemon.context.Context, context.Context) {
		g_daemon.context = context
		g_daemon.drop_cache()
	}
	switch g_config.PackageLookupMode {
	case "gb":
		// when package lookup mode is gb, we set GOPATH to "" explicitly and
		// GBProjectRoot becomes valid (or empty)
		var err error
		g_daemon.context.GOPATH = ""
		g_daemon.context.GBProjectRoot, err = find_gb_project_root(filename)
		if *g_debug && err != nil {
			log.Printf("Gb project root not found: %s", err)
		}
	case "go":
		// get current package path for GO15VENDOREXPERIMENT hack
		g_daemon.context.CurrentPackagePath = ""
		pkg, err := g_daemon.context.ImportDir(filepath.Dir(filename), build.FindOnly)
		if err == nil {
			if *g_debug {
				log.Printf("Go project path: %s", pkg.ImportPath)
			}
			g_daemon.context.CurrentPackagePath = pkg.ImportPath
		} else if *g_debug {
			log.Printf("Go project path not found: %s", err)
		}
	}
	if *g_debug {
		var buf bytes.Buffer
		log.Printf("Got autocompletion request for '%s'\n", filename)
		log.Printf("Cursor at: %d\n", cursor)
		buf.WriteString("-------------------------------------------------------\n")
		buf.Write(file[:cursor])
		buf.WriteString("#")
		buf.Write(file[cursor:])
		log.Print(buf.String())
		log.Println("-------------------------------------------------------")
	}
	candidates, d := g_daemon.autocomplete.apropos(file, filename, cursor)
	if *g_debug {
		log.Printf("Offset: %d\n", d)
		log.Printf("Number of candidates found: %d\n", len(candidates))
		log.Printf("Candidates are:\n")
		for _, c := range candidates {
			abbr := fmt.Sprintf("%s %s %s", c.Class, c.Name, c.Type)
			if c.Class == decl_func {
				abbr = fmt.Sprintf("%s %s%s", c.Class, c.Name, c.Type[len("func"):])
			}
			log.Printf("  %s\n", abbr)
		}
		log.Println("=======================================================")
	}
	return candidates, d
}

func server_close(notused int) int {
	g_daemon.close()
	return 0
}

func server_status(notused int) string {
	return g_daemon.autocomplete.status()
}

func server_drop_cache(notused int) int {
	// drop cache
	g_daemon.drop_cache()
	return 0
}

func server_set(key, value string) string {
	if key == "\x00" {
		return g_config.list()
	} else if value == "\x00" {
		return g_config.list_option(key)
	}
	// drop cache on settings changes
	g_daemon.drop_cache()
	return g_config.set_option(key, value)
}
<|endoftext|>


<commit_before>package tarantool

import (
	"context"
	"fmt"
	"time"
)

type QueryOptions struct {
	Timeout time.Duration
}

func (conn *Connection) doExecute(r *request, ctx context.Context) *Result {
	requestID := conn.nextID()

	code, packed, err := r.query.Pack(conn.packData)
	if err != nil {
		return &Result{
			Error: &QueryError{
				error: err,
			},
		}
	}

	oldRequest := conn.requests.Put(requestID, r)
	if oldRequest != nil {
		oldRequest.replyChan <- &Result{
			Error: NewConnectionError(conn, "Shred old requests", false), // wtf?
		}
		close(oldRequest.replyChan)
	}

	select {
	case conn.writeChan <- packIproto(code, requestID, packed):
		// pass
	case <-ctx.Done():
		err := ctx.Err()
		return &Result{
			Error: NewConnectionError(conn, fmt.Sprintf("Send error: %s", err), err == context.DeadlineExceeded),
		}
	case <-conn.exit:
		return &Result{
			Error: ConnectionClosedError(conn),
		}
	}

	var res *Result
	select {
	case res = <-r.replyChan:
		// pass
	case <-ctx.Done():
		err := ctx.Err()
		return &Result{
			Error: NewConnectionError(conn, fmt.Sprintf("Recv error: %s", err), err == context.DeadlineExceeded),
		}
	case <-conn.exit:
		return &Result{
			Error: ConnectionClosedError(conn),
		}
	}

	return res
}

func (conn *Connection) Exec(q Query, ctx context.Context) *Result {
	var cancel context.CancelFunc = func() {}

	request := &request{
		query:     q,
		replyChan: make(chan *Result, 1),
	}

	if _, ok := ctx.Deadline(); !ok && conn.queryTimeout != 0 {
		ctx, cancel = context.WithTimeout(ctx, conn.queryTimeout)
	}
	result := conn.doExecute(request, ctx)
	cancel()
	return result
}

func (conn *Connection) ExecuteOptions(q Query, opts *QueryOptions) ([][]interface{}, error) {
	var res *Result
	var ctx context.Context = context.Background()
	var cancel context.CancelFunc = func() {}

	// make options
	if opts == nil {
		opts = &QueryOptions{}
	}

	if opts.Timeout != 0 {
		ctx, cancel = context.WithTimeout(ctx, opts.Timeout)
	}

	res = conn.Exec(q, ctx)
	cancel()

	return res.Data, res.Error
}

func (conn *Connection) Execute(q Query) ([][]interface{}, error) {
	return conn.ExecuteOptions(q, nil)
}
<commit_msg>return connection and timeout error codes<commit_after>package tarantool

import (
	"context"
	"fmt"
	"time"
)

type QueryOptions struct {
	Timeout time.Duration
}

func (conn *Connection) doExecute(r *request, ctx context.Context) *Result {
	requestID := conn.nextID()

	code, packed, err := r.query.Pack(conn.packData)
	if err != nil {
		return &Result{
			Error: &QueryError{
				error: err,
			},
		}
	}

	oldRequest := conn.requests.Put(requestID, r)
	if oldRequest != nil {
		oldRequest.replyChan <- &Result{
			Error: NewConnectionError(conn, "Shred old requests", false), // wtf?
		}
		close(oldRequest.replyChan)
	}

	select {
	case conn.writeChan <- packIproto(code, requestID, packed):
		// pass
	case <-ctx.Done():
		err := ctx.Err()
		return &Result{
			Error:     NewConnectionError(conn, fmt.Sprintf("Send error: %s", err), err == context.DeadlineExceeded),
			ErrorCode: ErrTimeout,
		}
	case <-conn.exit:
		return &Result{
			Error:     ConnectionClosedError(conn),
			ErrorCode: ErrNoConnection,
		}
	}

	var res *Result
	select {
	case res = <-r.replyChan:
		// pass
	case <-ctx.Done():
		err := ctx.Err()
		return &Result{
			Error:     NewConnectionError(conn, fmt.Sprintf("Recv error: %s", err), err == context.DeadlineExceeded),
			ErrorCode: ErrTimeout,
		}
	case <-conn.exit:
		return &Result{
			Error:     ConnectionClosedError(conn),
			ErrorCode: ErrNoConnection,
		}
	}

	return res
}

func (conn *Connection) Exec(q Query, ctx context.Context) *Result {
	var cancel context.CancelFunc = func() {}

	request := &request{
		query:     q,
		replyChan: make(chan *Result, 1),
	}

	if _, ok := ctx.Deadline(); !ok && conn.queryTimeout != 0 {
		ctx, cancel = context.WithTimeout(ctx, conn.queryTimeout)
	}
	result := conn.doExecute(request, ctx)
	cancel()
	return result
}

func (conn *Connection) ExecuteOptions(q Query, opts *QueryOptions) ([][]interface{}, error) {
	var res *Result
	var ctx context.Context = context.Background()
	var cancel context.CancelFunc = func() {}

	// make options
	if opts == nil {
		opts = &QueryOptions{}
	}

	if opts.Timeout != 0 {
		ctx, cancel = context.WithTimeout(ctx, opts.Timeout)
	}

	res = conn.Exec(q, ctx)
	cancel()

	return res.Data, res.Error
}

func (conn *Connection) Execute(q Query) ([][]interface{}, error) {
	return conn.ExecuteOptions(q, nil)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"io"
	"io/ioutil"

	"crypto/tls"
	"log"
	"net/http"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"

	"github.com/codegangsta/cli"
	"github.com/concourse/atc/api/resources"
	"github.com/concourse/atc/api/routes"
	"github.com/concourse/fly/eventstream"
	tbuilds "github.com/concourse/turbine/api/builds"
	"github.com/fraenkel/candiedyaml"
	"github.com/gorilla/websocket"
	"github.com/pivotal-golang/archiver/compressor"
	"github.com/tedsuo/rata"
)

type Input struct {
	Name string
	Path string
	Pipe resources.Pipe
}

func execute(c *cli.Context) {
	atc := c.GlobalString("atcURL")
	buildConfig := c.String("config")
	insecure := c.GlobalBool("insecure")

	atcRequester := newAtcRequester(atc, insecure)

	inputMappings := c.StringSlice("input")
	if len(inputMappings) == 0 {
		wd, err := os.Getwd()
		if err != nil {
			log.Fatalln(err)
		}

		inputMappings = append(inputMappings, filepath.Base(wd)+"="+wd)
	}

	inputs := []Input{}
	for _, i := range inputMappings {
		segs := strings.SplitN(i, "=", 2)
		if len(segs) < 2 {
			log.Println("malformed input:", i)
			os.Exit(1)
		}

		inputName := segs[0]

		absPath, err := filepath.Abs(segs[1])
		if err != nil {
			log.Printf("could not locate input %s: %s\n", inputName, err)
			os.Exit(1)
		}

		pipe := createPipe(atcRequester)

		inputs = append(inputs, Input{
			Name: inputName,
			Path: absPath,
			Pipe: pipe,
		})
	}

	absConfig, err := filepath.Abs(buildConfig)
	if err != nil {
		log.Println("could not locate config file:", err)
		os.Exit(1)
	}

	build, cookies := createBuild(
		atcRequester,
		inputs,
		loadConfig(absConfig, c.Args()),
	)

	terminate := make(chan os.Signal, 1)

	go abortOnSignal(atcRequester, terminate, build)

	signal.Notify(terminate, syscall.SIGINT, syscall.SIGTERM)

	logOutput, err := atcRequester.CreateRequest(
		routes.BuildEvents,
		rata.Params{"build_id": strconv.Itoa(build.ID)},
		nil,
	)
	if err != nil {
		log.Fatalln(err)
	}

	logOutput.URL.Scheme = "ws"
	logOutput.URL.User = nil

	cookieHeaders := []string{}
	for _, cookie := range cookies {
		cookieHeaders = append(cookieHeaders, cookie.String())
	}

	conn, res, err := atcRequester.websocketDialer.Dial(
		logOutput.URL.String(),
		http.Header{"Cookie": cookieHeaders},
	)
	if err != nil {
		log.Println("failed to stream output:", err, res)
		os.Exit(1)
	}

	go func() {
		for _, i := range inputs {
			upload(i, atcRequester)
		}
	}()

	exitCode, err := eventstream.RenderStream(conn)
	if err != nil {
		log.Println("failed to render stream:", err)
		os.Exit(1)
	}

	res.Body.Close()
	conn.Close()

	os.Exit(exitCode)
}

func createPipe(atcRequester *atcRequester) resources.Pipe {
	cPipe, err := atcRequester.CreateRequest(routes.CreatePipe, nil, nil)
	if err != nil {
		log.Fatalln(err)
	}

	response, err := atcRequester.httpClient.Do(cPipe)
	if err != nil {
		log.Fatalln("request failed:", err)
	}

	defer response.Body.Close()

	if response.StatusCode != http.StatusCreated {
		log.Println("bad response when creating pipe:", response)
		response.Write(os.Stderr)
		os.Exit(1)
	}

	var pipe resources.Pipe
	err = json.NewDecoder(response.Body).Decode(&pipe)
	if err != nil {
		log.Println("malformed response when creating pipe:", err)
		os.Exit(1)
	}

	return pipe
}

func loadConfig(configPath string, args []string) tbuilds.Config {
	configFile, err := os.Open(configPath)
	if err != nil {
		log.Fatalln("could not open config file:", err)
	}

	var config tbuilds.Config

	err = candiedyaml.NewDecoder(configFile).Decode(&config)
	if err != nil {
		log.Fatalln("could not parse config file:", err)
	}

	config.Run.Args = append(config.Run.Args, args...)

	for k, _ := range config.Params {
		env, found := syscall.Getenv(k)
		if found {
			config.Params[k] = env
		}
	}

	return config
}

func createBuild(
	atcRequester *atcRequester,
	inputs []Input,
	config tbuilds.Config,
) (resources.Build, []*http.Cookie) {
	buffer := &bytes.Buffer{}

	buildInputs := make([]tbuilds.Input, len(inputs))
	for idx, i := range inputs {
		readPipe, err := atcRequester.CreateHTTPRequest(
			routes.ReadPipe,
			rata.Params{"pipe_id": i.Pipe.ID},
			nil,
		)
		if err != nil {
			log.Fatalln(err)
		}

		readPipe.URL.Host = i.Pipe.PeerAddr

		buildInputs[idx] = tbuilds.Input{
			Name: i.Name,
			Type: "archive",
			Source: tbuilds.Source{
				"uri": readPipe.URL.String(),
			},
		}
	}

	turbineBuild := tbuilds.Build{
		Privileged: true,
		Config:     config,
		Inputs:     buildInputs,
	}

	err := json.NewEncoder(buffer).Encode(turbineBuild)
	if err != nil {
		log.Fatalln("encoding build failed:", err)
	}

	createBuild, err := atcRequester.CreateRequest(routes.CreateBuild, nil, buffer)
	if err != nil {
		log.Fatalln(err)
	}

	createBuild.Header.Set("Content-Type", "application/json")

	response, err := atcRequester.httpClient.Do(createBuild)
	if err != nil {
		log.Fatalln("request failed:", err)
	}

	defer response.Body.Close()

	if response.StatusCode != http.StatusCreated {
		log.Println("bad response when creating build:", response)
		response.Write(os.Stderr)
		os.Exit(1)
	}

	var build resources.Build
	err = json.NewDecoder(response.Body).Decode(&build)
	if err != nil {
		log.Fatalln("response decoding failed:", err)
	}

	return build, response.Cookies()
}

func abortOnSignal(
	atcRequester *atcRequester,
	terminate <-chan os.Signal,
	build resources.Build,
) {
	<-terminate

	println("\naborting...")

	abortReq, err := atcRequester.CreateRequest(
		routes.AbortBuild,
		rata.Params{"build_id": strconv.Itoa(build.ID)},
		nil,
	)

	if err != nil {
		log.Fatalln(err)
	}

	resp, err := atcRequester.httpClient.Do(abortReq)
	if err != nil {
		log.Println("failed to abort:", err)
	}

	resp.Body.Close()

	// if told to terminate again, exit immediately
	<-terminate
	println("exiting immediately")
	os.Exit(2)
}

func upload(input Input, atcRequester *atcRequester) {
	path := input.Path
	pipe := input.Pipe

	var archive io.ReadCloser
	if tarPath, err := exec.LookPath("tar"); err != nil {
		compressor := compressor.NewTgz()

		tmpfile, err := ioutil.TempFile("", "fly")
		if err != nil {
			log.Fatalln("creating tempfile failed:", err)
		}

		tmpfile.Close()

		defer os.Remove(tmpfile.Name())

		err = compressor.Compress(path+"/", tmpfile.Name())
		if err != nil {
			log.Fatalln("creating archive failed:", err)
		}

		archive, err = os.Open(tmpfile.Name())
		if err != nil {
			log.Fatalln("could not open archive:", err)
		}
	} else {
		tarCmd := exec.Command(tarPath, "--exclude", ".git", "-czf", "-", ".")
		tarCmd.Dir = path
		tarCmd.Stderr = os.Stderr

		archive, err = tarCmd.StdoutPipe()
		if err != nil {
			log.Fatalln("could not create tar pipe:", err)
		}

		err = tarCmd.Start()
		if err != nil {
			log.Fatalln("could not run tar:", err)
		}
	}

	defer archive.Close()

	uploadBits, err := atcRequester.CreateRequest(
		routes.WritePipe,
		rata.Params{"pipe_id": pipe.ID},
		archive,
	)
	if err != nil {
		log.Fatalln(err)
	}

	response, err := atcRequester.httpClient.Do(uploadBits)
	if err != nil {
		log.Fatalln("request failed:", err)
	}

	defer response.Body.Close()

	if response.StatusCode != http.StatusOK {
		log.Println("bad response when uploading bits:", response)
		response.Write(os.Stderr)
		os.Exit(1)
	}
}

type atcRequester struct {
	*rata.RequestGenerator
	httpClient      *http.Client
	websocketDialer *websocket.Dialer
}

func newAtcRequester(atcUrl string, insecure bool) *atcRequester {
	tlsClientConfig := &tls.Config{InsecureSkipVerify: insecure}

	return &atcRequester{
		rata.NewRequestGenerator(atcUrl, routes.Routes),
		&http.Client{Transport: &http.Transport{TLSClientConfig: tlsClientConfig}},
		&websocket.Dialer{TLSClientConfig: tlsClientConfig},
	}
}

func (ar *atcRequester) CreateHTTPRequest(
	name string,
	params rata.Params,
	body io.Reader,
) (*http.Request, error) {
	request, err := ar.CreateRequest(name, params, body)
	if err != nil {
		return nil, err
	}

	url := request.URL
	url.Scheme = "http"
	request.URL = url
	return request, nil
}
<commit_msg>use git ls-files to determine files to upload<commit_after>package main

import (
	"bufio"
	"bytes"
	"encoding/json"
	"io"
	"io/ioutil"

	"crypto/tls"
	"log"
	"net/http"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"

	"github.com/codegangsta/cli"
	"github.com/concourse/atc/api/resources"
	"github.com/concourse/atc/api/routes"
	"github.com/concourse/fly/eventstream"
	tbuilds "github.com/concourse/turbine/api/builds"
	"github.com/fraenkel/candiedyaml"
	"github.com/gorilla/websocket"
	"github.com/pivotal-golang/archiver/compressor"
	"github.com/tedsuo/rata"
)

type Input struct {
	Name string
	Path string
	Pipe resources.Pipe
}

func execute(c *cli.Context) {
	atc := c.GlobalString("atcURL")
	buildConfig := c.String("config")
	insecure := c.GlobalBool("insecure")

	atcRequester := newAtcRequester(atc, insecure)

	inputMappings := c.StringSlice("input")
	if len(inputMappings) == 0 {
		wd, err := os.Getwd()
		if err != nil {
			log.Fatalln(err)
		}

		inputMappings = append(inputMappings, filepath.Base(wd)+"="+wd)
	}

	inputs := []Input{}
	for _, i := range inputMappings {
		segs := strings.SplitN(i, "=", 2)
		if len(segs) < 2 {
			log.Println("malformed input:", i)
			os.Exit(1)
		}

		inputName := segs[0]

		absPath, err := filepath.Abs(segs[1])
		if err != nil {
			log.Printf("could not locate input %s: %s\n", inputName, err)
			os.Exit(1)
		}

		pipe := createPipe(atcRequester)

		inputs = append(inputs, Input{
			Name: inputName,
			Path: absPath,
			Pipe: pipe,
		})
	}

	absConfig, err := filepath.Abs(buildConfig)
	if err != nil {
		log.Println("could not locate config file:", err)
		os.Exit(1)
	}

	build, cookies := createBuild(
		atcRequester,
		inputs,
		loadConfig(absConfig, c.Args()),
	)

	terminate := make(chan os.Signal, 1)

	go abortOnSignal(atcRequester, terminate, build)

	signal.Notify(terminate, syscall.SIGINT, syscall.SIGTERM)

	logOutput, err := atcRequester.CreateRequest(
		routes.BuildEvents,
		rata.Params{"build_id": strconv.Itoa(build.ID)},
		nil,
	)
	if err != nil {
		log.Fatalln(err)
	}

	logOutput.URL.Scheme = "ws"
	logOutput.URL.User = nil

	cookieHeaders := []string{}
	for _, cookie := range cookies {
		cookieHeaders = append(cookieHeaders, cookie.String())
	}

	conn, res, err := atcRequester.websocketDialer.Dial(
		logOutput.URL.String(),
		http.Header{"Cookie": cookieHeaders},
	)
	if err != nil {
		log.Println("failed to stream output:", err, res)
		os.Exit(1)
	}

	go func() {
		for _, i := range inputs {
			upload(i, atcRequester)
		}
	}()

	exitCode, err := eventstream.RenderStream(conn)
	if err != nil {
		log.Println("failed to render stream:", err)
		os.Exit(1)
	}

	res.Body.Close()
	conn.Close()

	os.Exit(exitCode)
}

func createPipe(atcRequester *atcRequester) resources.Pipe {
	cPipe, err := atcRequester.CreateRequest(routes.CreatePipe, nil, nil)
	if err != nil {
		log.Fatalln(err)
	}

	response, err := atcRequester.httpClient.Do(cPipe)
	if err != nil {
		log.Fatalln("request failed:", err)
	}

	defer response.Body.Close()

	if response.StatusCode != http.StatusCreated {
		log.Println("bad response when creating pipe:", response)
		response.Write(os.Stderr)
		os.Exit(1)
	}

	var pipe resources.Pipe
	err = json.NewDecoder(response.Body).Decode(&pipe)
	if err != nil {
		log.Println("malformed response when creating pipe:", err)
		os.Exit(1)
	}

	return pipe
}

func loadConfig(configPath string, args []string) tbuilds.Config {
	configFile, err := os.Open(configPath)
	if err != nil {
		log.Fatalln("could not open config file:", err)
	}

	var config tbuilds.Config

	err = candiedyaml.NewDecoder(configFile).Decode(&config)
	if err != nil {
		log.Fatalln("could not parse config file:", err)
	}

	config.Run.Args = append(config.Run.Args, args...)

	for k, _ := range config.Params {
		env, found := syscall.Getenv(k)
		if found {
			config.Params[k] = env
		}
	}

	return config
}

func createBuild(
	atcRequester *atcRequester,
	inputs []Input,
	config tbuilds.Config,
) (resources.Build, []*http.Cookie) {
	buffer := &bytes.Buffer{}

	buildInputs := make([]tbuilds.Input, len(inputs))
	for idx, i := range inputs {
		readPipe, err := atcRequester.CreateHTTPRequest(
			routes.ReadPipe,
			rata.Params{"pipe_id": i.Pipe.ID},
			nil,
		)
		if err != nil {
			log.Fatalln(err)
		}

		readPipe.URL.Host = i.Pipe.PeerAddr

		buildInputs[idx] = tbuilds.Input{
			Name: i.Name,
			Type: "archive",
			Source: tbuilds.Source{
				"uri": readPipe.URL.String(),
			},
		}
	}

	turbineBuild := tbuilds.Build{
		Privileged: true,
		Config:     config,
		Inputs:     buildInputs,
	}

	err := json.NewEncoder(buffer).Encode(turbineBuild)
	if err != nil {
		log.Fatalln("encoding build failed:", err)
	}

	createBuild, err := atcRequester.CreateRequest(routes.CreateBuild, nil, buffer)
	if err != nil {
		log.Fatalln(err)
	}

	createBuild.Header.Set("Content-Type", "application/json")

	response, err := atcRequester.httpClient.Do(createBuild)
	if err != nil {
		log.Fatalln("request failed:", err)
	}

	defer response.Body.Close()

	if response.StatusCode != http.StatusCreated {
		log.Println("bad response when creating build:", response)
		response.Write(os.Stderr)
		os.Exit(1)
	}

	var build resources.Build
	err = json.NewDecoder(response.Body).Decode(&build)
	if err != nil {
		log.Fatalln("response decoding failed:", err)
	}

	return build, response.Cookies()
}

func abortOnSignal(
	atcRequester *atcRequester,
	terminate <-chan os.Signal,
	build resources.Build,
) {
	<-terminate

	println("\naborting...")

	abortReq, err := atcRequester.CreateRequest(
		routes.AbortBuild,
		rata.Params{"build_id": strconv.Itoa(build.ID)},
		nil,
	)

	if err != nil {
		log.Fatalln(err)
	}

	resp, err := atcRequester.httpClient.Do(abortReq)
	if err != nil {
		log.Println("failed to abort:", err)
	}

	resp.Body.Close()

	// if told to terminate again, exit immediately
	<-terminate
	println("exiting immediately")
	os.Exit(2)
}

func upload(input Input, atcRequester *atcRequester) {
	path := input.Path
	pipe := input.Pipe

	var archive io.ReadCloser
	if tarPath, err := exec.LookPath("tar"); err != nil {
		compressor := compressor.NewTgz()

		tmpfile, err := ioutil.TempFile("", "fly")
		if err != nil {
			log.Fatalln("creating tempfile failed:", err)
		}

		tmpfile.Close()

		defer os.Remove(tmpfile.Name())

		err = compressor.Compress(path+"/", tmpfile.Name())
		if err != nil {
			log.Fatalln("creating archive failed:", err)
		}

		archive, err = os.Open(tmpfile.Name())
		if err != nil {
			log.Fatalln("could not open archive:", err)
		}
	} else {
		files, err := getGitFiles(path)
		if err != nil {
			files = []string{"."}
		}

		tarCmd := exec.Command(tarPath, append([]string{"-czf", "-"}, files...)...)
		tarCmd.Dir = path
		tarCmd.Stderr = os.Stderr

		archive, err = tarCmd.StdoutPipe()
		if err != nil {
			log.Fatalln("could not create tar pipe:", err)
		}

		err = tarCmd.Start()
		if err != nil {
			log.Fatalln("could not run tar:", err)
		}
	}

	defer archive.Close()

	uploadBits, err := atcRequester.CreateRequest(
		routes.WritePipe,
		rata.Params{"pipe_id": pipe.ID},
		archive,
	)
	if err != nil {
		log.Fatalln(err)
	}

	response, err := atcRequester.httpClient.Do(uploadBits)
	if err != nil {
		log.Fatalln("request failed:", err)
	}

	defer response.Body.Close()

	if response.StatusCode != http.StatusOK {
		log.Println("bad response when uploading bits:", response)
		response.Write(os.Stderr)
		os.Exit(1)
	}
}

type atcRequester struct {
	*rata.RequestGenerator
	httpClient      *http.Client
	websocketDialer *websocket.Dialer
}

func newAtcRequester(atcUrl string, insecure bool) *atcRequester {
	tlsClientConfig := &tls.Config{InsecureSkipVerify: insecure}

	return &atcRequester{
		rata.NewRequestGenerator(atcUrl, routes.Routes),
		&http.Client{Transport: &http.Transport{TLSClientConfig: tlsClientConfig}},
		&websocket.Dialer{TLSClientConfig: tlsClientConfig},
	}
}

func (ar *atcRequester) CreateHTTPRequest(
	name string,
	params rata.Params,
	body io.Reader,
) (*http.Request, error) {
	request, err := ar.CreateRequest(name, params, body)
	if err != nil {
		return nil, err
	}

	url := request.URL
	url.Scheme = "http"
	request.URL = url
	return request, nil
}

func getGitFiles(dir string) ([]string, error) {
	files := []string{}

	gitLS := exec.Command("git", "ls-files", "-z")
	gitLS.Dir = dir

	gitOut, err := gitLS.StdoutPipe()
	if err != nil {
		return nil, err
	}

	outScan := bufio.NewScanner(gitOut)
	outScan.Split(scanNull)

	err = gitLS.Start()
	if err == nil {
		return nil, err
	}

	for outScan.Scan() {
		files = append(files, outScan.Text())
	}

	err = gitLS.Wait()
	if err != nil {
		return nil, err
	}

	return files, nil
}

func scanNull(data []byte, atEOF bool) (int, []byte, error) {
	// eof, no more data; terminate
	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}

	// look for terminating null byte
	if i := bytes.IndexByte(data, 0); i >= 0 {
		return i + 1, data[0:i], nil
	}

	// no final terminator; return what's left
	if atEOF {
		return len(data), data, nil
	}

	// request more data
	return 0, nil, nil
}
<|endoftext|>


<commit_before>package eval

import (
	"fmt"
	"os"

	"github.com/xiaq/elvish/parse"
)

// Definition of Op and friends and combinators.

// Op operates on an Evaluator.
type Op func(*Evaluator)

// valuesOp operates on an Evaluator and results in some values.
type valuesOp struct {
	ts []Type
	f  func(*Evaluator) []Value
}

// portOp operates on an Evaluator and results in a port.
type portOp func(*Evaluator) *port

// stateUpdatesOp operates on an Evaluator and results in a receiving channel
// of StateUpdate's.
type stateUpdatesOp func(*Evaluator) <-chan *StateUpdate

func combineChunk(ops []valuesOp) Op {
	return func(ev *Evaluator) {
		for _, op := range ops {
			s := op.f(ev)
			if ev.statusCb != nil {
				ev.statusCb(s)
			}
		}
	}
}

func combineClosure(ops []valuesOp, enclosed map[string]Type, bounds [2]StreamType) valuesOp {
	op := combineChunk(ops)
	// BUG(xiaq): Closure arguments is (again) not supported
	ts := []Type{&ClosureType{bounds}}
	f := func(ev *Evaluator) []Value {
		enclosed := make(map[string]*Value, len(enclosed))
		for name := range enclosed {
			enclosed[name] = ev.scope[name]
		}
		return []Value{NewClosure(nil, op, enclosed, bounds)}
	}
	return valuesOp{ts, f}
}

func combinePipeline(n parse.Node, ops []stateUpdatesOp, bounds [2]StreamType, internals []StreamType) valuesOp {
	ts := make([]Type, len(ops))
	for i := 0; i < len(ops); i++ {
		ts[i] = &StringType{}
	}
	f := func(ev *Evaluator) []Value {
		// TODO(xiaq): Should catch when compiling
		if !ev.ports[0].compatible(bounds[0]) {
			ev.errorfNode(n, "pipeline input not satisfiable")
		}
		if !ev.ports[1].compatible(bounds[1]) {
			ev.errorfNode(n, "pipeline output not satisfiable")
		}
		var nextIn *port
		updates := make([]<-chan *StateUpdate, len(ops))
		// For each form, create a dedicated Evaluator and run
		for i, op := range ops {
			newEv := ev.copy(fmt.Sprintf("<form op %v>", op), false)
			if i > 0 {
				newEv.ports[0] = nextIn
			}
			if i < len(ops)-1 {
				switch internals[i] {
				case unusedStream:
					newEv.ports[1] = nil
					nextIn = nil
				case fdStream:
					// os.Pipe sets O_CLOEXEC, which is what we want.
					reader, writer, e := os.Pipe()
					if e != nil {
						ev.errorfNode(n, "failed to create pipe: %s", e)
					}
					newEv.ports[1] = &port{f: writer, shouldClose: true}
					nextIn = &port{f: reader, shouldClose: true}
				case chanStream:
					// TODO Buffered channel?
					ch := make(chan Value)
					// Only the writer closes the channel port
					newEv.ports[1] = &port{ch: ch, shouldClose: true}
					nextIn = &port{ch: ch}
				default:
					panic("bad StreamType value")
				}
			}
			updates[i] = op(newEv)
		}
		// Collect exit values
		exits := make([]Value, len(ops))
		for i, update := range updates {
			for up := range update {
				exits[i] = NewString(up.Msg)
			}
		}
		return exits
	}
	return valuesOp{ts, f}
}

func combineForm(n parse.Node, cmd valuesOp, tlist valuesOp, ports []portOp, a *formAnnotation) stateUpdatesOp {
	return func(ev *Evaluator) <-chan *StateUpdate {
		// XXX Currently it's guaranteed that cmd evaluates into a single
		// Value.
		cmd := cmd.f(ev)[0]
		cmdStr := cmd.String()
		fm := &form{
			name: cmdStr,
		}
		if tlist.f != nil {
			fm.args = tlist.f(ev)
		}

		switch a.commandType {
		case commandBuiltinFunction:
			fm.Command.Func = a.builtinFunc.fn
		case commandBuiltinSpecial:
			fm.Command.Special = a.specialOp
		case commandDefinedFunction:
			v, ok1 := ev.scope["fn-"+cmdStr]
			fn, ok2 := (*v).(*Closure)
			if !(ok1 && ok2) {
				panic("Checker bug")
			}
			fm.Command.Closure = fn
		case commandClosure:
			fm.Command.Closure = cmd.(*Closure)
		case commandExternal:
			path, e := ev.search(cmdStr)
			if e != nil {
				ev.errorfNode(n, "%s", e)
			}
			fm.Command.Path = path
		default:
			panic("bad commandType value")
		}

		newEv := ev.copy(fmt.Sprintf("<form redir %v>", fm), true)
		newEv.growPorts(len(ports))

		for i, op := range ports {
			if op != nil {
				newEv.ports[i] = op(ev)
			}
		}

		return newEv.execForm(fm)
	}
}

func combineTermList(ops []valuesOp) valuesOp {
	ts := make([]Type, 0, len(ops))
	for _, op := range ops {
		ts = append(ts, op.ts...)
	}
	f := func(ev *Evaluator) []Value {
		// Use number of terms as an estimation of the number of values
		vs := make([]Value, 0, len(ops))
		for _, op := range ops {
			us := op.f(ev)
			vs = append(vs, us...)
		}
		return vs
	}
	return valuesOp{ts, f}
}

func combineTerm(ops []valuesOp) valuesOp {
	ts := ops[0].ts
	for _, op := range ops[1:] {
		rs := op.ts
		if len(rs) == 1 {
			r := rs[0]
			for i := range ts {
				ts[i] = ts[i].Caret(r)
			}
		} else {
			// Do a cartesian product
			newts := make([]Type, len(ts)*len(rs))
			for i, t := range ts {
				for j, r := range rs {
					newts[i*len(rs)+j] = t.Caret(r)
				}
			}
			ts = newts
		}
	}

	f := func(ev *Evaluator) []Value {
		vs := ops[0].f(ev)
		for _, op := range ops[1:] {
			us := op.f(ev)
			if len(us) == 1 {
				u := us[0]
				for i := range vs {
					vs[i] = vs[i].Caret(ev, u)
				}
			} else {
				// Do a cartesian product
				newvs := make([]Value, len(vs)*len(us))
				for i, v := range vs {
					for j, u := range us {
						newvs[i*len(us)+j] = v.Caret(ev, u)
					}
				}
				vs = newvs
			}
		}
		return vs
	}
	return valuesOp{ts, f}
}

func literalValue(v ...Value) valuesOp {
	ts := make([]Type, len(v))
	for i := 0; i < len(v); i++ {
		ts[i] = v[i].Type()
	}
	f := func(e *Evaluator) []Value {
		return v
	}
	return valuesOp{ts, f}
}

func makeString(text string) valuesOp {
	return literalValue(NewString(text))
}

func makeVar(cp *Compiler, name string, fn *parse.FactorNode) valuesOp {
	ts := []Type{cp.resolveVar(name, fn)}
	f := func(ev *Evaluator) []Value {
		val, ok := ev.scope[name]
		if !ok {
			panic("Checker bug")
		}
		return []Value{*val}
	}
	return valuesOp{ts, f}
}

func combineTable(n parse.Node, list valuesOp, keys []valuesOp, values []valuesOp) valuesOp {
	ts := []Type{TableType{}}
	f := func(ev *Evaluator) []Value {
		t := NewTable()
		t.append(list.f(ev)...)
		for i, kop := range keys {
			vop := values[i]
			ks := kop.f(ev)
			vs := vop.f(ev)
			if len(ks) != len(vs) {
				ev.errorfNode(n, "Number of keys doesn't match number of values: %d vs. %d", len(ks), len(vs))
			}
			for j, k := range ks {
				t.Dict[k] = vs[j]
			}
		}
		return []Value{t}
	}
	return valuesOp{ts, f}
}

func combineOutputCapture(op valuesOp, bounds [2]StreamType) valuesOp {
	// XXX Wrong type; ts should be variadic
	ts := []Type{}
	f := func(ev *Evaluator) []Value {
		vs := []Value{}
		newEv := ev.copy(fmt.Sprintf("<output capture %v>", op), true)
		newEv.ports = make([]*port, len(ev.ports))
		copy(newEv.ports, ev.ports)
		ch := make(chan Value)
		newEv.ports[1] = &port{ch: ch}
		go func() {
			for v := range ch {
				vs = append(vs, v)
			}
		}()
		op.f(newEv)
		return vs
	}
	return valuesOp{ts, f}
}
<commit_msg>eval: s/Checker/Compiler/g<commit_after>package eval

import (
	"fmt"
	"os"

	"github.com/xiaq/elvish/parse"
)

// Definition of Op and friends and combinators.

// Op operates on an Evaluator.
type Op func(*Evaluator)

// valuesOp operates on an Evaluator and results in some values.
type valuesOp struct {
	ts []Type
	f  func(*Evaluator) []Value
}

// portOp operates on an Evaluator and results in a port.
type portOp func(*Evaluator) *port

// stateUpdatesOp operates on an Evaluator and results in a receiving channel
// of StateUpdate's.
type stateUpdatesOp func(*Evaluator) <-chan *StateUpdate

func combineChunk(ops []valuesOp) Op {
	return func(ev *Evaluator) {
		for _, op := range ops {
			s := op.f(ev)
			if ev.statusCb != nil {
				ev.statusCb(s)
			}
		}
	}
}

func combineClosure(ops []valuesOp, enclosed map[string]Type, bounds [2]StreamType) valuesOp {
	op := combineChunk(ops)
	// BUG(xiaq): Closure arguments is (again) not supported
	ts := []Type{&ClosureType{bounds}}
	f := func(ev *Evaluator) []Value {
		enclosed := make(map[string]*Value, len(enclosed))
		for name := range enclosed {
			enclosed[name] = ev.scope[name]
		}
		return []Value{NewClosure(nil, op, enclosed, bounds)}
	}
	return valuesOp{ts, f}
}

func combinePipeline(n parse.Node, ops []stateUpdatesOp, bounds [2]StreamType, internals []StreamType) valuesOp {
	ts := make([]Type, len(ops))
	for i := 0; i < len(ops); i++ {
		ts[i] = &StringType{}
	}
	f := func(ev *Evaluator) []Value {
		// TODO(xiaq): Should catch when compiling
		if !ev.ports[0].compatible(bounds[0]) {
			ev.errorfNode(n, "pipeline input not satisfiable")
		}
		if !ev.ports[1].compatible(bounds[1]) {
			ev.errorfNode(n, "pipeline output not satisfiable")
		}
		var nextIn *port
		updates := make([]<-chan *StateUpdate, len(ops))
		// For each form, create a dedicated Evaluator and run
		for i, op := range ops {
			newEv := ev.copy(fmt.Sprintf("<form op %v>", op), false)
			if i > 0 {
				newEv.ports[0] = nextIn
			}
			if i < len(ops)-1 {
				switch internals[i] {
				case unusedStream:
					newEv.ports[1] = nil
					nextIn = nil
				case fdStream:
					// os.Pipe sets O_CLOEXEC, which is what we want.
					reader, writer, e := os.Pipe()
					if e != nil {
						ev.errorfNode(n, "failed to create pipe: %s", e)
					}
					newEv.ports[1] = &port{f: writer, shouldClose: true}
					nextIn = &port{f: reader, shouldClose: true}
				case chanStream:
					// TODO Buffered channel?
					ch := make(chan Value)
					// Only the writer closes the channel port
					newEv.ports[1] = &port{ch: ch, shouldClose: true}
					nextIn = &port{ch: ch}
				default:
					panic("bad StreamType value")
				}
			}
			updates[i] = op(newEv)
		}
		// Collect exit values
		exits := make([]Value, len(ops))
		for i, update := range updates {
			for up := range update {
				exits[i] = NewString(up.Msg)
			}
		}
		return exits
	}
	return valuesOp{ts, f}
}

func combineForm(n parse.Node, cmd valuesOp, tlist valuesOp, ports []portOp, a *formAnnotation) stateUpdatesOp {
	return func(ev *Evaluator) <-chan *StateUpdate {
		// XXX Currently it's guaranteed that cmd evaluates into a single
		// Value.
		cmd := cmd.f(ev)[0]
		cmdStr := cmd.String()
		fm := &form{
			name: cmdStr,
		}
		if tlist.f != nil {
			fm.args = tlist.f(ev)
		}

		switch a.commandType {
		case commandBuiltinFunction:
			fm.Command.Func = a.builtinFunc.fn
		case commandBuiltinSpecial:
			fm.Command.Special = a.specialOp
		case commandDefinedFunction:
			v, ok1 := ev.scope["fn-"+cmdStr]
			fn, ok2 := (*v).(*Closure)
			if !(ok1 && ok2) {
				panic("Compiler bug")
			}
			fm.Command.Closure = fn
		case commandClosure:
			fm.Command.Closure = cmd.(*Closure)
		case commandExternal:
			path, e := ev.search(cmdStr)
			if e != nil {
				ev.errorfNode(n, "%s", e)
			}
			fm.Command.Path = path
		default:
			panic("bad commandType value")
		}

		newEv := ev.copy(fmt.Sprintf("<form redir %v>", fm), true)
		newEv.growPorts(len(ports))

		for i, op := range ports {
			if op != nil {
				newEv.ports[i] = op(ev)
			}
		}

		return newEv.execForm(fm)
	}
}

func combineTermList(ops []valuesOp) valuesOp {
	ts := make([]Type, 0, len(ops))
	for _, op := range ops {
		ts = append(ts, op.ts...)
	}
	f := func(ev *Evaluator) []Value {
		// Use number of terms as an estimation of the number of values
		vs := make([]Value, 0, len(ops))
		for _, op := range ops {
			us := op.f(ev)
			vs = append(vs, us...)
		}
		return vs
	}
	return valuesOp{ts, f}
}

func combineTerm(ops []valuesOp) valuesOp {
	ts := ops[0].ts
	for _, op := range ops[1:] {
		rs := op.ts
		if len(rs) == 1 {
			r := rs[0]
			for i := range ts {
				ts[i] = ts[i].Caret(r)
			}
		} else {
			// Do a cartesian product
			newts := make([]Type, len(ts)*len(rs))
			for i, t := range ts {
				for j, r := range rs {
					newts[i*len(rs)+j] = t.Caret(r)
				}
			}
			ts = newts
		}
	}

	f := func(ev *Evaluator) []Value {
		vs := ops[0].f(ev)
		for _, op := range ops[1:] {
			us := op.f(ev)
			if len(us) == 1 {
				u := us[0]
				for i := range vs {
					vs[i] = vs[i].Caret(ev, u)
				}
			} else {
				// Do a cartesian product
				newvs := make([]Value, len(vs)*len(us))
				for i, v := range vs {
					for j, u := range us {
						newvs[i*len(us)+j] = v.Caret(ev, u)
					}
				}
				vs = newvs
			}
		}
		return vs
	}
	return valuesOp{ts, f}
}

func literalValue(v ...Value) valuesOp {
	ts := make([]Type, len(v))
	for i := 0; i < len(v); i++ {
		ts[i] = v[i].Type()
	}
	f := func(e *Evaluator) []Value {
		return v
	}
	return valuesOp{ts, f}
}

func makeString(text string) valuesOp {
	return literalValue(NewString(text))
}

func makeVar(cp *Compiler, name string, fn *parse.FactorNode) valuesOp {
	ts := []Type{cp.resolveVar(name, fn)}
	f := func(ev *Evaluator) []Value {
		val, ok := ev.scope[name]
		if !ok {
			panic("Compiler bug")
		}
		return []Value{*val}
	}
	return valuesOp{ts, f}
}

func combineTable(n parse.Node, list valuesOp, keys []valuesOp, values []valuesOp) valuesOp {
	ts := []Type{TableType{}}
	f := func(ev *Evaluator) []Value {
		t := NewTable()
		t.append(list.f(ev)...)
		for i, kop := range keys {
			vop := values[i]
			ks := kop.f(ev)
			vs := vop.f(ev)
			if len(ks) != len(vs) {
				ev.errorfNode(n, "Number of keys doesn't match number of values: %d vs. %d", len(ks), len(vs))
			}
			for j, k := range ks {
				t.Dict[k] = vs[j]
			}
		}
		return []Value{t}
	}
	return valuesOp{ts, f}
}

func combineOutputCapture(op valuesOp, bounds [2]StreamType) valuesOp {
	// XXX Wrong type; ts should be variadic
	ts := []Type{}
	f := func(ev *Evaluator) []Value {
		vs := []Value{}
		newEv := ev.copy(fmt.Sprintf("<output capture %v>", op), true)
		newEv.ports = make([]*port, len(ev.ports))
		copy(newEv.ports, ev.ports)
		ch := make(chan Value)
		newEv.ports[1] = &port{ch: ch}
		go func() {
			for v := range ch {
				vs = append(vs, v)
			}
		}()
		op.f(newEv)
		return vs
	}
	return valuesOp{ts, f}
}
<|endoftext|>