<commit_before>// Copyright 2016 Google, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/*
Package credhelper implements a Docker credential helper with special facilities
for GCR authentication.
*/
package credhelper

import (
	"bytes"
	"fmt"
	"net/url"
	"os/exec"
	"strings"

	"github.com/GoogleCloudPlatform/docker-credential-gcr/config"
	"github.com/GoogleCloudPlatform/docker-credential-gcr/store"
	"github.com/docker/docker-credential-helpers/credentials"

	"golang.org/x/oauth2/google"
)

const gcrOAuth2Username = "oauth2accesstoken"

// gcrCredHelper implements a credentials.Helper interface backed by a GCR
// credential store.
type gcrCredHelper struct {
	store store.GCRCredStore

	// helper methods, package exposed for testing
	defaultCredsToken func() (string, error)
	gcloudSDKToken    func() (string, error)
	credStoreToken    func(store.GCRCredStore) (string, error)
}

// NewGCRCredentialHelper returns a Docker credential helper which
// specializes in GCR's authentication schemes.
func NewGCRCredentialHelper(store store.GCRCredStore) credentials.Helper {
	return &gcrCredHelper{
		store:             store,
		credStoreToken:    tokenFromPrivateStore,
		gcloudSDKToken:    tokenFromGcloudSDK,
		defaultCredsToken: tokenFromAppDefaultCreds,
	}
}

// Delete lists all credentials stored and associated usernames.
func (ch *gcrCredHelper) List() ([]string, []string, error) {
	all3pCreds, err := ch.store.AllThirdPartyCreds()
	if err != nil {
		return nil, nil, helperErr("could not retrieve 3p credentials", err)
	}

	numRegistries := len(all3pCreds) + len(config.SupportedGCRRegistries)
	var registries, usernames []string
	registries = make([]string, 0, numRegistries)
	usernames = make([]string, 0, numRegistries)

	for registry, creds := range all3pCreds {
		registries = append(registries, registry)
		usernames = append(usernames, creds.Username)
	}

	for gcrRegistry := range config.SupportedGCRRegistries {
		registries = append(registries, gcrRegistry)
		usernames = append(usernames, gcrOAuth2Username)
	}

	return registries, usernames, nil
}

// Add adds new third-party credentials to the keychain.
func (ch *gcrCredHelper) Add(creds *credentials.Credentials) error {
	serverURL := creds.ServerURL
	if isAGCRHostname(serverURL) {
		return helperErr("this operation is unsupported for GCR, please see docker-credential-gcr documentation for supported login methods", nil)
	}
	if err := ch.store.SetOtherCreds(creds); err != nil {
		return helperErr("could not store 3p credentials for "+serverURL, err)
	}
	return nil
}

// Delete removes third-party credentials from the store.
func (ch *gcrCredHelper) Delete(serverURL string) error {
	if isAGCRHostname(serverURL) {
		return helperErr("delete is unimplemented for GCR: "+serverURL, nil)
	}
	if err := ch.store.DeleteOtherCreds(serverURL); err != nil {
		return helperErr("could not delete 3p credentials for "+serverURL, err)
	}
	return nil
}

// Get returns the username and secret to use for a given registry server URL.
func (ch *gcrCredHelper) Get(serverURL string) (string, string, error) {
	if isAGCRHostname(serverURL) {
		// Return GCR's access token.
		accessToken, err := ch.getGCRAccessToken()
		if err != nil {
			return "", "", helperErr(fmt.Sprintf("could not retrieve %s's access token", serverURL), err)
		}
		return gcrOAuth2Username, accessToken, nil
	}

	// Attempt to retrieve credentials for another repository
	creds, err := ch.store.GetOtherCreds(serverURL)
	if err != nil {
		if credentials.IsErrCredentialsNotFound(err) {
			return "", "", err
		}
		return "", "", helperErr("could not retrieve 3p credentials for "+serverURL, err)
	}
	return creds.Username, creds.Secret, nil
}

// getGCRAccessToken attempts to generate an access_token from credentials
// from the helper's own credential store.
func (ch *gcrCredHelper) getGCRAccessToken() (string, error) {
	// First, look for the Application Default Credentials.
	// https://developers.google.com/identity/protocols/application-default-credentials
	token, err := ch.defaultCredsToken()
	if err != nil {
		// Second, attempt to retrieve credentials from the gcloud SDK's store.
		token, err = ch.gcloudSDKToken()
	}
	if err != nil {
		// Finally, retrieve credentials from our credential store.
		token, err = ch.credStoreToken(ch.store)
		if err != nil {
			return "", err
		}
	}

	return token, nil
}

// tokenFromAppDefaultCreds retrieves a gcloud
// access_token from the "Application Default Credentials".
func tokenFromAppDefaultCreds() (string, error) {
	/*
		From https://godoc.org/golang.org/x/oauth2/google:

		DefaultTokenSource is a token source that uses "Application Default Credentials".

		It looks for credentials in the following places, preferring the first location found:

		1. A JSON file whose path is specified by the
		   GOOGLE_APPLICATION_CREDENTIALS environment variable.
		2. A JSON file in a location known to the gcloud command-line tool.
		   On Windows, this is %APPDATA%/gcloud/application_default_credentials.json.
		   On other systems, $HOME/.config/gcloud/application_default_credentials.json.
		3. On Google App Engine it uses the appengine.AccessToken function.
		4. On Google Compute Engine and Google App Engine Managed VMs, it fetches
		   credentials from the metadata server.
		   (In this final case any provided scopes are ignored.)
	*/
	ts, err := google.DefaultTokenSource(config.OAuthHTTPContext, config.GCRScopes...)
	if err != nil {
		return "", err
	}

	token, err := ts.Token()
	if err != nil {
		return "", err
	}

	if !token.Valid() {
		return "", helperErr("token was invalid", nil)
	}

	if token.Type() != "Bearer" {
		return "", helperErr(fmt.Sprintf("expected token type \"Bearer\" but got \"%s\"", token.Type()), nil)
	}

	return token.AccessToken, nil
}

// tokenFromGcloudSDK attempts to generate an access_token using the gcloud SDK.
func tokenFromGcloudSDK() (string, error) {
	// shelling out to gcloud is the only currently supported way of
	// obtaining the gcloud access_token
	if _, err := exec.LookPath("gcloud"); err != nil {
		return "", helperErr("gcloud not found on PATH", nil)
	}

	cmd := exec.Command("gcloud", "auth", "print-access-token")

	var out bytes.Buffer
	cmd.Stdout = &out
	if err := cmd.Run(); err != nil {
		return "", helperErr("gcloud auth print-access-token failed", err)
	}

	token := strings.TrimSpace(out.String())
	if token == "" {
		return "", helperErr("gcloud auth print-access-token returned empty access_token", nil)
	}
	return token, nil
}

func tokenFromPrivateStore(store store.GCRCredStore) (string, error) {
	gcrAuth, err := store.GetGCRAuth()
	if err != nil {
		return "", err
	}
	ts := gcrAuth.TokenSource(config.OAuthHTTPContext)
	tok, err := ts.Token()
	if err != nil {
		return "", err
	}
	if !tok.Valid() {
		return "", helperErr("token was invalid", nil)
	}

	return tok.AccessToken, nil
}

// isAGCRHostname returns true if the given hostname is one of GCR's
func isAGCRHostname(serverURL string) bool {
	URL, err := url.Parse(serverURL)
	if err != nil {
		return false
	}
	return config.SupportedGCRRegistries[URL.Host] || config.SupportedGCRRegistries[serverURL]
}

func helperErr(message string, err error) error {
	if err == nil {
		return fmt.Errorf("docker-credential-gcr/helper: %s", message)
	}
	return fmt.Errorf("docker-credential-gcr/helper: %s: %v", message, err)
}
<commit_msg>Update error message for unsupported 'add's<commit_after>// Copyright 2016 Google, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/*
Package credhelper implements a Docker credential helper with special facilities
for GCR authentication.
*/
package credhelper

import (
	"bytes"
	"fmt"
	"net/url"
	"os/exec"
	"strings"

	"github.com/GoogleCloudPlatform/docker-credential-gcr/config"
	"github.com/GoogleCloudPlatform/docker-credential-gcr/store"
	"github.com/docker/docker-credential-helpers/credentials"

	"golang.org/x/oauth2/google"
)

const gcrOAuth2Username = "oauth2accesstoken"

// gcrCredHelper implements a credentials.Helper interface backed by a GCR
// credential store.
type gcrCredHelper struct {
	store store.GCRCredStore

	// helper methods, package exposed for testing
	defaultCredsToken func() (string, error)
	gcloudSDKToken    func() (string, error)
	credStoreToken    func(store.GCRCredStore) (string, error)
}

// NewGCRCredentialHelper returns a Docker credential helper which
// specializes in GCR's authentication schemes.
func NewGCRCredentialHelper(store store.GCRCredStore) credentials.Helper {
	return &gcrCredHelper{
		store:             store,
		credStoreToken:    tokenFromPrivateStore,
		gcloudSDKToken:    tokenFromGcloudSDK,
		defaultCredsToken: tokenFromAppDefaultCreds,
	}
}

// Delete lists all credentials stored and associated usernames.
func (ch *gcrCredHelper) List() ([]string, []string, error) {
	all3pCreds, err := ch.store.AllThirdPartyCreds()
	if err != nil {
		return nil, nil, helperErr("could not retrieve 3p credentials", err)
	}

	numRegistries := len(all3pCreds) + len(config.SupportedGCRRegistries)
	var registries, usernames []string
	registries = make([]string, 0, numRegistries)
	usernames = make([]string, 0, numRegistries)

	for registry, creds := range all3pCreds {
		registries = append(registries, registry)
		usernames = append(usernames, creds.Username)
	}

	for gcrRegistry := range config.SupportedGCRRegistries {
		registries = append(registries, gcrRegistry)
		usernames = append(usernames, gcrOAuth2Username)
	}

	return registries, usernames, nil
}

// Add adds new third-party credentials to the keychain.
func (ch *gcrCredHelper) Add(creds *credentials.Credentials) error {
	serverURL := creds.ServerURL
	if isAGCRHostname(serverURL) {
		errStr := "this operation is unsupported for GCR, please see docker-credential-gcr documentation for supported login methods. " +
			"'gcloud docker' is unnecessary when using docker-credential-gcr, use 'docker' instead."
		return helperErr(errStr, nil)
	}
	if err := ch.store.SetOtherCreds(creds); err != nil {
		return helperErr("could not store 3p credentials for "+serverURL, err)
	}
	return nil
}

// Delete removes third-party credentials from the store.
func (ch *gcrCredHelper) Delete(serverURL string) error {
	if isAGCRHostname(serverURL) {
		return helperErr("delete is unimplemented for GCR: "+serverURL, nil)
	}
	if err := ch.store.DeleteOtherCreds(serverURL); err != nil {
		return helperErr("could not delete 3p credentials for "+serverURL, err)
	}
	return nil
}

// Get returns the username and secret to use for a given registry server URL.
func (ch *gcrCredHelper) Get(serverURL string) (string, string, error) {
	if isAGCRHostname(serverURL) {
		// Return GCR's access token.
		accessToken, err := ch.getGCRAccessToken()
		if err != nil {
			return "", "", helperErr(fmt.Sprintf("could not retrieve %s's access token", serverURL), err)
		}
		return gcrOAuth2Username, accessToken, nil
	}

	// Attempt to retrieve credentials for another repository
	creds, err := ch.store.GetOtherCreds(serverURL)
	if err != nil {
		if credentials.IsErrCredentialsNotFound(err) {
			return "", "", err
		}
		return "", "", helperErr("could not retrieve 3p credentials for "+serverURL, err)
	}
	return creds.Username, creds.Secret, nil
}

// getGCRAccessToken attempts to generate an access_token from credentials
// from the helper's own credential store.
func (ch *gcrCredHelper) getGCRAccessToken() (string, error) {
	// First, look for the Application Default Credentials.
	// https://developers.google.com/identity/protocols/application-default-credentials
	token, err := ch.defaultCredsToken()
	if err != nil {
		// Second, attempt to retrieve credentials from the gcloud SDK's store.
		token, err = ch.gcloudSDKToken()
	}
	if err != nil {
		// Finally, retrieve credentials from our credential store.
		token, err = ch.credStoreToken(ch.store)
		if err != nil {
			return "", err
		}
	}

	return token, nil
}

// tokenFromAppDefaultCreds retrieves a gcloud
// access_token from the "Application Default Credentials".
func tokenFromAppDefaultCreds() (string, error) {
	/*
		From https://godoc.org/golang.org/x/oauth2/google:

		DefaultTokenSource is a token source that uses "Application Default Credentials".

		It looks for credentials in the following places, preferring the first location found:

		1. A JSON file whose path is specified by the
		   GOOGLE_APPLICATION_CREDENTIALS environment variable.
		2. A JSON file in a location known to the gcloud command-line tool.
		   On Windows, this is %APPDATA%/gcloud/application_default_credentials.json.
		   On other systems, $HOME/.config/gcloud/application_default_credentials.json.
		3. On Google App Engine it uses the appengine.AccessToken function.
		4. On Google Compute Engine and Google App Engine Managed VMs, it fetches
		   credentials from the metadata server.
		   (In this final case any provided scopes are ignored.)
	*/
	ts, err := google.DefaultTokenSource(config.OAuthHTTPContext, config.GCRScopes...)
	if err != nil {
		return "", err
	}

	token, err := ts.Token()
	if err != nil {
		return "", err
	}

	if !token.Valid() {
		return "", helperErr("token was invalid", nil)
	}

	if token.Type() != "Bearer" {
		return "", helperErr(fmt.Sprintf("expected token type \"Bearer\" but got \"%s\"", token.Type()), nil)
	}

	return token.AccessToken, nil
}

// tokenFromGcloudSDK attempts to generate an access_token using the gcloud SDK.
func tokenFromGcloudSDK() (string, error) {
	// shelling out to gcloud is the only currently supported way of
	// obtaining the gcloud access_token
	if _, err := exec.LookPath("gcloud"); err != nil {
		return "", helperErr("gcloud not found on PATH", nil)
	}

	cmd := exec.Command("gcloud", "auth", "print-access-token")

	var out bytes.Buffer
	cmd.Stdout = &out
	if err := cmd.Run(); err != nil {
		return "", helperErr("gcloud auth print-access-token failed", err)
	}

	token := strings.TrimSpace(out.String())
	if token == "" {
		return "", helperErr("gcloud auth print-access-token returned empty access_token", nil)
	}
	return token, nil
}

func tokenFromPrivateStore(store store.GCRCredStore) (string, error) {
	gcrAuth, err := store.GetGCRAuth()
	if err != nil {
		return "", err
	}
	ts := gcrAuth.TokenSource(config.OAuthHTTPContext)
	tok, err := ts.Token()
	if err != nil {
		return "", err
	}
	if !tok.Valid() {
		return "", helperErr("token was invalid", nil)
	}

	return tok.AccessToken, nil
}

// isAGCRHostname returns true if the given hostname is one of GCR's
func isAGCRHostname(serverURL string) bool {
	URL, err := url.Parse(serverURL)
	if err != nil {
		return false
	}
	return config.SupportedGCRRegistries[URL.Host] || config.SupportedGCRRegistries[serverURL]
}

func helperErr(message string, err error) error {
	if err == nil {
		return fmt.Errorf("docker-credential-gcr/helper: %s", message)
	}
	return fmt.Errorf("docker-credential-gcr/helper: %s: %v", message, err)
}
<|endoftext|>


<commit_before>package rpc

import (
	"encoding/binary"
	"log"
	"net"
	"net/rpc"
	"strings"

	"github.com/yosisa/craft/mux"
)

func AllocStream(c *rpc.Client, addr string) (id uint32, conn net.Conn, err error) {
	var resp AllocResponse
	if err = c.Call("StreamConn.Alloc", Empty{}, &resp); err != nil {
		return
	}
	id = resp.ID
	if conn, err = mux.DialTimeout("tcp", addr, chanNewStream, dialTimeout); err != nil {
		return
	}
	err = binary.Write(conn, binary.BigEndian, id)
	return
}

func StartContainer(addrs []string, container string) error {
	CallAll(addrs, func(c *rpc.Client, addr string) (interface{}, error) {
		err := c.Call("Docker.StartContainer", container, &Empty{})
		if err != nil && strings.Contains(err.Error(), "No such container") {
			err = nil
		}
		return nil, err
	})
	return nil
}

func StopContainer(addrs []string, container string, timeout uint) error {
	CallAll(addrs, func(c *rpc.Client, addr string) (interface{}, error) {
		req := StopContainerRequest{ID: container, Timeout: timeout}
		err := c.Call("Docker.StopContainer", req, &Empty{})
		if err != nil && strings.Contains(err.Error(), "No such container") {
			err = nil
		}
		return nil, err
	})
	return nil
}

func RemoveContainer(addrs []string, container string, force bool) error {
	CallAll(addrs, func(c *rpc.Client, addr string) (interface{}, error) {
		req := RemoveContainerRequest{ID: container, Force: force}
		err := c.Call("Docker.RemoveContainer", req, &Empty{})
		if err != nil && strings.HasPrefix(err.Error(), "No such container:") {
			err = nil
		}
		return nil, err
	})
	return nil
}

func PullImage(addrs []string, image string) error {
	p := newProgress()
	go p.show()
	CallAll(addrs, func(c *rpc.Client, addr string) (interface{}, error) {
		id, sc, err := AllocStream(c, addr)
		if err == nil {
			p.add(sc, addr)
		} else {
			log.Print(err)
		}
		req := PullImageRequest{Image: image, StreamID: id}
		var resp Empty
		err = c.Call("Docker.PullImage", req, &resp)
		return nil, err
	})
	p.wait()
	return nil
}
<commit_msg>Remove duplicated code<commit_after>package rpc

import (
	"encoding/binary"
	"log"
	"net"
	"net/rpc"
	"strings"

	"github.com/yosisa/craft/mux"
)

func AllocStream(c *rpc.Client, addr string) (id uint32, conn net.Conn, err error) {
	var resp AllocResponse
	if err = c.Call("StreamConn.Alloc", Empty{}, &resp); err != nil {
		return
	}
	id = resp.ID
	if conn, err = mux.DialTimeout("tcp", addr, chanNewStream, dialTimeout); err != nil {
		return
	}
	err = binary.Write(conn, binary.BigEndian, id)
	return
}

func StartContainer(addrs []string, container string) error {
	CallAll(addrs, func(c *rpc.Client, addr string) (interface{}, error) {
		err := safeError(c.Call("Docker.StartContainer", container, &Empty{}))
		return nil, err
	})
	return nil
}

func StopContainer(addrs []string, container string, timeout uint) error {
	CallAll(addrs, func(c *rpc.Client, addr string) (interface{}, error) {
		req := StopContainerRequest{ID: container, Timeout: timeout}
		err := safeError(c.Call("Docker.StopContainer", req, &Empty{}))
		return nil, err
	})
	return nil
}

func RemoveContainer(addrs []string, container string, force bool) error {
	CallAll(addrs, func(c *rpc.Client, addr string) (interface{}, error) {
		req := RemoveContainerRequest{ID: container, Force: force}
		err := safeError(c.Call("Docker.RemoveContainer", req, &Empty{}))
		return nil, err
	})
	return nil
}

func PullImage(addrs []string, image string) error {
	p := newProgress()
	go p.show()
	CallAll(addrs, func(c *rpc.Client, addr string) (interface{}, error) {
		id, sc, err := AllocStream(c, addr)
		if err == nil {
			p.add(sc, addr)
		} else {
			log.Print(err)
		}
		req := PullImageRequest{Image: image, StreamID: id}
		var resp Empty
		err = c.Call("Docker.PullImage", req, &resp)
		return nil, err
	})
	p.wait()
	return nil
}

func safeError(err error) error {
	if err != nil && strings.Contains(err.Error(), "No such container") {
		return nil
	}
	return err
}
<|endoftext|>


<commit_before>package scheduler

import (
	"encoding/json"
	"io"
	"net/http"
	"strconv"
	"sync"
	"time"

	"github.com/concourse/atc/db"
	"github.com/concourse/turbine"
	"github.com/concourse/turbine/event"
	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/rata"
	"github.com/vito/go-sse/sse"
)

type tracker struct {
	logger lager.Logger

	db     TrackerDB
	locker Locker

	trackingBuilds map[int]bool
	lock           *sync.Mutex
}

type TrackerDB interface {
	GetLastBuildEventID(buildID int) (int, error)
	SaveBuildEvent(buildID int, event db.BuildEvent) error

	SaveBuildStartTime(buildID int, startTime time.Time) error
	SaveBuildEndTime(buildID int, startTime time.Time) error

	SaveBuildInput(buildID int, vr db.VersionedResource) error
	SaveBuildOutput(buildID int, vr db.VersionedResource) error

	SaveBuildStatus(buildID int, status db.Status) error
}

func NewTracker(logger lager.Logger, db TrackerDB, locker Locker) BuildTracker {
	return &tracker{
		logger: logger,
		db:     db,
		locker: locker,

		trackingBuilds: make(map[int]bool),
		lock:           new(sync.Mutex),
	}
}

func (tracker *tracker) TrackBuild(build db.Build) error {
	lock, err := tracker.locker.AcquireWriteLockImmediately([]db.NamedLock{db.BuildTrackingLock(build.Guid)})
	if err != nil {
		return nil
	}
	defer lock.Release()

	tLog := tracker.logger.Session("track-build", lager.Data{
		"buld": build.ID,
	})

	alreadyTracking := tracker.markTracking(build.ID)
	if alreadyTracking {
		tLog.Info("already-tracking")
		return nil
	}

	tLog.Info("tracking")

	defer func() {
		tLog.Info("done-tracking")
		tracker.unmarkTracking(build.ID)
	}()

	generator := rata.NewRequestGenerator(build.Endpoint, turbine.Routes)

	events, err := generator.CreateRequest(
		turbine.GetBuildEvents,
		rata.Params{"guid": build.Guid},
		nil,
	)
	if err != nil {
		tLog.Error("failed-to-create-events-request", err)
		return err
	}

	resp, err := http.DefaultClient.Do(events)
	if err != nil {
		tLog.Error("failed-to-get-events", err)
		return err
	}

	defer resp.Body.Close()

	if resp.StatusCode == http.StatusNotFound {
		tLog.Info("saving-orphaned-build-as-errored")

		err := tracker.db.SaveBuildStatus(build.ID, db.StatusErrored)
		if err != nil {
			tLog.Error("failed-to-save-build-as-errored", err)
			return err
		}

		return nil
	}

	reader := sse.NewReader(resp.Body)

	outputs := map[string]db.VersionedResource{}

	var currentVersion string

	for {
		se, err := reader.Next()
		if err != nil {
			if err == io.EOF {
				tLog.Info("event-stream-completed")
				return nil
			}

			return err
		}

		id, err := strconv.Atoi(se.ID)
		if err != nil {
			tLog.Error("non-numerical-event-id", err, lager.Data{
				"id": se.ID,
			})

			return err
		}

		err = tracker.db.SaveBuildEvent(build.ID, db.BuildEvent{
			ID:      id,
			Type:    se.Name,
			Payload: string(se.Data),
		})
		if err != nil {
			tLog.Error("failed-to-save-build-event", err, lager.Data{
				"event": se,
			})

			return err
		}

		if se.Name == "version" {
			var version event.Version
			err := json.Unmarshal(se.Data, &version)
			if err != nil {
				tLog.Error("failed-to-unmarshal-version", err)
				return err
			}

			tLog.Info("event-stream-version", lager.Data{
				"version": string(version),
			})

			currentVersion = string(version)
			continue
		}

		if se.Name == "end" {
			tLog.Info("event-stream-ended")

			del, err := generator.CreateRequest(
				turbine.DeleteBuild,
				rata.Params{"guid": build.Guid},
				nil,
			)
			if err != nil {
				tLog.Error("failed-to-create-delete-request", err)
				return err
			}

			resp, err := http.DefaultClient.Do(del)
			if err != nil {
				tLog.Error("failed-to-delete-build", err)
				return err
			}

			resp.Body.Close()
			continue
		}

		switch currentVersion {
		case "1.0":
			fallthrough
		case "1.1":
			switch se.Name {
			case "status":
				tLog.Info("processing-build-status", lager.Data{
					"event": string(se.Data),
				})

				var status event.Status
				err := json.Unmarshal(se.Data, &status)
				if err != nil {
					tLog.Error("failed-to-unmarshal-status", err)
					return err
				}

				if status.Status == turbine.StatusStarted {
					err = tracker.db.SaveBuildStartTime(build.ID, time.Unix(status.Time, 0))
					if err != nil {
						tLog.Error("failed-to-save-build-start-time", err)
						return err
					}
				} else {
					err = tracker.db.SaveBuildEndTime(build.ID, time.Unix(status.Time, 0))
					if err != nil {
						tLog.Error("failed-to-save-build-end-time", err)
						return err
					}
				}

				err = tracker.db.SaveBuildStatus(build.ID, db.Status(status.Status))
				if err != nil {
					tLog.Error("failed-to-save-build-status", err)
					return err
				}

				if status.Status == turbine.StatusSucceeded {
					for _, output := range outputs {
						err := tracker.db.SaveBuildOutput(build.ID, output)
						if err != nil {
							tLog.Error("failed-to-save-build-output", err)
							return err
						}
					}
				}

			case "input":
				if build.JobName == "" {
					tLog.Info("ignoring-build-input-for-one-off")
					break
				}

				tLog.Info("processing-build-input", lager.Data{
					"event": string(se.Data),
				})

				var input event.Input
				err := json.Unmarshal(se.Data, &input)
				if err != nil {
					tLog.Error("failed-to-unarshal-input", err)
					return err
				}

				if input.Input.Resource == "" {
					input.Input.Resource = input.Input.Name
				}

				err = tracker.db.SaveBuildInput(build.ID, vrFromInput(input.Input))
				if err != nil {
					tLog.Error("failed-to-save-build-input", err)
					return err
				}

				// record implicit output
				outputs[input.Input.Resource] = vrFromInput(input.Input)

			case "output":
				if build.JobName == "" {
					tLog.Info("ignoring-build-output-for-one-off")
					break
				}

				var output event.Output
				err := json.Unmarshal(se.Data, &output)
				if err != nil {
					tLog.Error("failed-to-unarshal-output", err)
					return err
				}

				outputs[output.Output.Name] = vrFromOutput(output.Output)
			}
		}
	}

	return nil
}

func (tracker *tracker) markTracking(buildID int) bool {
	tracker.lock.Lock()
	alreadyTracking, found := tracker.trackingBuilds[buildID]
	if !found {
		tracker.trackingBuilds[buildID] = true
	}
	tracker.lock.Unlock()

	return alreadyTracking
}

func (tracker *tracker) unmarkTracking(buildID int) {
	tracker.lock.Lock()
	delete(tracker.trackingBuilds, buildID)
	tracker.lock.Unlock()
}

func vrFromInput(input turbine.Input) db.VersionedResource {
	metadata := make([]db.MetadataField, len(input.Metadata))
	for i, md := range input.Metadata {
		metadata[i] = db.MetadataField{
			Name:  md.Name,
			Value: md.Value,
		}
	}

	return db.VersionedResource{
		Name:     input.Resource,
		Type:     input.Type,
		Source:   db.Source(input.Source),
		Version:  db.Version(input.Version),
		Metadata: metadata,
	}
}

// same as input, but type is different.
//
// :(
func vrFromOutput(output turbine.Output) db.VersionedResource {
	metadata := make([]db.MetadataField, len(output.Metadata))
	for i, md := range output.Metadata {
		metadata[i] = db.MetadataField{
			Name:  md.Name,
			Value: md.Value,
		}
	}

	return db.VersionedResource{
		Name:     output.Name,
		Type:     output.Type,
		Source:   db.Source(output.Source),
		Version:  db.Version(output.Version),
		Metadata: metadata,
	}
}
<commit_msg>fix log message typo<commit_after>package scheduler

import (
	"encoding/json"
	"io"
	"net/http"
	"strconv"
	"sync"
	"time"

	"github.com/concourse/atc/db"
	"github.com/concourse/turbine"
	"github.com/concourse/turbine/event"
	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/rata"
	"github.com/vito/go-sse/sse"
)

type tracker struct {
	logger lager.Logger

	db     TrackerDB
	locker Locker

	trackingBuilds map[int]bool
	lock           *sync.Mutex
}

type TrackerDB interface {
	GetLastBuildEventID(buildID int) (int, error)
	SaveBuildEvent(buildID int, event db.BuildEvent) error

	SaveBuildStartTime(buildID int, startTime time.Time) error
	SaveBuildEndTime(buildID int, startTime time.Time) error

	SaveBuildInput(buildID int, vr db.VersionedResource) error
	SaveBuildOutput(buildID int, vr db.VersionedResource) error

	SaveBuildStatus(buildID int, status db.Status) error
}

func NewTracker(logger lager.Logger, db TrackerDB, locker Locker) BuildTracker {
	return &tracker{
		logger: logger,
		db:     db,
		locker: locker,

		trackingBuilds: make(map[int]bool),
		lock:           new(sync.Mutex),
	}
}

func (tracker *tracker) TrackBuild(build db.Build) error {
	lock, err := tracker.locker.AcquireWriteLockImmediately([]db.NamedLock{db.BuildTrackingLock(build.Guid)})
	if err != nil {
		return nil
	}
	defer lock.Release()

	tLog := tracker.logger.Session("track-build", lager.Data{
		"build": build.ID,
	})

	alreadyTracking := tracker.markTracking(build.ID)
	if alreadyTracking {
		tLog.Info("already-tracking")
		return nil
	}

	tLog.Info("tracking")

	defer func() {
		tLog.Info("done-tracking")
		tracker.unmarkTracking(build.ID)
	}()

	generator := rata.NewRequestGenerator(build.Endpoint, turbine.Routes)

	events, err := generator.CreateRequest(
		turbine.GetBuildEvents,
		rata.Params{"guid": build.Guid},
		nil,
	)
	if err != nil {
		tLog.Error("failed-to-create-events-request", err)
		return err
	}

	resp, err := http.DefaultClient.Do(events)
	if err != nil {
		tLog.Error("failed-to-get-events", err)
		return err
	}

	defer resp.Body.Close()

	if resp.StatusCode == http.StatusNotFound {
		tLog.Info("saving-orphaned-build-as-errored")

		err := tracker.db.SaveBuildStatus(build.ID, db.StatusErrored)
		if err != nil {
			tLog.Error("failed-to-save-build-as-errored", err)
			return err
		}

		return nil
	}

	reader := sse.NewReader(resp.Body)

	outputs := map[string]db.VersionedResource{}

	var currentVersion string

	for {
		se, err := reader.Next()
		if err != nil {
			if err == io.EOF {
				tLog.Info("event-stream-completed")
				return nil
			}

			return err
		}

		id, err := strconv.Atoi(se.ID)
		if err != nil {
			tLog.Error("non-numerical-event-id", err, lager.Data{
				"id": se.ID,
			})

			return err
		}

		err = tracker.db.SaveBuildEvent(build.ID, db.BuildEvent{
			ID:      id,
			Type:    se.Name,
			Payload: string(se.Data),
		})
		if err != nil {
			tLog.Error("failed-to-save-build-event", err, lager.Data{
				"event": se,
			})

			return err
		}

		if se.Name == "version" {
			var version event.Version
			err := json.Unmarshal(se.Data, &version)
			if err != nil {
				tLog.Error("failed-to-unmarshal-version", err)
				return err
			}

			tLog.Info("event-stream-version", lager.Data{
				"version": string(version),
			})

			currentVersion = string(version)
			continue
		}

		if se.Name == "end" {
			tLog.Info("event-stream-ended")

			del, err := generator.CreateRequest(
				turbine.DeleteBuild,
				rata.Params{"guid": build.Guid},
				nil,
			)
			if err != nil {
				tLog.Error("failed-to-create-delete-request", err)
				return err
			}

			resp, err := http.DefaultClient.Do(del)
			if err != nil {
				tLog.Error("failed-to-delete-build", err)
				return err
			}

			resp.Body.Close()
			continue
		}

		switch currentVersion {
		case "1.0":
			fallthrough
		case "1.1":
			switch se.Name {
			case "status":
				tLog.Info("processing-build-status", lager.Data{
					"event": string(se.Data),
				})

				var status event.Status
				err := json.Unmarshal(se.Data, &status)
				if err != nil {
					tLog.Error("failed-to-unmarshal-status", err)
					return err
				}

				if status.Status == turbine.StatusStarted {
					err = tracker.db.SaveBuildStartTime(build.ID, time.Unix(status.Time, 0))
					if err != nil {
						tLog.Error("failed-to-save-build-start-time", err)
						return err
					}
				} else {
					err = tracker.db.SaveBuildEndTime(build.ID, time.Unix(status.Time, 0))
					if err != nil {
						tLog.Error("failed-to-save-build-end-time", err)
						return err
					}
				}

				err = tracker.db.SaveBuildStatus(build.ID, db.Status(status.Status))
				if err != nil {
					tLog.Error("failed-to-save-build-status", err)
					return err
				}

				if status.Status == turbine.StatusSucceeded {
					for _, output := range outputs {
						err := tracker.db.SaveBuildOutput(build.ID, output)
						if err != nil {
							tLog.Error("failed-to-save-build-output", err)
							return err
						}
					}
				}

			case "input":
				if build.JobName == "" {
					tLog.Info("ignoring-build-input-for-one-off")
					break
				}

				tLog.Info("processing-build-input", lager.Data{
					"event": string(se.Data),
				})

				var input event.Input
				err := json.Unmarshal(se.Data, &input)
				if err != nil {
					tLog.Error("failed-to-unarshal-input", err)
					return err
				}

				if input.Input.Resource == "" {
					input.Input.Resource = input.Input.Name
				}

				err = tracker.db.SaveBuildInput(build.ID, vrFromInput(input.Input))
				if err != nil {
					tLog.Error("failed-to-save-build-input", err)
					return err
				}

				// record implicit output
				outputs[input.Input.Resource] = vrFromInput(input.Input)

			case "output":
				if build.JobName == "" {
					tLog.Info("ignoring-build-output-for-one-off")
					break
				}

				var output event.Output
				err := json.Unmarshal(se.Data, &output)
				if err != nil {
					tLog.Error("failed-to-unarshal-output", err)
					return err
				}

				outputs[output.Output.Name] = vrFromOutput(output.Output)
			}
		}
	}

	return nil
}

func (tracker *tracker) markTracking(buildID int) bool {
	tracker.lock.Lock()
	alreadyTracking, found := tracker.trackingBuilds[buildID]
	if !found {
		tracker.trackingBuilds[buildID] = true
	}
	tracker.lock.Unlock()

	return alreadyTracking
}

func (tracker *tracker) unmarkTracking(buildID int) {
	tracker.lock.Lock()
	delete(tracker.trackingBuilds, buildID)
	tracker.lock.Unlock()
}

func vrFromInput(input turbine.Input) db.VersionedResource {
	metadata := make([]db.MetadataField, len(input.Metadata))
	for i, md := range input.Metadata {
		metadata[i] = db.MetadataField{
			Name:  md.Name,
			Value: md.Value,
		}
	}

	return db.VersionedResource{
		Name:     input.Resource,
		Type:     input.Type,
		Source:   db.Source(input.Source),
		Version:  db.Version(input.Version),
		Metadata: metadata,
	}
}

// same as input, but type is different.
//
// :(
func vrFromOutput(output turbine.Output) db.VersionedResource {
	metadata := make([]db.MetadataField, len(output.Metadata))
	for i, md := range output.Metadata {
		metadata[i] = db.MetadataField{
			Name:  md.Name,
			Value: md.Value,
		}
	}

	return db.VersionedResource{
		Name:     output.Name,
		Type:     output.Type,
		Source:   db.Source(output.Source),
		Version:  db.Version(output.Version),
		Metadata: metadata,
	}
}
<|endoftext|>


<commit_before>package integration_test

import (
	"encoding/json"
	"fmt"
	"os"
	"testing"
	"time"

	"github.com/cloudfoundry/gunk/natsrunner"
	"github.com/cloudfoundry/gunk/timeprovider"
	"github.com/cloudfoundry/yagnats"
	"github.com/pivotal-golang/lager/lagertest"
	"github.com/tedsuo/ifrit"

	Bbs "github.com/cloudfoundry-incubator/runtime-schema/bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/cc_messages"
	"github.com/cloudfoundry-incubator/stager/integration/stager_runner"
	"github.com/cloudfoundry/storeadapter/storerunner/etcdstorerunner"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
)

var stagerPath string
var etcdRunner *etcdstorerunner.ETCDClusterRunner
var natsRunner *natsrunner.NATSRunner
var runner *stager_runner.StagerRunner

var _ = Describe("Main", func() {
	var (
		natsClient        yagnats.NATSClient
		bbs               *Bbs.BBS
		fileServerProcess ifrit.Process
	)

	BeforeEach(func() {
		etcdPort := 5001 + GinkgoParallelNode()
		natsPort := 4001 + GinkgoParallelNode()

		etcdRunner = etcdstorerunner.NewETCDClusterRunner(etcdPort, 1)
		etcdRunner.Start()

		natsRunner = natsrunner.NewNATSRunner(natsPort)
		natsRunner.Start()

		natsClient = natsRunner.MessageBus

		bbs = Bbs.NewBBS(etcdRunner.Adapter(), timeprovider.NewTimeProvider(), lagertest.NewTestLogger("test"))

		fileServerProcess = ifrit.Envoke(bbs.NewFileServerHeartbeat("http://example.com", "file-server-id", time.Second))

		runner = stager_runner.New(
			stagerPath,
			[]string{fmt.Sprintf("http://127.0.0.1:%d", etcdPort)},
			[]string{fmt.Sprintf("127.0.0.1:%d", natsPort)},
		)
	})

	AfterEach(func(done Done) {
		runner.Stop()
		fileServerProcess.Signal(os.Kill)
		etcdRunner.Stop()
		natsRunner.Stop()
		close(done)
	}, 10.0)

	Context("when started", func() {
		BeforeEach(func() {
			runner.Start("--circuses", `{"lucid64":"lifecycle.zip"}`, "--minDiskMB", "2048", "--minMemoryMB", "256", "--minFileDescriptors", "2")
		})

		Describe("when a 'diego.staging.start' message is recieved", func() {
			BeforeEach(func() {
				natsClient.Publish("diego.staging.start", []byte(`
				      {
				        "app_id":"my-app-guid",
                "task_id":"my-task-guid",
                "stack":"lucid64",
                "app_bits_download_uri":"http://example.com/app_bits",
                "file_descriptors":3,
                "memory_mb" : 1024,
                "disk_mb" : 128,
                "buildpacks" : [],
                "environment" : []
				      }
				    `))
			})

			It("desires a staging task via the BBS", func() {
				Eventually(bbs.GetAllPendingTasks, 1.0).Should(HaveLen(1))
				tasks, err := bbs.GetAllPendingTasks()
				Ω(err).ShouldNot(HaveOccurred())
				Ω(tasks[0].MemoryMB).Should(Equal(1024))
				Ω(tasks[0].DiskMB).Should(Equal(2048))
			})

			It("does not exit", func() {
				Consistently(runner.Session()).ShouldNot(gexec.Exit())
			})
		})

		Describe("when a 'diego.docker.staging.start' message is recieved", func() {
			var stagingFinished chan cc_messages.DockerStagingResponseForCC

			BeforeEach(func() {
				// local var to prevent data race with callback
				finished := make(chan cc_messages.DockerStagingResponseForCC, 1)

				stagingFinished = finished

				natsClient.Subscribe("diego.docker.staging.finished", func(msg *yagnats.Message) {
					stagingMsg := cc_messages.DockerStagingResponseForCC{}
					err := json.Unmarshal(msg.Payload, &stagingMsg)
					Ω(err).ShouldNot(HaveOccurred())
					finished <- stagingMsg
				})

				natsClient.Publish("diego.docker.staging.start", []byte(`
				      {
				        "app_id":"my-app-guid",
                "task_id":"my-task-guid",
                "stack":"lucid64",
                "docker_image_url":"http://docker.docker/docker",
                "file_descriptors":3,
                "memory_mb" : 1024,
                "disk_mb" : 128,
                "environment" : []
				      }
				    `))
			})

			It("sends a docker staging finished NATS message", func() {
				expectedMsg := cc_messages.DockerStagingResponseForCC{
					AppId:  "my-app-guid",
					TaskId: "my-task-guid",
				}
				Eventually(stagingFinished).Should(Receive(&expectedMsg))
			})

			It("does not exit", func() {
				Consistently(runner.Session()).ShouldNot(gexec.Exit())
			})
		})
	})
})

func TestStagerMain(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Integration Suite")
}

var _ = BeforeSuite(func() {
	var err error
	stagerPath, err = gexec.Build("github.com/cloudfoundry-incubator/stager", "-race")
	Ω(err).ShouldNot(HaveOccurred())
})

var _ = AfterSuite(func() {
	gexec.CleanupBuildArtifacts()
	if etcdRunner != nil {
		etcdRunner.Stop()
	}
	if natsRunner != nil {
		natsRunner.Stop()
	}
	if runner != nil {
		runner.Stop()
	}
})
<commit_msg>optimize integration suite: only build assets on node 1<commit_after>package integration_test

import (
	"encoding/json"
	"fmt"
	"os"
	"testing"
	"time"

	"github.com/cloudfoundry/gunk/natsrunner"
	"github.com/cloudfoundry/gunk/timeprovider"
	"github.com/cloudfoundry/yagnats"
	"github.com/pivotal-golang/lager/lagertest"
	"github.com/tedsuo/ifrit"

	Bbs "github.com/cloudfoundry-incubator/runtime-schema/bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/cc_messages"
	"github.com/cloudfoundry-incubator/stager/integration/stager_runner"
	"github.com/cloudfoundry/storeadapter/storerunner/etcdstorerunner"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
)

var stagerPath string
var etcdRunner *etcdstorerunner.ETCDClusterRunner
var natsRunner *natsrunner.NATSRunner
var runner *stager_runner.StagerRunner

var _ = Describe("Main", func() {
	var (
		natsClient        yagnats.NATSClient
		bbs               *Bbs.BBS
		fileServerProcess ifrit.Process
	)

	BeforeEach(func() {
		etcdPort := 5001 + GinkgoParallelNode()
		natsPort := 4001 + GinkgoParallelNode()

		etcdRunner = etcdstorerunner.NewETCDClusterRunner(etcdPort, 1)
		etcdRunner.Start()

		natsRunner = natsrunner.NewNATSRunner(natsPort)
		natsRunner.Start()

		natsClient = natsRunner.MessageBus

		bbs = Bbs.NewBBS(etcdRunner.Adapter(), timeprovider.NewTimeProvider(), lagertest.NewTestLogger("test"))

		fileServerProcess = ifrit.Envoke(bbs.NewFileServerHeartbeat("http://example.com", "file-server-id", time.Second))

		runner = stager_runner.New(
			stagerPath,
			[]string{fmt.Sprintf("http://127.0.0.1:%d", etcdPort)},
			[]string{fmt.Sprintf("127.0.0.1:%d", natsPort)},
		)
	})

	AfterEach(func(done Done) {
		runner.Stop()
		fileServerProcess.Signal(os.Kill)
		etcdRunner.Stop()
		natsRunner.Stop()
		close(done)
	}, 10.0)

	Context("when started", func() {
		BeforeEach(func() {
			runner.Start("--circuses", `{"lucid64":"lifecycle.zip"}`, "--minDiskMB", "2048", "--minMemoryMB", "256", "--minFileDescriptors", "2")
		})

		Describe("when a 'diego.staging.start' message is recieved", func() {
			BeforeEach(func() {
				natsClient.Publish("diego.staging.start", []byte(`
				      {
				        "app_id":"my-app-guid",
                "task_id":"my-task-guid",
                "stack":"lucid64",
                "app_bits_download_uri":"http://example.com/app_bits",
                "file_descriptors":3,
                "memory_mb" : 1024,
                "disk_mb" : 128,
                "buildpacks" : [],
                "environment" : []
				      }
				    `))
			})

			It("desires a staging task via the BBS", func() {
				Eventually(bbs.GetAllPendingTasks, 1.0).Should(HaveLen(1))
				tasks, err := bbs.GetAllPendingTasks()
				Ω(err).ShouldNot(HaveOccurred())
				Ω(tasks[0].MemoryMB).Should(Equal(1024))
				Ω(tasks[0].DiskMB).Should(Equal(2048))
			})

			It("does not exit", func() {
				Consistently(runner.Session()).ShouldNot(gexec.Exit())
			})
		})

		Describe("when a 'diego.docker.staging.start' message is recieved", func() {
			var stagingFinished chan cc_messages.DockerStagingResponseForCC

			BeforeEach(func() {
				// local var to prevent data race with callback
				finished := make(chan cc_messages.DockerStagingResponseForCC, 1)

				stagingFinished = finished

				natsClient.Subscribe("diego.docker.staging.finished", func(msg *yagnats.Message) {
					stagingMsg := cc_messages.DockerStagingResponseForCC{}
					err := json.Unmarshal(msg.Payload, &stagingMsg)
					Ω(err).ShouldNot(HaveOccurred())
					finished <- stagingMsg
				})

				natsClient.Publish("diego.docker.staging.start", []byte(`
				      {
				        "app_id":"my-app-guid",
                "task_id":"my-task-guid",
                "stack":"lucid64",
                "docker_image_url":"http://docker.docker/docker",
                "file_descriptors":3,
                "memory_mb" : 1024,
                "disk_mb" : 128,
                "environment" : []
				      }
				    `))
			})

			It("sends a docker staging finished NATS message", func() {
				expectedMsg := cc_messages.DockerStagingResponseForCC{
					AppId:  "my-app-guid",
					TaskId: "my-task-guid",
				}
				Eventually(stagingFinished).Should(Receive(&expectedMsg))
			})

			It("does not exit", func() {
				Consistently(runner.Session()).ShouldNot(gexec.Exit())
			})
		})
	})
})

func TestStagerMain(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Integration Suite")
}

var _ = SynchronizedBeforeSuite(func() []byte {
	stager, err := gexec.Build("github.com/cloudfoundry-incubator/stager", "-race")
	Ω(err).ShouldNot(HaveOccurred())
	return []byte(stager)
}, func(stager []byte) {
	stagerPath = string(stager)
})

var _ = SynchronizedAfterSuite(func() {
	if etcdRunner != nil {
		etcdRunner.Stop()
	}
	if natsRunner != nil {
		natsRunner.Stop()
	}
	if runner != nil {
		runner.Stop()
	}
}, func() {
	gexec.CleanupBuildArtifacts()
})
<|endoftext|>


<commit_before>// Copyright 2016 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build js

package clock

import (
	"syscall/js"
	"time"
)

func now() int64 {
	// time.Now() is not reliable until GopherJS supports performance.now().
	//
	// performance.now is monotonic:
	// https://www.w3.org/TR/hr-time-2/#sec-monotonic-clock
	return int64(js.Global().Get("performance").Call("now").Float() * float64(time.Millisecond))
}
<commit_msg>clock: Avoid calling (js.Value).Get() for performance<commit_after>// Copyright 2016 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build js

package clock

import (
	"syscall/js"
	"time"
)

var (
	jsPerformance = js.Global().Get("performance")
	jsNow         = jsPerformance.Get("now").Call("bind", jsPerformance)
)

func now() int64 {
	// time.Now() is not reliable until GopherJS supports performance.now().
	//
	// performance.now is monotonic:
	// https://www.w3.org/TR/hr-time-2/#sec-monotonic-clock
	return int64(jsNow.Invoke().Float() * float64(time.Millisecond))
}
<|endoftext|>


<commit_before>package main

import (
        "./comms"
        "./EncryptionEngine"

        "flag"

        "net"
        "fmt"
        "io"

        "net/http"
        "net/url"
        "html"
        "log"
       )

type State struct {
    Kc [16]byte
    clk uint32
    BD_ADDR [6]byte
}

func server_main(s *State) net.Conn {
    ln, err := net.Listen("tcp", ":8080")

    if err != nil {
        fmt.Println("net.Listen failed:", err)
    }

    conn, err := ln.Accept()

    fmt.Printf("Accepted Connection\n")

    if err != nil {
        fmt.Println("ln.Accept failed:", err)
    }

    s.BD_ADDR = [6]byte{}
    s.Kc = [16]byte{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}
    s.clk = 0

    comms.Send_neg(conn, s.BD_ADDR)
    comms.Send_init(conn, s.clk, [16]byte{}, s.Kc)

    return conn
}

func client_main() net.Conn {
    conn, err := net.Dial("tcp", "127.0.0.1:8080")

    if err != nil {
        fmt.Println("net.Dial failed:", err)
    }

    return conn
}

func receiver(conn io.Reader, s *State) {
  LOOP:
  for {
        packet_type := comms.Recv_packet(conn)

        switch packet_type {
            case 0:
                s.BD_ADDR = comms.Recv_neg(conn)
            case 1:
                s.clk, _, s.Kc = comms.Recv_init(conn)
            case 2:
                var msg []byte
                s.clk, msg = comms.Recv_data(conn)

                fmt.Println("Recieved: ", string(msg))

                keyStream := EncryptionEngine.GetKeyStream(s.Kc, s.BD_ADDR, s.clk, len(msg))
                msg = EncryptionEngine.Encrypt(msg, keyStream)

                fmt.Println("Decypted as: ", string(msg))

                _, err := http.PostForm("http://127.0.0.1:9999/bar",
                    url.Values{"msg": {string(msg)}})

                if err != nil {
                    fmt.Println("There was an http error: ", err)
                }

            case 99: break LOOP
        }
    }
}

func main() {
    isServerPtr := flag.Bool("server", false, "Run in server mode?")
    flag.Parse()
    fmt.Println("Is Server: ", *isServerPtr)

    var conn net.Conn
    var p string

    var state State

    if *isServerPtr {
        conn = server_main(&state)
        p = ":8888"
    } else {
        conn = client_main()
        p = ":6666"
    }

    go receiver(conn, &state)

    http.HandleFunc("/bar", func(w http.ResponseWriter, r *http.Request) {

        if r.Method == "POST" {
            r.ParseForm()

            fmt.Println("Sending: ", r.PostForm["msg"][0])

            fmt.Fprintf(w, html.EscapeString("Sending packet"))

            pt := []byte(r.PostForm["msg"][0])

            keyStream := EncryptionEngine.GetKeyStream(
                state.Kc, state.BD_ADDR, state.clk, len(pt))
            msg := EncryptionEngine.Encrypt(pt, keyStream)

            comms.Send_data(conn, state.clk, msg)

            fmt.Println("Encrypted as: ", string(msg))
        }
    })

    log.Fatal(http.ListenAndServe(p, nil))

    conn.Close()
}
<commit_msg>Increment the clock on sending a message<commit_after>package main

import (
        "./comms"
        "./EncryptionEngine"

        "flag"

        "net"
        "fmt"
        "io"

        "net/http"
        "net/url"
        "html"
        "log"
       )

type State struct {
    Kc [16]byte
    clk uint32
    BD_ADDR [6]byte
}

func server_main(s *State) net.Conn {
    ln, err := net.Listen("tcp", ":8080")

    if err != nil {
        fmt.Println("net.Listen failed:", err)
    }

    conn, err := ln.Accept()

    fmt.Printf("Accepted Connection\n")

    if err != nil {
        fmt.Println("ln.Accept failed:", err)
    }

    s.BD_ADDR = [6]byte{}
    s.Kc = [16]byte{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}
    s.clk = 0

    comms.Send_neg(conn, s.BD_ADDR)
    comms.Send_init(conn, s.clk, [16]byte{}, s.Kc)

    return conn
}

func client_main() net.Conn {
    conn, err := net.Dial("tcp", "127.0.0.1:8080")

    if err != nil {
        fmt.Println("net.Dial failed:", err)
    }

    return conn
}

func receiver(conn io.Reader, s *State) {
  LOOP:
  for {
        packet_type := comms.Recv_packet(conn)

        switch packet_type {
            case 0:
                s.BD_ADDR = comms.Recv_neg(conn)
            case 1:
                s.clk, _, s.Kc = comms.Recv_init(conn)
            case 2:
                var msg []byte
                s.clk, msg = comms.Recv_data(conn)

                fmt.Println("Recieved: ", string(msg))

                keyStream := EncryptionEngine.GetKeyStream(s.Kc, s.BD_ADDR, s.clk, len(msg))
                msg = EncryptionEngine.Encrypt(msg, keyStream)

                fmt.Println("Decypted as: ", string(msg))

                _, err := http.PostForm("http://127.0.0.1:9999/bar",
                    url.Values{"msg": {string(msg)}})

                if err != nil {
                    fmt.Println("There was an http error: ", err)
                }

            case 99: break LOOP
        }
    }
}

func main() {
    isServerPtr := flag.Bool("server", false, "Run in server mode?")
    flag.Parse()
    fmt.Println("Is Server: ", *isServerPtr)

    var conn net.Conn
    var p string

    var state State

    if *isServerPtr {
        conn = server_main(&state)
        p = ":8888"
    } else {
        conn = client_main()
        p = ":6666"
    }

    go receiver(conn, &state)

    http.HandleFunc("/bar", func(w http.ResponseWriter, r *http.Request) {

        if r.Method == "POST" {
            state.clk++
            r.ParseForm()

            fmt.Println("Sending: ", r.PostForm["msg"][0])

            fmt.Fprintf(w, html.EscapeString("Sending packet"))

            pt := []byte(r.PostForm["msg"][0])

            keyStream := EncryptionEngine.GetKeyStream(
                state.Kc, state.BD_ADDR, state.clk, len(pt))
            msg := EncryptionEngine.Encrypt(pt, keyStream)

            comms.Send_data(conn, state.clk, msg)

            fmt.Println("Encrypted as: ", string(msg))
        }
    })

    log.Fatal(http.ListenAndServe(p, nil))

    conn.Close()
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/flynn/go-discover/discover"
	lc "github.com/flynn/lorne/client"
	"github.com/flynn/lorne/types"
	sc "github.com/flynn/sampi/client"
	"github.com/flynn/sampi/types"
	"github.com/flynn/go-dockerclient"
)

// WARNING: assumes one host at the moment

var sd *discover.Client
var sched *sc.Client
var host *lc.Client
var hostid string

func init() {
	var err error
	sd, err = discover.NewClient()
	if err != nil {
		log.Fatal(err)
	}

	sched, err = sc.New()
	if err != nil {
		log.Fatal(err)
	}

	hostid = findHost()
	host, err = lc.New(hostid)
	if err != nil {
		log.Fatal(err)
	}
}

func main() {
	root := "/var/lib/demo/apps"

	set, _ := sd.Services("shelf")
	addrs := set.OnlineAddrs()
	if len(addrs) < 1 {
		panic("Shelf is not discoverable")
	}
	shelfHost := addrs[0]

	app := os.Args[2]
	os.MkdirAll(root+"/"+app, 0755)

	fmt.Printf("-----> Building %s...\n", app)

	scheduleAndAttach(app+"-build.1", docker.Config{
		Image:        "flynn/slugbuilder",
		Cmd:          []string{"http://" + shelfHost + "/" + app + ".tgz"},
		Tty:          false,
		AttachStdin:  true,
		AttachStdout: true,
		AttachStderr: true,
		OpenStdin:    true,
		StdinOnce:    true,
	})

	fmt.Printf("-----> Deploying %s ...\n", app)

	jobid := app + "-web.1"

	stopIfExists(jobid)
	scheduleWithTcpPort(jobid, docker.Config{
		Image:        "flynn/slugrunner",
		Cmd:          []string{"start", "web"},
		Tty:          false,
		AttachStdin:  false,
		AttachStdout: false,
		AttachStderr: false,
		OpenStdin:    false,
		StdinOnce:    false,
		Env: []string{
			"SLUG_URL=http://" + shelfHost + "/" + app + ".tgz",
			"SD_NAME=" + app,
		},
	})

	time.Sleep(1 * time.Second)
	fmt.Printf("=====> Application deployed:\n")
	fmt.Printf("       http://10.0.2.15:%s\n", getPort(jobid))
	fmt.Println("")

}

func shell(cmdline string) string {
	out, err := exec.Command("bash", "-c", cmdline).Output()
	if err != nil {
		panic(err)
	}
	return strings.Trim(string(out), " \n")
}

func stopIfExists(jobid string) {
	_, err := host.GetJob(jobid)
	if err != nil {
		return
	}
	if err := host.StopJob(jobid); err != nil {
		return
	}
}

func scheduleWithTcpPort(jobid string, config docker.Config) {
	schedReq := &sampi.ScheduleReq{
		Incremental: true,
		HostJobs:    map[string][]*sampi.Job{hostid: {{ID: jobid, Config: &config, TCPPorts: 1}}},
	}
	if _, err := sched.Schedule(schedReq); err != nil {
		log.Fatal(err)
	}
}

func getPort(jobid string) string {
	job, err := host.GetJob(jobid)
	if err != nil {
		log.Fatal(err)
	}
	for portspec := range job.Job.Config.ExposedPorts {
		return strings.Split(portspec, "/")[0]
	}
	return ""
}

func findHost() string {
	state, err := sched.State()
	if err != nil {
		log.Fatal(err)
	}

	var firstHost string
	for k := range state {
		firstHost = k
		break
	}
	if firstHost == "" {
		log.Fatal("no hosts")
	}
	return firstHost
}

func scheduleAndAttach(jobid string, config docker.Config) {
	client, err := lc.New(hostid)
	if err != nil {
		log.Fatal(err)
	}
	conn, attachWait, err := client.Attach(&lorne.AttachReq{
		JobID: jobid,
		Flags: lorne.AttachFlagStdout | lorne.AttachFlagStderr | lorne.AttachFlagStdin | lorne.AttachFlagStream,
	}, true)
	if err != nil {
		log.Fatal(err)
	}

	schedReq := &sampi.ScheduleReq{
		Incremental: true,
		HostJobs:    map[string][]*sampi.Job{hostid: {{ID: jobid, Config: &config}}},
	}
	if _, err := sched.Schedule(schedReq); err != nil {
		log.Fatal(err)
	}

	if err := attachWait(); err != nil {
		log.Fatal(err)
	}

	go func() {
		io.Copy(conn, os.Stdin)
		conn.CloseWrite()
	}()
	scanner := bufio.NewScanner(conn)
	for scanner.Scan() {
		fmt.Fprintln(os.Stdout, scanner.Text()[8:])
	}
	conn.Close()
}
<commit_msg>receiver: Use random IDs for jobs<commit_after>package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/flynn/go-discover/discover"
	"github.com/flynn/go-dockerclient"
	lc "github.com/flynn/lorne/client"
	"github.com/flynn/lorne/types"
	sc "github.com/flynn/sampi/client"
	"github.com/flynn/sampi/types"
)

// WARNING: assumes one host at the moment

var sd *discover.Client
var sched *sc.Client
var host *lc.Client
var hostid string

func init() {
	var err error
	sd, err = discover.NewClient()
	if err != nil {
		log.Fatal(err)
	}

	sched, err = sc.New()
	if err != nil {
		log.Fatal(err)
	}

	hostid = findHost()
	host, err = lc.New(hostid)
	if err != nil {
		log.Fatal(err)
	}
}

func main() {
	root := "/var/lib/demo/apps"

	set, _ := sd.Services("shelf")
	addrs := set.OnlineAddrs()
	if len(addrs) < 1 {
		panic("Shelf is not discoverable")
	}
	shelfHost := addrs[0]

	app := os.Args[2]
	os.MkdirAll(root+"/"+app, 0755)

	fmt.Printf("-----> Building %s...\n", app)

	scheduleAndAttach(sc.RandomJobID(app+"-build."), docker.Config{
		Image:        "flynn/slugbuilder",
		Cmd:          []string{"http://" + shelfHost + "/" + app + ".tgz"},
		Tty:          false,
		AttachStdin:  true,
		AttachStdout: true,
		AttachStderr: true,
		OpenStdin:    true,
		StdinOnce:    true,
	})

	fmt.Printf("-----> Deploying %s ...\n", app)

	jobid := sc.RandomJobID(app + "-web.")

	stopIfExists(jobid)
	scheduleWithTcpPort(jobid, docker.Config{
		Image:        "flynn/slugrunner",
		Cmd:          []string{"start", "web"},
		Tty:          false,
		AttachStdin:  false,
		AttachStdout: false,
		AttachStderr: false,
		OpenStdin:    false,
		StdinOnce:    false,
		Env: []string{
			"SLUG_URL=http://" + shelfHost + "/" + app + ".tgz",
			"SD_NAME=" + app,
		},
	})

	time.Sleep(1 * time.Second)
	fmt.Printf("=====> Application deployed:\n")
	fmt.Printf("       http://10.0.2.15:%s\n", getPort(jobid))
	fmt.Println("")

}

func shell(cmdline string) string {
	out, err := exec.Command("bash", "-c", cmdline).Output()
	if err != nil {
		panic(err)
	}
	return strings.Trim(string(out), " \n")
}

func stopIfExists(jobid string) {
	_, err := host.GetJob(jobid)
	if err != nil {
		return
	}
	if err := host.StopJob(jobid); err != nil {
		return
	}
}

func scheduleWithTcpPort(jobid string, config docker.Config) {
	schedReq := &sampi.ScheduleReq{
		Incremental: true,
		HostJobs:    map[string][]*sampi.Job{hostid: {{ID: jobid, Config: &config, TCPPorts: 1}}},
	}
	if _, err := sched.Schedule(schedReq); err != nil {
		log.Fatal(err)
	}
}

func getPort(jobid string) string {
	job, err := host.GetJob(jobid)
	if err != nil {
		log.Fatal(err)
	}
	for portspec := range job.Job.Config.ExposedPorts {
		return strings.Split(portspec, "/")[0]
	}
	return ""
}

func findHost() string {
	state, err := sched.State()
	if err != nil {
		log.Fatal(err)
	}

	var firstHost string
	for k := range state {
		firstHost = k
		break
	}
	if firstHost == "" {
		log.Fatal("no hosts")
	}
	return firstHost
}

func scheduleAndAttach(jobid string, config docker.Config) {
	client, err := lc.New(hostid)
	if err != nil {
		log.Fatal(err)
	}
	conn, attachWait, err := client.Attach(&lorne.AttachReq{
		JobID: jobid,
		Flags: lorne.AttachFlagStdout | lorne.AttachFlagStderr | lorne.AttachFlagStdin | lorne.AttachFlagStream,
	}, true)
	if err != nil {
		log.Fatal(err)
	}

	schedReq := &sampi.ScheduleReq{
		Incremental: true,
		HostJobs:    map[string][]*sampi.Job{hostid: {{ID: jobid, Config: &config}}},
	}
	if _, err := sched.Schedule(schedReq); err != nil {
		log.Fatal(err)
	}

	if err := attachWait(); err != nil {
		log.Fatal(err)
	}

	go func() {
		io.Copy(conn, os.Stdin)
		conn.CloseWrite()
	}()
	scanner := bufio.NewScanner(conn)
	for scanner.Scan() {
		fmt.Fprintln(os.Stdout, scanner.Text()[8:])
	}
	conn.Close()
}
<|endoftext|>


<commit_before>package main

import (
	"os/exec"
	"io"
	"os"
	"io/ioutil"
	"strings"
	"syscall"
	"fmt"

	"github.com/flynn/go-discover/discover"
)

func main() {
	root := "/var/lib/demo/apps"
	hostname := shell("curl -s icanhazip.com")

	client, err := discover.NewClient()
	if err != nil {
		panic(err)
	}
	set, _ := client.Services("shelf")
	addrs := set.OnlineAddrs()
	if len(addrs) < 1 {
		panic("Shelf is not discoverable")
	}
	shelfHost := addrs[0]

	app := os.Args[2]
	os.MkdirAll(root + "/" + app, 0755)

	fmt.Printf("-----> Building %s on %s ...\n", app, hostname)

	cmd := exec.Command("docker", "run", "-i", "-a=stdin", "-a=stdout", "flynn/slugbuilder", "http://"+shelfHost+"/"+app+".tgz")
	errCh, startCh := attachCmd(cmd, os.Stdout, os.Stderr, os.Stdin)
	err = cmd.Start()
	if err != nil {
		panic(err)
	}
	close(startCh)
	exitCh := exitStatusCh(cmd)
	if err = <-errCh; err != nil {
		panic(err)
	}
	<-exitCh

	fmt.Printf("-----> Deploying %s ...\n", app)
	if _, err := os.Stat(root + "/" + app + "/CONTAINER"); err == nil {
    	oldid := readFile(root + "/" + app + "/CONTAINER")
    	shell("docker kill " + oldid)
	}

	id := shell("docker run -d -p 5000 -e PORT=5000 -e SLUG_URL=http://"+shelfHost+"/"+app+".tgz flynn/slugrunner start web")
	writeFile(root + "/" + app + "/CONTAINER", id)
	port := shell("docker port "+id+" 5000 | sed 's/0.0.0.0://'")
	writeFile(root + "/" + app + "/PORT", port)
	writeFile(root + "/" + app + "/URL", "http://"+hostname+":"+port)

	fmt.Printf("=====> Application deployed:\n")
	fmt.Printf("       %s\n", readFile(root + "/" + app + "/URL"))
	fmt.Println("")

}

func readFile(filename string) string {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		panic(err)
	}
	return string(data)
}

func writeFile(filename, data string) {
	err := ioutil.WriteFile(filename, []byte(data), 0644)
	if err != nil {
		panic(err)
	}
}

func shell(cmdline string) string {
	out, err := exec.Command("bash", "-c", cmdline).Output()
	if err != nil {
		panic(err)
	}
	return strings.Trim(string(out), " \n")
}

func attachCmd(cmd *exec.Cmd, stdout, stderr io.Writer, stdin io.Reader) (chan error, chan interface{}) {
	errCh := make(chan error)
	startCh := make(chan interface{})

	stdinIn, err := cmd.StdinPipe()
	if err != nil {
		panic(err)
	}
	stdoutOut, err := cmd.StdoutPipe()
	if err != nil {
		panic(err)
	}
	stderrOut, err := cmd.StderrPipe()
	if err != nil {
		panic(err)
	}

	go func() {
		<-startCh
		go func() {
			_, e := io.Copy(stdinIn, stdin)
			errCh <- e
		}()
		go func() {
			_, e := io.Copy(stdout, stdoutOut)
			errCh <- e
		}()
		go func() {
			_, e := io.Copy(stderr, stderrOut)
			errCh <- e
		}()
	}()

	return errCh, startCh
}

func exitStatusCh(cmd *exec.Cmd) chan uint {
	exitCh := make(chan uint)
	go func() {
		err := cmd.Wait()
		if err != nil {
			if exiterr, ok := err.(*exec.ExitError); ok {
				// There is no plattform independent way to retrieve
				// the exit code, but the following will work on Unix
				if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
					exitCh <- uint(status.ExitStatus())
				}
			} else {
				panic(err)
			}
			return
		}
		exitCh <- uint(0)
	}()
	return exitCh
}<commit_msg>never mind<commit_after>package main

import (
	"os/exec"
	"io"
	"os"
	"io/ioutil"
	"strings"
	"syscall"
	"fmt"

	"github.com/flynn/go-discover/discover"
)

func main() {
	root := "/var/lib/demo/apps"
	hostname := shell("curl -s icanhazip.com")

	client, err := discover.NewClient()
	if err != nil {
		panic(err)
	}
	set, _ := client.Services("shelf")
	addrs := set.OnlineAddrs()
	if len(addrs) < 1 {
		panic("Shelf is not discoverable")
	}
	shelfHost := addrs[0]

	app := os.Args[2]
	os.MkdirAll(root + "/" + app, 0755)

	fmt.Printf("-----> Building %s on %s ...\n", app, hostname)

	cmd := exec.Command("docker", "run", "-i", "-a=stdin", "-a=stdout", "flynn/slugbuilder", "http://"+shelfHost+"/"+app+".tgz")
	errCh := attachCmd(cmd, os.Stdout, os.Stderr, os.Stdin)
	err = cmd.Start()
	if err != nil {
		panic(err)
	}
	exitCh := exitStatusCh(cmd)
	if err = <-errCh; err != nil {
		panic(err)
	}
	<-exitCh

	fmt.Printf("-----> Deploying %s ...\n", app)
	if _, err := os.Stat(root + "/" + app + "/CONTAINER"); err == nil {
    	oldid := readFile(root + "/" + app + "/CONTAINER")
    	shell("docker kill " + oldid)
	}

	id := shell("docker run -d -p 5000 -e PORT=5000 -e SLUG_URL=http://"+shelfHost+"/"+app+".tgz flynn/slugrunner start web")
	writeFile(root + "/" + app + "/CONTAINER", id)
	port := shell("docker port "+id+" 5000 | sed 's/0.0.0.0://'")
	writeFile(root + "/" + app + "/PORT", port)
	writeFile(root + "/" + app + "/URL", "http://"+hostname+":"+port)

	fmt.Printf("=====> Application deployed:\n")
	fmt.Printf("       %s\n", readFile(root + "/" + app + "/URL"))
	fmt.Println("")

}

func readFile(filename string) string {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		panic(err)
	}
	return string(data)
}

func writeFile(filename, data string) {
	err := ioutil.WriteFile(filename, []byte(data), 0644)
	if err != nil {
		panic(err)
	}
}

func shell(cmdline string) string {
	out, err := exec.Command("bash", "-c", cmdline).Output()
	if err != nil {
		panic(err)
	}
	return strings.Trim(string(out), " \n")
}

func attachCmd(cmd *exec.Cmd, stdout, stderr io.Writer, stdin io.Reader) chan error {
	errCh := make(chan error)

	stdinIn, err := cmd.StdinPipe()
	if err != nil {
		panic(err)
	}
	stdoutOut, err := cmd.StdoutPipe()
	if err != nil {
		panic(err)
	}
	stderrOut, err := cmd.StderrPipe()
	if err != nil {
		panic(err)
	}

	go func() {
		_, e := io.Copy(stdinIn, stdin)
		errCh <- e
	}()
	go func() {
		_, e := io.Copy(stdout, stdoutOut)
		errCh <- e
	}()
	go func() {
		_, e := io.Copy(stderr, stderrOut)
		errCh <- e
	}()

	return errCh
}

func exitStatusCh(cmd *exec.Cmd) chan uint {
	exitCh := make(chan uint)
	go func() {
		err := cmd.Wait()
		if err != nil {
			if exiterr, ok := err.(*exec.ExitError); ok {
				// There is no plattform independent way to retrieve
				// the exit code, but the following will work on Unix
				if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
					exitCh <- uint(status.ExitStatus())
				}
			} else {
				panic(err)
			}
			return
		}
		exitCh <- uint(0)
	}()
	return exitCh
}<|endoftext|>


<commit_before>package main

import (
	"os/exec"
	"io"
	"os"
	"io/ioutil"
	"strings"
	"syscall"
	"fmt"

	"github.com/flynn/go-discover/discover"
)

func main() {
	root := "/var/lib/demo/apps"
	hostname := shell("curl -s icanhazip.com")

	client, err := discover.NewClient()
	if err != nil {
		panic(err)
	}
	set, _ := client.Services("shelf")
	addrs := set.OnlineAddrs()
	if len(addrs) < 1 {
		panic("Shelf is not discoverable")
	}
	shelfHost := addrs[0]

	app := os.Args[2]
	os.MkdirAll(root + "/" + app, 0755)

	fmt.Printf("-----> Building %s on %s ...\n", app, hostname)

	cmd := exec.Command("docker", "run", "-i", "-a=stdin", "-a=stdout", "flynn/slugbuilder", "http://"+shelfHost+"/"+app+".tgz")
	errCh := attachCmd(cmd, os.Stdout, os.Stderr, os.Stdin)
	err = cmd.Start()
	if err != nil {
		panic(err)
	}
	exitCh := exitStatusCh(cmd)
	if err = <-errCh; err != nil {
		panic(err)
	}
	<-exitCh

	fmt.Printf("-----> Deploying %s ...\n", app)
	if _, err := os.Stat(root + "/" + app + "/CONTAINER"); err == nil {
    	oldid := readFile(root + "/" + app + "/CONTAINER")
    	shell("docker kill " + oldid)
	}

	id := shell("docker run -d -p 5000 -e PORT=5000 -e SLUG_URL=http://"+shelfHost+"/"+app+".tgz flynn/slugrunner start web")
	writeFile(root + "/" + app + "/CONTAINER", id)
	port := shell("docker port "+id+" 5000 | sed 's/0.0.0.0://'")
	writeFile(root + "/" + app + "/PORT", port)
	writeFile(root + "/" + app + "/URL", "http://"+hostname+":"+port)

	fmt.Printf("=====> Application deployed:\n")
	fmt.Printf("       %s\n", readFile(root + "/" + app + "/URL"))
	fmt.Println("")

}

func readFile(filename string) string {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		panic(err)
	}
	return string(data)
}

func writeFile(filename, data string) {
	err := ioutil.WriteFile(filename, []byte(data), 0644)
	if err != nil {
		panic(err)
	}
}

func shell(cmdline string) string {
	out, err := exec.Command("bash", "-c", cmdline).Output()
	if err != nil {
		panic(err)
	}
	return strings.Trim(string(out), " \n")
}

func attachCmd(cmd *exec.Cmd, stdout, stderr io.Writer, stdin io.Reader) chan error {
	errCh := make(chan error)

	stdinIn, err := cmd.StdinPipe()
	if err != nil {
		panic(err)
	}
	stdoutOut, err := cmd.StdoutPipe()
	if err != nil {
		panic(err)
	}
	stderrOut, err := cmd.StderrPipe()
	if err != nil {
		panic(err)
	}

	go func() {
		_, e := io.Copy(stdinIn, stdin)
		errCh <- e
	}()
	go func() {
		_, e := io.Copy(stdout, stdoutOut)
		errCh <- e
	}()
	go func() {
		_, e := io.Copy(stderr, stderrOut)
		errCh <- e
	}()

	return errCh
}

func exitStatusCh(cmd *exec.Cmd) chan uint {
	exitCh := make(chan uint)
	go func() {
		err := cmd.Wait()
		if err != nil {
			if exiterr, ok := err.(*exec.ExitError); ok {
				// There is no plattform independent way to retrieve
				// the exit code, but the following will work on Unix
				if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
					exitCh <- uint(status.ExitStatus())
				}
			} else {
				panic(err)
			}
			return
		}
		exitCh <- uint(0)
	}()
	return exitCh
}<commit_msg>receiver: starting copy after command is started<commit_after>package main

import (
	"os/exec"
	"io"
	"os"
	"io/ioutil"
	"strings"
	"syscall"
	"fmt"

	"github.com/flynn/go-discover/discover"
)

func main() {
	root := "/var/lib/demo/apps"
	hostname := shell("curl -s icanhazip.com")

	client, err := discover.NewClient()
	if err != nil {
		panic(err)
	}
	set, _ := client.Services("shelf")
	addrs := set.OnlineAddrs()
	if len(addrs) < 1 {
		panic("Shelf is not discoverable")
	}
	shelfHost := addrs[0]

	app := os.Args[2]
	os.MkdirAll(root + "/" + app, 0755)

	fmt.Printf("-----> Building %s on %s ...\n", app, hostname)

	cmd := exec.Command("docker", "run", "-i", "-a=stdin", "-a=stdout", "flynn/slugbuilder", "http://"+shelfHost+"/"+app+".tgz")
	errCh, startCh := attachCmd(cmd, os.Stdout, os.Stderr, os.Stdin)
	err = cmd.Start()
	if err != nil {
		panic(err)
	}
	close(startCh)
	exitCh := exitStatusCh(cmd)
	if err = <-errCh; err != nil {
		panic(err)
	}
	<-exitCh

	fmt.Printf("-----> Deploying %s ...\n", app)
	if _, err := os.Stat(root + "/" + app + "/CONTAINER"); err == nil {
    	oldid := readFile(root + "/" + app + "/CONTAINER")
    	shell("docker kill " + oldid)
	}

	id := shell("docker run -d -p 5000 -e PORT=5000 -e SLUG_URL=http://"+shelfHost+"/"+app+".tgz flynn/slugrunner start web")
	writeFile(root + "/" + app + "/CONTAINER", id)
	port := shell("docker port "+id+" 5000 | sed 's/0.0.0.0://'")
	writeFile(root + "/" + app + "/PORT", port)
	writeFile(root + "/" + app + "/URL", "http://"+hostname+":"+port)

	fmt.Printf("=====> Application deployed:\n")
	fmt.Printf("       %s\n", readFile(root + "/" + app + "/URL"))
	fmt.Println("")

}

func readFile(filename string) string {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		panic(err)
	}
	return string(data)
}

func writeFile(filename, data string) {
	err := ioutil.WriteFile(filename, []byte(data), 0644)
	if err != nil {
		panic(err)
	}
}

func shell(cmdline string) string {
	out, err := exec.Command("bash", "-c", cmdline).Output()
	if err != nil {
		panic(err)
	}
	return strings.Trim(string(out), " \n")
}

func attachCmd(cmd *exec.Cmd, stdout, stderr io.Writer, stdin io.Reader) (chan error, chan interface{}) {
	errCh := make(chan error)
	startCh := make(chan interface{})

	stdinIn, err := cmd.StdinPipe()
	if err != nil {
		panic(err)
	}
	stdoutOut, err := cmd.StdoutPipe()
	if err != nil {
		panic(err)
	}
	stderrOut, err := cmd.StderrPipe()
	if err != nil {
		panic(err)
	}

	go func() {
		<-startCh
		go func() {
			_, e := io.Copy(stdinIn, stdin)
			errCh <- e
		}()
		go func() {
			_, e := io.Copy(stdout, stdoutOut)
			errCh <- e
		}()
		go func() {
			_, e := io.Copy(stderr, stderrOut)
			errCh <- e
		}()
	}()

	return errCh, startCh
}

func exitStatusCh(cmd *exec.Cmd) chan uint {
	exitCh := make(chan uint)
	go func() {
		err := cmd.Wait()
		if err != nil {
			if exiterr, ok := err.(*exec.ExitError); ok {
				// There is no plattform independent way to retrieve
				// the exit code, but the following will work on Unix
				if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
					exitCh <- uint(status.ExitStatus())
				}
			} else {
				panic(err)
			}
			return
		}
		exitCh <- uint(0)
	}()
	return exitCh
}<|endoftext|>


<commit_before>//
// daemon.go
//
package srnd
import (
  "log"
  "net"
  "net/textproto"
  "strconv"
  "strings"
  "os"
  "time"
)

type NNTPDaemon struct {
  instance_name string
  bind_addr string
  conf *SRNdConfig
  store ArticleStore
  database Database
  mod ModEngine
  expire ExpirationCore
  listener net.Listener
  debug bool
  sync_on_start bool
  // anon settings
  allow_anon bool
  allow_anon_attachments bool

  running bool
  // http frontend
  frontend Frontend

  // map of addr -> NNTPConnection
  feeds map[string]nntpConnection
  // for registering and deregistering outbound feeds
  register_outfeed chan nntpConnection
  deregister_outfeed chan nntpConnection
  // infeed for articles
  infeed chan NNTPMessage
  // channel to load messages to infeed given their message id
  infeed_load chan string
  // channel for broadcasting a message to all feeds given their newsgroup, message_id
  send_all_feeds chan ArticleEntry
  // channel for broadcasting an ARTICLE command to all feeds in reader mode
  ask_for_article chan ArticleEntry
}

func (self NNTPDaemon) End() {
  self.listener.Close()
}

func (self NNTPDaemon) persistFeed(conf FeedConfig, mode string) {
  for {
    if self.running {

      var conn net.Conn
      var err error
      proxy_type := strings.ToLower(conf.proxy_type)

      if proxy_type ==  "" || proxy_type == "none" {
        // connect out without proxy
        log.Println("dial out to ", conf.addr)
        conn, err = net.Dial("tcp", conf.addr)
        if err != nil {
          log.Println("cannot connect to outfeed", conf.addr, err)
					time.Sleep(time.Second * 5)
          continue
        }
      } else if proxy_type == "socks4a" {
        // connect via socks4a
        log.Println("dial out via proxy", conf.proxy_addr)
        conn, err = net.Dial("tcp", conf.proxy_addr)
        if err != nil {
          log.Println("cannot connect to proxy", conf.proxy_addr)
					time.Sleep(time.Second * 5)
          continue
        }
        // generate request
        idx := strings.LastIndex(conf.addr, ":")
        if idx == -1 {
          log.Fatal("invalid outfeed address")
        }
        var port uint64
        addr := conf.addr[:idx]
        port, err = strconv.ParseUint(conf.addr[idx+1:], 10, 16)
        if port >= 25536 {
          log.Fatal("bad proxy port" , port)
        }
        var proxy_port uint16
        proxy_port = uint16(port)
        proxy_ident := "srndv2"
        req_len := len(addr) + 1 + len(proxy_ident) + 1 + 8

        req := make([]byte, req_len)
        // pack request
        req[0] = '\x04'
        req[1] = '\x01'
        req[2] = byte(proxy_port & 0xff00 >> 8)
        req[3] = byte(proxy_port & 0x00ff)
        req[7] = '\x01'
        idx = 8

        proxy_ident_b := []byte(proxy_ident)
        addr_b := []byte(addr)

        var bi int
        for bi = range proxy_ident_b {
          req[idx] = proxy_ident_b[bi]
          idx += 1
        }
        idx += 1
        for bi = range addr_b {
          req[idx] = addr_b[bi]
          idx += 1
        }

        // send request
        conn.Write(req)
        resp := make([]byte, 8)

        // receive response
        conn.Read(resp)
        if resp[1] == '\x5a' {
          // success
          log.Println("connected to", conf.addr)
        } else {
          log.Println("failed to connect to", conf.addr)
					time.Sleep(time.Second * 5)
          continue
        }
      }
      nntp := createNNTPConnection()
      nntp.policy = conf.policy
      nntp.name = conf.addr + "-" + mode
      c := textproto.NewConn(conn)
      stream, reader, err := nntp.outboundHandshake(c)
      if err == nil {
        if self.sync_on_start {
          go func() {
            log.Println(nntp.name, "will do full sync")
            for _, article := range self.database.GetAllArticles() {
              if nntp.policy.AllowsNewsgroup(article.Newsgroup()) {
                nntp.check <- article.MessageID()
              }
            }

          }()
        }

        if mode == "reader" && ! reader {
          log.Println(nntp.name, "we don't support reader on this feed, dropping")
          return
        }
        self.register_outfeed <- nntp
        nntp.runConnection(self, false, stream, reader, mode, c)
        self.deregister_outfeed <- nntp

      } else {
        log.Println("error doing outbound hanshake", err)
      }
    }
  }
  time.Sleep(1 * time.Second)
}

// run daemon
func (self NNTPDaemon) Run() {

  self.bind_addr = self.conf.daemon["bind"]

  listener , err := net.Listen("tcp", self.bind_addr)
  if err != nil {
    log.Fatal("failed to bind to", self.bind_addr, err)
  }
  self.listener = listener
  log.Printf("SRNd NNTPD bound at %s", listener.Addr())

  self.register_outfeed = make(chan nntpConnection)
  self.deregister_outfeed = make(chan nntpConnection)
  self.infeed = make(chan NNTPMessage, 8)
  self.infeed_load = make(chan string)
  self.send_all_feeds = make(chan ArticleEntry, 64)
  self.feeds = make(map[string]nntpConnection)
  self.ask_for_article = make(chan ArticleEntry, 64)

  self.expire = createExpirationCore(self.database, self.store)
  self.sync_on_start = self.conf.daemon["sync_on_start"] == "1"
  self.debug = self.conf.daemon["log"] == "debug"
  self.instance_name = self.conf.daemon["instance_name"]
  self.allow_anon = self.conf.daemon["allow_anon"] == "1"
  self.allow_anon_attachments = self.conf.daemon["allow_anon_attachments"] == "1"

  if self.debug {
    log.Println("debug mode activated")
  }

  // do we enable the frontend?
  if self.conf.frontend["enable"] == "1" {
    log.Printf("frontend %s enabled", self.conf.frontend["name"])
    http_frontend := NewHTTPFrontend(&self, self.conf.frontend, self.conf.worker["url"])
    nntp_frontend := NewNNTPFrontend(&self, self.conf.frontend["nntp"])
    self.frontend = MuxFrontends(http_frontend, nntp_frontend)
    go self.frontend.Mainloop()
  }

  // set up admin user if it's specified in the config
  pubkey , ok := self.conf.frontend["admin_key"]
  if ok {
    // TODO: check for valid format
    log.Println("add admin key", pubkey)
    err = self.database.MarkModPubkeyGlobal(pubkey)
    if err != nil {
      log.Printf("failed to add admin mod key, %s", err)
    }
  }

  defer self.listener.Close()
  // run expiration mainloop
  go self.expire.Mainloop()
  // we are now running
  self.running = true

  // persist outfeeds
  for idx := range self.conf.feeds {
    go self.persistFeed(self.conf.feeds[idx], "reader")
    go self.persistFeed(self.conf.feeds[idx], "stream")
  }

  // start accepting incoming connections
  go self.acceptloop()

  go func () {
    // if we have no initial posts create one
    if self.database.ArticleCount() == 0 {
      nntp := newPlaintextArticle("welcome to nntpchan, this post was inserted on startup automatically", "system@"+self.instance_name, "Welcome to NNTPChan", "system", self.instance_name, genMessageID(self.instance_name), "overchan.test")
      nntp.Pack()
      file := self.store.CreateTempFile(nntp.MessageID())
      if file != nil {
        err := self.store.WriteMessage(nntp, file)
        file.Close()
        if err == nil {
          self.infeed <- nntp
        } else {
          log.Println("failed to create startup messge?", err)
        }
      }
    }
  }()

  // get all pending articles from infeed and load them
  go func() {
    f, err := os.Open(self.store.TempDir())
    if err == nil {
      names, err := f.Readdirnames(0)
      if err == nil {
        for _, name := range names {
          self.infeed_load <- name
        }
      }
    }

  }()

  // if we have no frontend this does nothing
  if self.frontend != nil {
    go self.pollfrontend()
  }
  go self.pollinfeed()
  go self.pollmessages()
  self.polloutfeeds()
}

func (self NNTPDaemon) pollfrontend() {
  chnl := self.frontend.NewPostsChan()
  for {
    nntp := <- chnl
    // new post from frontend
    log.Println("frontend post", nntp.MessageID())
    self.infeed <- nntp
  }
}
func (self NNTPDaemon) pollinfeed() {
  for {
    msgid := <- self.infeed_load
    log.Println("load from infeed", msgid)
    msg := self.store.ReadTempMessage(msgid)
    if msg != nil {
      self.infeed <- msg
    }
  }
}

func (self NNTPDaemon) polloutfeeds() {

  for {
    select {

    case outfeed := <- self.register_outfeed:
      log.Println("outfeed", outfeed.name, "registered")
      self.feeds[outfeed.name] = outfeed
    case outfeed := <- self.deregister_outfeed:
      log.Println("outfeed", outfeed.name, "de-registered")
      delete(self.feeds, outfeed.name)
    case nntp := <- self.send_all_feeds:
      log.Println("federate", nntp.MessageID())
      feeds := self.feeds
      for _, feed := range feeds {
        if feed.policy.AllowsNewsgroup(nntp.Newsgroup()) {
          if strings.HasSuffix(feed.name, "-stream") {
            log.Println("send", nntp.MessageID(), "to", feed.name)
            feed.check <- nntp.MessageID()
          }
        } else {
          log.Println("not allowed", feed.name)
        }
      }
    case nntp := <- self.ask_for_article:
      feeds := self.feeds
      for _, feed := range feeds {
        if feed.policy.AllowsNewsgroup(nntp.Newsgroup()) {
          if strings.HasSuffix(feed.name, "-reader") {
            log.Println("asking", feed.name, "for", nntp.MessageID(), "mode", feed.mode)
            feed.article <- nntp.MessageID()
          }
        }
      }
    }
  }
}

func (self NNTPDaemon) pollmessages() {
  var chnl chan NNTPMessage
  modchnl := self.mod.MessageChan()
  if self.frontend != nil {
    chnl = self.frontend.PostsChan()
  }
  for {

    nntp := <- self.infeed
    // ammend path
    nntp.AppendPath(self.instance_name)
    msgid := nntp.MessageID()
    log.Println("daemon got", msgid)

    // store article and attachments
    // register with database
    // this also generates thumbnails
    self.store.StorePost(nntp)

    ref := nntp.Reference()
    if ref != "" && ValidMessageID(ref) && ! self.database.HasArticleLocal(ref) {
      // we don't have the root post
      // generate it
      //log.Println("creating temp root post for", ref , "in", nntp.Newsgroup())
      //root := newPlaintextArticle("temporary placeholder", "lol@lol", "root post "+ref+" not found", "system", "temp", ref, nntp.Newsgroup())
      //self.store.StorePost(root)
    }

    // prepare for content rollover
    // fallback rollover
    rollover := 100

    group := nntp.Newsgroup()
    tpp, err := self.database.GetThreadsPerPage(group)
    ppb, err := self.database.GetPagesPerBoard(group)
    if err == nil {
      rollover = tpp * ppb
    }

    // roll over old content
    self.expire.ExpireGroup(group, rollover)
    // handle mod events
    if group == "ctl" {
      modchnl <- nntp
    }

    // queue to all outfeeds
    // XXX: blocking ?
    self.send_all_feeds <- ArticleEntry{msgid, group}
    // tell frontend
    // XXX: blocking ?
    if chnl != nil {
      if self.frontend.AllowNewsgroup(group) {
        chnl <- nntp
      } else {
        log.Println("frontend does not allow", group, "not sending")
      }
    }
  }
}

func (self NNTPDaemon) acceptloop() {
  for {
    // accept
    conn, err := self.listener.Accept()
    if err != nil {
      log.Fatal(err)
    }
    // make a new inbound nntp connection handler
    nntp := createNNTPConnection()
    c := textproto.NewConn(conn)
    // send banners and shit
    err = nntp.inboundHandshake(c)
    if err == nil {
      // run, we support stream and reader
      go nntp.runConnection(self, true, true, true, "stream", c)
    } else {
      log.Println("failed to send banners", err)
      c.Close()
    }
  }
}

func (self NNTPDaemon) Setup() NNTPDaemon {
  log.Println("checking for configs...")
  // check that are configs exist
  CheckConfig()
  log.Println("loading config...")
  // read the config
  self.conf = ReadConfig()
  if self.conf == nil {
    log.Fatal("failed to load config")
  }
  // validate the config
  log.Println("validating configs...")
  self.conf.Validate()
  log.Println("configs are valid")

  db_host := self.conf.database["host"]
  db_port := self.conf.database["port"]
  db_user := self.conf.database["user"]
  db_passwd := self.conf.database["password"]

  // set up database stuff
  log.Println("connecting to database...")
  self.database = NewDatabase(self.conf.database["type"], self.conf.database["schema"], db_host, db_port, db_user, db_passwd)
  log.Println("ensure that the database is created...")
  self.database.CreateTables()

  // set up store
  log.Println("set up article store...")
  self.store = createArticleStore(self.conf.store, self.database)

  self.mod = modEngine{
    store: self.store,
    database:  self.database,
    chnl: make(chan NNTPMessage),
  }
  return self
}
<commit_msg>logging<commit_after>//
// daemon.go
//
package srnd
import (
  "fmt"
  "log"
  "net"
  "net/textproto"
  "strconv"
  "strings"
  "os"
  "time"
)

type NNTPDaemon struct {
  instance_name string
  bind_addr string
  conf *SRNdConfig
  store ArticleStore
  database Database
  mod ModEngine
  expire ExpirationCore
  listener net.Listener
  debug bool
  sync_on_start bool
  // anon settings
  allow_anon bool
  allow_anon_attachments bool

  running bool
  // http frontend
  frontend Frontend

  // map of addr -> NNTPConnection
  feeds map[string]nntpConnection
  // for registering and deregistering outbound feeds
  register_outfeed chan nntpConnection
  deregister_outfeed chan nntpConnection
  // infeed for articles
  infeed chan NNTPMessage
  // channel to load messages to infeed given their message id
  infeed_load chan string
  // channel for broadcasting a message to all feeds given their newsgroup, message_id
  send_all_feeds chan ArticleEntry
  // channel for broadcasting an ARTICLE command to all feeds in reader mode
  ask_for_article chan ArticleEntry
}

func (self NNTPDaemon) End() {
  self.listener.Close()
}

func (self NNTPDaemon) persistFeed(conf FeedConfig, mode string) {
  for {
    if self.running {

      var conn net.Conn
      var err error
      proxy_type := strings.ToLower(conf.proxy_type)

      if proxy_type ==  "" || proxy_type == "none" {
        // connect out without proxy
        log.Println("dial out to ", conf.addr)
        conn, err = net.Dial("tcp", conf.addr)
        if err != nil {
          log.Println("cannot connect to outfeed", conf.addr, err)
					time.Sleep(time.Second * 5)
          continue
        }
      } else if proxy_type == "socks4a" {
        // connect via socks4a
        log.Println("dial out via proxy", conf.proxy_addr)
        conn, err = net.Dial("tcp", conf.proxy_addr)
        if err != nil {
          log.Println("cannot connect to proxy", conf.proxy_addr)
					time.Sleep(time.Second * 5)
          continue
        }
        // generate request
        idx := strings.LastIndex(conf.addr, ":")
        if idx == -1 {
          log.Fatal("invalid outfeed address")
        }
        var port uint64
        addr := conf.addr[:idx]
        port, err = strconv.ParseUint(conf.addr[idx+1:], 10, 16)
        if port >= 25536 {
          log.Fatal("bad proxy port" , port)
        }
        var proxy_port uint16
        proxy_port = uint16(port)
        proxy_ident := "srndv2"
        req_len := len(addr) + 1 + len(proxy_ident) + 1 + 8

        req := make([]byte, req_len)
        // pack request
        req[0] = '\x04'
        req[1] = '\x01'
        req[2] = byte(proxy_port & 0xff00 >> 8)
        req[3] = byte(proxy_port & 0x00ff)
        req[7] = '\x01'
        idx = 8

        proxy_ident_b := []byte(proxy_ident)
        addr_b := []byte(addr)

        var bi int
        for bi = range proxy_ident_b {
          req[idx] = proxy_ident_b[bi]
          idx += 1
        }
        idx += 1
        for bi = range addr_b {
          req[idx] = addr_b[bi]
          idx += 1
        }

        // send request
        conn.Write(req)
        resp := make([]byte, 8)

        // receive response
        conn.Read(resp)
        if resp[1] == '\x5a' {
          // success
          log.Println("connected to", conf.addr)
        } else {
          log.Println("failed to connect to", conf.addr)
					time.Sleep(time.Second * 5)
          continue
        }
      }
      nntp := createNNTPConnection()
      nntp.policy = conf.policy
      nntp.name = conf.addr + "-" + mode
      c := textproto.NewConn(conn)
      stream, reader, err := nntp.outboundHandshake(c)
      if err == nil {
        if self.sync_on_start {
          go func() {
            log.Println(nntp.name, "will do full sync")
            for _, article := range self.database.GetAllArticles() {
              if nntp.policy.AllowsNewsgroup(article.Newsgroup()) {
                nntp.check <- article.MessageID()
              }
            }

          }()
        }

        if mode == "reader" && ! reader {
          log.Println(nntp.name, "we don't support reader on this feed, dropping")
          return
        }
        self.register_outfeed <- nntp
        nntp.runConnection(self, false, stream, reader, mode, c)
        self.deregister_outfeed <- nntp

      } else {
        log.Println("error doing outbound hanshake", err)
      }
    }
  }
  time.Sleep(1 * time.Second)
}

// run daemon
func (self NNTPDaemon) Run() {

  self.bind_addr = self.conf.daemon["bind"]

  listener , err := net.Listen("tcp", self.bind_addr)
  if err != nil {
    log.Fatal("failed to bind to", self.bind_addr, err)
  }
  self.listener = listener
  log.Printf("SRNd NNTPD bound at %s", listener.Addr())

  self.register_outfeed = make(chan nntpConnection)
  self.deregister_outfeed = make(chan nntpConnection)
  self.infeed = make(chan NNTPMessage, 8)
  self.infeed_load = make(chan string)
  self.send_all_feeds = make(chan ArticleEntry, 64)
  self.feeds = make(map[string]nntpConnection)
  self.ask_for_article = make(chan ArticleEntry, 64)

  self.expire = createExpirationCore(self.database, self.store)
  self.sync_on_start = self.conf.daemon["sync_on_start"] == "1"
  self.debug = self.conf.daemon["log"] == "debug"
  self.instance_name = self.conf.daemon["instance_name"]
  self.allow_anon = self.conf.daemon["allow_anon"] == "1"
  self.allow_anon_attachments = self.conf.daemon["allow_anon_attachments"] == "1"

  if self.debug {
    log.Println("debug mode activated")
  }

  // do we enable the frontend?
  if self.conf.frontend["enable"] == "1" {
    log.Printf("frontend %s enabled", self.conf.frontend["name"])
    http_frontend := NewHTTPFrontend(&self, self.conf.frontend, self.conf.worker["url"])
    nntp_frontend := NewNNTPFrontend(&self, self.conf.frontend["nntp"])
    self.frontend = MuxFrontends(http_frontend, nntp_frontend)
    go self.frontend.Mainloop()
  }

  // set up admin user if it's specified in the config
  pubkey , ok := self.conf.frontend["admin_key"]
  if ok {
    // TODO: check for valid format
    log.Println("add admin key", pubkey)
    err = self.database.MarkModPubkeyGlobal(pubkey)
    if err != nil {
      log.Printf("failed to add admin mod key, %s", err)
    }
  }

  defer self.listener.Close()
  // run expiration mainloop
  go self.expire.Mainloop()
  // we are now running
  self.running = true

  // persist outfeeds
  for idx := range self.conf.feeds {
    go self.persistFeed(self.conf.feeds[idx], "reader")
    go self.persistFeed(self.conf.feeds[idx], "stream")
  }

  // start accepting incoming connections
  go self.acceptloop()

  go func () {
    // if we have no initial posts create one
    if self.database.ArticleCount() == 0 {
      nntp := newPlaintextArticle("welcome to nntpchan, this post was inserted on startup automatically", "system@"+self.instance_name, "Welcome to NNTPChan", "system", self.instance_name, genMessageID(self.instance_name), "overchan.test")
      nntp.Pack()
      file := self.store.CreateTempFile(nntp.MessageID())
      if file != nil {
        err := self.store.WriteMessage(nntp, file)
        file.Close()
        if err == nil {
          self.infeed <- nntp
        } else {
          log.Println("failed to create startup messge?", err)
        }
      }
    }
  }()

  // get all pending articles from infeed and load them
  go func() {
    f, err := os.Open(self.store.TempDir())
    if err == nil {
      names, err := f.Readdirnames(0)
      if err == nil {
        for _, name := range names {
          self.infeed_load <- name
        }
      }
    }

  }()

  // if we have no frontend this does nothing
  if self.frontend != nil {
    go self.pollfrontend()
  }
  go self.pollinfeed()
  go self.pollmessages()
  self.polloutfeeds()
}

func (self NNTPDaemon) pollfrontend() {
  chnl := self.frontend.NewPostsChan()
  for {
    nntp := <- chnl
    // new post from frontend
    log.Println("frontend post", nntp.MessageID())
    self.infeed <- nntp
  }
}
func (self NNTPDaemon) pollinfeed() {
  for {
    msgid := <- self.infeed_load
    log.Println("load from infeed", msgid)
    msg := self.store.ReadTempMessage(msgid)
    if msg != nil {
      self.infeed <- msg
    }
  }
}

func (self NNTPDaemon) polloutfeeds() {

  for {
    select {

    case outfeed := <- self.register_outfeed:
      log.Println("outfeed", outfeed.name, "registered")
      self.feeds[outfeed.name] = outfeed
    case outfeed := <- self.deregister_outfeed:
      log.Println("outfeed", outfeed.name, "de-registered")
      delete(self.feeds, outfeed.name)
    case nntp := <- self.send_all_feeds:
      log.Println("federate", nntp.MessageID())
      feeds := self.feeds
      for _, feed := range feeds {
        if feed.policy.AllowsNewsgroup(nntp.Newsgroup()) {
          if strings.HasSuffix(feed.name, "-stream") {
            log.Println("send", nntp.MessageID(), "to", feed.name)
            feed.check <- nntp.MessageID()
          }
        } else {
          log.Println("not allowed", feed.name)
        }
      }
    case nntp := <- self.ask_for_article:
      feeds := self.feeds
      for _, feed := range feeds {
        if feed.policy.AllowsNewsgroup(nntp.Newsgroup()) {
          if strings.HasSuffix(feed.name, "-reader") {
            log.Println("asking", feed.name, "for", nntp.MessageID(), "mode", feed.mode)
            feed.article <- nntp.MessageID()
          }
        }
      }
    }
  }
}

func (self NNTPDaemon) pollmessages() {
  var chnl chan NNTPMessage
  modchnl := self.mod.MessageChan()
  if self.frontend != nil {
    chnl = self.frontend.PostsChan()
  }
  for {

    nntp := <- self.infeed
    // ammend path
    nntp.AppendPath(self.instance_name)
    msgid := nntp.MessageID()
    log.Println("daemon got", msgid)

    // store article and attachments
    // register with database
    // this also generates thumbnails
    self.store.StorePost(nntp)

    ref := nntp.Reference()
    if ref != "" && ValidMessageID(ref) && ! self.database.HasArticleLocal(ref) {
      // we don't have the root post
      // generate it
      //log.Println("creating temp root post for", ref , "in", nntp.Newsgroup())
      //root := newPlaintextArticle("temporary placeholder", "lol@lol", "root post "+ref+" not found", "system", "temp", ref, nntp.Newsgroup())
      //self.store.StorePost(root)
    }

    // prepare for content rollover
    // fallback rollover
    rollover := 100

    group := nntp.Newsgroup()
    tpp, err := self.database.GetThreadsPerPage(group)
    ppb, err := self.database.GetPagesPerBoard(group)
    if err == nil {
      rollover = tpp * ppb
    }

    // roll over old content
    self.expire.ExpireGroup(group, rollover)
    // handle mod events
    if group == "ctl" {
      modchnl <- nntp
    }

    // queue to all outfeeds
    // XXX: blocking ?
    self.send_all_feeds <- ArticleEntry{msgid, group}
    // tell frontend
    // XXX: blocking ?
    if chnl != nil {
      if self.frontend.AllowNewsgroup(group) {
        chnl <- nntp
      } else {
        log.Println("frontend does not allow", group, "not sending")
      }
    }
  }
}

func (self NNTPDaemon) acceptloop() {
  for {
    // accept
    conn, err := self.listener.Accept()
    if err != nil {
      log.Fatal(err)
    }
    // make a new inbound nntp connection handler
    nntp := createNNTPConnection()
    addr := conn.RemoteAddr()
    nntp.name = fmt.Sprintf("%s-inbound-feed", addr.String())
    c := textproto.NewConn(conn)
    // send banners and shit
    err = nntp.inboundHandshake(c)
    if err == nil {
      // run, we support stream and reader
      go nntp.runConnection(self, true, true, true, "stream", c)
    } else {
      log.Println("failed to send banners", err)
      c.Close()
    }
  }
}

func (self NNTPDaemon) Setup() NNTPDaemon {
  log.Println("checking for configs...")
  // check that are configs exist
  CheckConfig()
  log.Println("loading config...")
  // read the config
  self.conf = ReadConfig()
  if self.conf == nil {
    log.Fatal("failed to load config")
  }
  // validate the config
  log.Println("validating configs...")
  self.conf.Validate()
  log.Println("configs are valid")

  db_host := self.conf.database["host"]
  db_port := self.conf.database["port"]
  db_user := self.conf.database["user"]
  db_passwd := self.conf.database["password"]

  // set up database stuff
  log.Println("connecting to database...")
  self.database = NewDatabase(self.conf.database["type"], self.conf.database["schema"], db_host, db_port, db_user, db_passwd)
  log.Println("ensure that the database is created...")
  self.database.CreateTables()

  // set up store
  log.Println("set up article store...")
  self.store = createArticleStore(self.conf.store, self.database)

  self.mod = modEngine{
    store: self.store,
    database:  self.database,
    chnl: make(chan NNTPMessage),
  }
  return self
}
<|endoftext|>


<commit_before>package fonts

// The list of font paths on Windows (could be improved to use registry)
var FontPaths = []string{
	`C:\WINDOWS\FONTS`,
}
<commit_msg>fix font paths for windows<commit_after>package fonts

// The list of font paths on Windows (could be improved to use registry)
var FontPaths = []string{
	`C:\WINDOWS\FONTS\*`,
}
<|endoftext|>