<commit_before>package api

import (
	"encoding/json"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/flexiant/concerto/api/types"
	"github.com/flexiant/concerto/utils"
	// "time"
)

// ReportService manages report operations
type ReportService struct {
	concertoService utils.ConcertoService
}

// NewReportService returns a Concerto report service
func NewReportService(concertoService utils.ConcertoService) (*ReportService, error) {
	if concertoService == nil {
		return nil, fmt.Errorf("Must initialize ConcertoService before using it")
	}

	return &ReportService{
		concertoService: concertoService,
	}, nil
}

// GetAdminReportList returns the list of reports as an array of Report
func (rs *ReportService) GetAdminReportList() (reports []types.Report, err error) {
	log.Debug("GetReportList")

	data, status, err := rs.concertoService.Get("/v1/admin/reports")
	if err != nil {
		return nil, err
	}

	if err = utils.CheckStandardStatus(status, data); err != nil {

	}

	if err = json.Unmarshal(data, &reports); err != nil {
		return nil, err
	}

	return reports, nil
}

// GetReport returns a report by its ID
func (rs *ReportService) GetAdminReport(ID string) (report *types.Report, err error) {
	log.Debug("GetReport")

	data, status, err := rs.concertoService.Get(fmt.Sprintf("/v1/admin/reports/%s", ID))
	if err != nil {
		return nil, err
	}

	if err = utils.CheckStandardStatus(status, data); err != nil {
		return nil, err
	}

	if err = json.Unmarshal(data, &report); err != nil {
		return nil, err
	}

	return report, nil
}
<commit_msg>add missing return<commit_after>package api

import (
	"encoding/json"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/flexiant/concerto/api/types"
	"github.com/flexiant/concerto/utils"
	// "time"
)

// ReportService manages report operations
type ReportService struct {
	concertoService utils.ConcertoService
}

// NewReportService returns a Concerto report service
func NewReportService(concertoService utils.ConcertoService) (*ReportService, error) {
	if concertoService == nil {
		return nil, fmt.Errorf("Must initialize ConcertoService before using it")
	}

	return &ReportService{
		concertoService: concertoService,
	}, nil
}

// GetAdminReportList returns the list of reports as an array of Report
func (rs *ReportService) GetAdminReportList() (reports []types.Report, err error) {
	log.Debug("GetReportList")

	data, status, err := rs.concertoService.Get("/v1/admin/reports")
	if err != nil {
		return nil, err
	}

	if err = utils.CheckStandardStatus(status, data); err != nil {
		return nil, err
	}

	if err = json.Unmarshal(data, &reports); err != nil {
		return nil, err
	}

	return reports, nil
}

// GetReport returns a report by its ID
func (rs *ReportService) GetAdminReport(ID string) (report *types.Report, err error) {
	log.Debug("GetReport")

	data, status, err := rs.concertoService.Get(fmt.Sprintf("/v1/admin/reports/%s", ID))
	if err != nil {
		return nil, err
	}

	if err = utils.CheckStandardStatus(status, data); err != nil {
		return nil, err
	}

	if err = json.Unmarshal(data, &report); err != nil {
		return nil, err
	}

	return report, nil
}
<|endoftext|>


<commit_before>package server

import (
	"bufio"
	"encoding/json"
	"io"
	"net/http"
	"runtime"
	"strings"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/api"
	"github.com/docker/docker/api/server/httputils"
	"github.com/docker/docker/dockerversion"
	"github.com/docker/docker/errors"
	"github.com/docker/docker/pkg/authorization"
	"github.com/docker/docker/pkg/ioutils"
	"github.com/docker/docker/pkg/version"
	"golang.org/x/net/context"
)

// middleware is an adapter to allow the use of ordinary functions as Docker API filters.
// Any function that has the appropriate signature can be register as a middleware.
type middleware func(handler httputils.APIFunc) httputils.APIFunc

// debugRequestMiddleware dumps the request to logger
func debugRequestMiddleware(handler httputils.APIFunc) httputils.APIFunc {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
		logrus.Debugf("%s %s", r.Method, r.RequestURI)

		if r.Method != "POST" {
			return handler(ctx, w, r, vars)
		}
		if err := httputils.CheckForJSON(r); err != nil {
			return handler(ctx, w, r, vars)
		}
		maxBodySize := 4096 // 4KB
		if r.ContentLength > int64(maxBodySize) {
			return handler(ctx, w, r, vars)
		}

		body := r.Body
		bufReader := bufio.NewReaderSize(body, maxBodySize)
		r.Body = ioutils.NewReadCloserWrapper(bufReader, func() error { return body.Close() })

		b, err := bufReader.Peek(maxBodySize)
		if err != io.EOF {
			// either there was an error reading, or the buffer is full (in which case the request is too large)
			return handler(ctx, w, r, vars)
		}

		var postForm map[string]interface{}
		if err := json.Unmarshal(b, &postForm); err == nil {
			if _, exists := postForm["password"]; exists {
				postForm["password"] = "*****"
			}
			formStr, errMarshal := json.Marshal(postForm)
			if errMarshal == nil {
				logrus.Debugf("form data: %s", string(formStr))
			} else {
				logrus.Debugf("form data: %q", postForm)
			}
		}

		return handler(ctx, w, r, vars)
	}
}

// authorizationMiddleware perform authorization on the request.
func (s *Server) authorizationMiddleware(handler httputils.APIFunc) httputils.APIFunc {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
		// FIXME: fill when authN gets in
		// User and UserAuthNMethod are taken from AuthN plugins
		// Currently tracked in https://github.com/docker/docker/pull/13994
		user := ""
		userAuthNMethod := ""
		authCtx := authorization.NewCtx(s.authZPlugins, user, userAuthNMethod, r.Method, r.RequestURI)

		if err := authCtx.AuthZRequest(w, r); err != nil {
			logrus.Errorf("AuthZRequest for %s %s returned error: %s", r.Method, r.RequestURI, err)
			return err
		}

		rw := authorization.NewResponseModifier(w)

		if err := handler(ctx, rw, r, vars); err != nil {
			logrus.Errorf("Handler for %s %s returned error: %s", r.Method, r.RequestURI, err)
			return err
		}

		if err := authCtx.AuthZResponse(rw, r); err != nil {
			logrus.Errorf("AuthZResponse for %s %s returned error: %s", r.Method, r.RequestURI, err)
			return err
		}
		return nil
	}
}

// userAgentMiddleware checks the User-Agent header looking for a valid docker client spec.
func (s *Server) userAgentMiddleware(handler httputils.APIFunc) httputils.APIFunc {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
		if strings.Contains(r.Header.Get("User-Agent"), "Docker-Client/") {
			dockerVersion := version.Version(s.cfg.Version)

			userAgent := strings.Split(r.Header.Get("User-Agent"), "/")

			// v1.20 onwards includes the GOOS of the client after the version
			// such as Docker/1.7.0 (linux)
			if len(userAgent) == 2 && strings.Contains(userAgent[1], " ") {
				userAgent[1] = strings.Split(userAgent[1], " ")[0]
			}

			if len(userAgent) == 2 && !dockerVersion.Equal(version.Version(userAgent[1])) {
				logrus.Warnf("Warning: client and server don't have the same version (client: %s, server: %s)", userAgent[1], dockerVersion)
			}
		}
		return handler(ctx, w, r, vars)
	}
}

// corsMiddleware sets the CORS header expectations in the server.
func (s *Server) corsMiddleware(handler httputils.APIFunc) httputils.APIFunc {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
		// If "api-cors-header" is not given, but "api-enable-cors" is true, we set cors to "*"
		// otherwise, all head values will be passed to HTTP handler
		corsHeaders := s.cfg.CorsHeaders
		if corsHeaders == "" && s.cfg.EnableCors {
			corsHeaders = "*"
		}

		if corsHeaders != "" {
			writeCorsHeaders(w, r, corsHeaders)
		}
		return handler(ctx, w, r, vars)
	}
}

// versionMiddleware checks the api version requirements before passing the request to the server handler.
func versionMiddleware(handler httputils.APIFunc) httputils.APIFunc {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
		apiVersion := version.Version(vars["version"])
		if apiVersion == "" {
			apiVersion = api.DefaultVersion
		}

		if apiVersion.GreaterThan(api.DefaultVersion) {
			return errors.ErrorCodeNewerClientVersion.WithArgs(apiVersion, api.DefaultVersion)
		}
		if apiVersion.LessThan(api.MinVersion) {
			return errors.ErrorCodeOldClientVersion.WithArgs(apiVersion, api.MinVersion)
		}

		w.Header().Set("Server", "Docker/"+dockerversion.Version+" ("+runtime.GOOS+")")
		ctx = context.WithValue(ctx, httputils.APIVersionKey, apiVersion)
		return handler(ctx, w, r, vars)
	}
}

// handleWithGlobalMiddlwares wraps the handler function for a request with
// the server's global middlewares. The order of the middlewares is backwards,
// meaning that the first in the list will be evaluated last.
//
// Example: handleWithGlobalMiddlewares(s.getContainersName)
//
//	s.loggingMiddleware(
//		s.userAgentMiddleware(
//			s.corsMiddleware(
//				versionMiddleware(s.getContainersName)
//			)
//		)
//	)
// )
func (s *Server) handleWithGlobalMiddlewares(handler httputils.APIFunc) httputils.APIFunc {
	middlewares := []middleware{
		versionMiddleware,
		s.corsMiddleware,
		s.userAgentMiddleware,
	}

	// Only want this on debug level
	if s.cfg.Logging && logrus.GetLevel() == logrus.DebugLevel {
		middlewares = append(middlewares, debugRequestMiddleware)
	}

	if len(s.cfg.AuthorizationPluginNames) > 0 {
		s.authZPlugins = authorization.NewPlugins(s.cfg.AuthorizationPluginNames)
		middlewares = append(middlewares, s.authorizationMiddleware)
	}

	h := handler
	for _, m := range middlewares {
		h = m(h)
	}
	return h
}
<commit_msg>Lower warning about old client to a debug<commit_after>package server

import (
	"bufio"
	"encoding/json"
	"io"
	"net/http"
	"runtime"
	"strings"

	"github.com/Sirupsen/logrus"
	"github.com/docker/docker/api"
	"github.com/docker/docker/api/server/httputils"
	"github.com/docker/docker/dockerversion"
	"github.com/docker/docker/errors"
	"github.com/docker/docker/pkg/authorization"
	"github.com/docker/docker/pkg/ioutils"
	"github.com/docker/docker/pkg/version"
	"golang.org/x/net/context"
)

// middleware is an adapter to allow the use of ordinary functions as Docker API filters.
// Any function that has the appropriate signature can be register as a middleware.
type middleware func(handler httputils.APIFunc) httputils.APIFunc

// debugRequestMiddleware dumps the request to logger
func debugRequestMiddleware(handler httputils.APIFunc) httputils.APIFunc {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
		logrus.Debugf("%s %s", r.Method, r.RequestURI)

		if r.Method != "POST" {
			return handler(ctx, w, r, vars)
		}
		if err := httputils.CheckForJSON(r); err != nil {
			return handler(ctx, w, r, vars)
		}
		maxBodySize := 4096 // 4KB
		if r.ContentLength > int64(maxBodySize) {
			return handler(ctx, w, r, vars)
		}

		body := r.Body
		bufReader := bufio.NewReaderSize(body, maxBodySize)
		r.Body = ioutils.NewReadCloserWrapper(bufReader, func() error { return body.Close() })

		b, err := bufReader.Peek(maxBodySize)
		if err != io.EOF {
			// either there was an error reading, or the buffer is full (in which case the request is too large)
			return handler(ctx, w, r, vars)
		}

		var postForm map[string]interface{}
		if err := json.Unmarshal(b, &postForm); err == nil {
			if _, exists := postForm["password"]; exists {
				postForm["password"] = "*****"
			}
			formStr, errMarshal := json.Marshal(postForm)
			if errMarshal == nil {
				logrus.Debugf("form data: %s", string(formStr))
			} else {
				logrus.Debugf("form data: %q", postForm)
			}
		}

		return handler(ctx, w, r, vars)
	}
}

// authorizationMiddleware perform authorization on the request.
func (s *Server) authorizationMiddleware(handler httputils.APIFunc) httputils.APIFunc {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
		// FIXME: fill when authN gets in
		// User and UserAuthNMethod are taken from AuthN plugins
		// Currently tracked in https://github.com/docker/docker/pull/13994
		user := ""
		userAuthNMethod := ""
		authCtx := authorization.NewCtx(s.authZPlugins, user, userAuthNMethod, r.Method, r.RequestURI)

		if err := authCtx.AuthZRequest(w, r); err != nil {
			logrus.Errorf("AuthZRequest for %s %s returned error: %s", r.Method, r.RequestURI, err)
			return err
		}

		rw := authorization.NewResponseModifier(w)

		if err := handler(ctx, rw, r, vars); err != nil {
			logrus.Errorf("Handler for %s %s returned error: %s", r.Method, r.RequestURI, err)
			return err
		}

		if err := authCtx.AuthZResponse(rw, r); err != nil {
			logrus.Errorf("AuthZResponse for %s %s returned error: %s", r.Method, r.RequestURI, err)
			return err
		}
		return nil
	}
}

// userAgentMiddleware checks the User-Agent header looking for a valid docker client spec.
func (s *Server) userAgentMiddleware(handler httputils.APIFunc) httputils.APIFunc {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
		if strings.Contains(r.Header.Get("User-Agent"), "Docker-Client/") {
			dockerVersion := version.Version(s.cfg.Version)

			userAgent := strings.Split(r.Header.Get("User-Agent"), "/")

			// v1.20 onwards includes the GOOS of the client after the version
			// such as Docker/1.7.0 (linux)
			if len(userAgent) == 2 && strings.Contains(userAgent[1], " ") {
				userAgent[1] = strings.Split(userAgent[1], " ")[0]
			}

			if len(userAgent) == 2 && !dockerVersion.Equal(version.Version(userAgent[1])) {
				logrus.Debug("Client and server don't have the same version (client: %s, server: %s)", userAgent[1], dockerVersion)
			}
		}
		return handler(ctx, w, r, vars)
	}
}

// corsMiddleware sets the CORS header expectations in the server.
func (s *Server) corsMiddleware(handler httputils.APIFunc) httputils.APIFunc {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
		// If "api-cors-header" is not given, but "api-enable-cors" is true, we set cors to "*"
		// otherwise, all head values will be passed to HTTP handler
		corsHeaders := s.cfg.CorsHeaders
		if corsHeaders == "" && s.cfg.EnableCors {
			corsHeaders = "*"
		}

		if corsHeaders != "" {
			writeCorsHeaders(w, r, corsHeaders)
		}
		return handler(ctx, w, r, vars)
	}
}

// versionMiddleware checks the api version requirements before passing the request to the server handler.
func versionMiddleware(handler httputils.APIFunc) httputils.APIFunc {
	return func(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
		apiVersion := version.Version(vars["version"])
		if apiVersion == "" {
			apiVersion = api.DefaultVersion
		}

		if apiVersion.GreaterThan(api.DefaultVersion) {
			return errors.ErrorCodeNewerClientVersion.WithArgs(apiVersion, api.DefaultVersion)
		}
		if apiVersion.LessThan(api.MinVersion) {
			return errors.ErrorCodeOldClientVersion.WithArgs(apiVersion, api.MinVersion)
		}

		w.Header().Set("Server", "Docker/"+dockerversion.Version+" ("+runtime.GOOS+")")
		ctx = context.WithValue(ctx, httputils.APIVersionKey, apiVersion)
		return handler(ctx, w, r, vars)
	}
}

// handleWithGlobalMiddlwares wraps the handler function for a request with
// the server's global middlewares. The order of the middlewares is backwards,
// meaning that the first in the list will be evaluated last.
//
// Example: handleWithGlobalMiddlewares(s.getContainersName)
//
//	s.loggingMiddleware(
//		s.userAgentMiddleware(
//			s.corsMiddleware(
//				versionMiddleware(s.getContainersName)
//			)
//		)
//	)
// )
func (s *Server) handleWithGlobalMiddlewares(handler httputils.APIFunc) httputils.APIFunc {
	middlewares := []middleware{
		versionMiddleware,
		s.corsMiddleware,
		s.userAgentMiddleware,
	}

	// Only want this on debug level
	if s.cfg.Logging && logrus.GetLevel() == logrus.DebugLevel {
		middlewares = append(middlewares, debugRequestMiddleware)
	}

	if len(s.cfg.AuthorizationPluginNames) > 0 {
		s.authZPlugins = authorization.NewPlugins(s.cfg.AuthorizationPluginNames)
		middlewares = append(middlewares, s.authorizationMiddleware)
	}

	h := handler
	for _, m := range middlewares {
		h = m(h)
	}
	return h
}
<|endoftext|>


<commit_before>package api

import (
	"net/http"
	"net/http/httptest"
	"testing"
	"time"

	"github.com/af83/edwig/core"
	"github.com/af83/edwig/model"
	"github.com/af83/edwig/siri"
)

func siriHandler_PrepareServer() (*Server, *core.Referential) {
	model.SetDefaultClock(model.NewFakeClock())
	defer model.SetDefaultClock(model.NewRealClock())

	// create a server with a fake clock and fake UUID generator
	server := NewTestServer()

	// Create the default referential with the appropriate connectors
	referential := server.CurrentReferentials().New("default")
	referential.Start()
	referential.Stop()

	partner := referential.Partners().New("partner")
	partner.Settings = map[string]string{
		"remote_url":           "",
		"remote_credential":    "",
		"remote_objectid_kind": "objectidKind",
		"local_credential":     "Edwig",
		"Address":              "edwig.edwig",
	}
	partner.ConnectorTypes = []string{"siri-check-status-client", "siri-stop-monitoring-request-collector"}
	partner.RefreshConnectors()
	siriPartner := core.NewSIRIPartner(partner)
	generator := core.NewFormatMessageIdentifierGenerator("Edwig:ResponseMessage::%v:LOC")
	generator.SetUUIDGenerator(model.NewFakeUUIDGenerator())
	siriPartner.SetMessageIdentifierGenerator(generator)
	partner.Context().SetValue(core.SIRI_PARTNER, siriPartner)

	partner.Save()
	referential.Save()

	return server, referential
}

func siriHandler_Request(server *Server, soapEnvelope *siri.SOAPEnvelopeBuffer, t *testing.T) *httptest.ResponseRecorder {
	model.SetDefaultClock(model.NewFakeClock())
	defer model.SetDefaultClock(model.NewRealClock())

	// Create a request
	request, err := http.NewRequest("POST", "/default/siri", soapEnvelope)
	if err != nil {
		t.Fatal(err)
	}

	// Create a ResponseRecorder
	responseRecorder := httptest.NewRecorder()
	handler := http.HandlerFunc(server.APIHandler)

	// Call ServeHTTP method and pass in our Request and ResponseRecorder.
	handler.ServeHTTP(responseRecorder, request)

	// Check the status code is what we expect.
	if status := responseRecorder.Code; status != http.StatusOK {
		t.Errorf("Handler returned wrong status code:\n got %v\n want %v",
			status, http.StatusOK)
	}

	if contentType := responseRecorder.Header().Get("Content-Type"); contentType != "text/xml" {
		t.Errorf("Handler returned wrong Content-Type:\n got: %v\n want: %v",
			contentType, "text/xml")
	}

	return responseRecorder
}

func Test_SIRIHandler_CheckStatus(t *testing.T) {
	// Generate the request Body
	soapEnvelope := siri.NewSOAPEnvelopeBuffer()
	soapEnvelope.WriteXML(siri.NewSIRICheckStatusRequest("Edwig",
		model.DefaultClock().Now(),
		"Edwig:Message::6ba7b814-9dad-11d1-0-00c04fd430c8:LOC").BuildXML())

	server, _ := siriHandler_PrepareServer()
	responseRecorder := siriHandler_Request(server, soapEnvelope, t)

	// Check the response body is what we expect.
	response, err := siri.NewXMLCheckStatusResponseFromContent(responseRecorder.Body.Bytes())
	if err != nil {
		t.Fatal(err)
	}

	if expected := "edwig.edwig"; response.Address() != expected {
		t.Errorf("Wrong Address in response:\n got: %v\n want: %v", response.Address(), expected)
	}

	if expected := "Edwig"; response.ProducerRef() != expected {
		t.Errorf("Wrong ProducerRef in response:\n got: %v\n want: %v", response.ProducerRef(), expected)
	}

	if expected := "Edwig:Message::6ba7b814-9dad-11d1-0-00c04fd430c8:LOC"; response.RequestMessageRef() != expected {
		t.Errorf("Wrong RequestMessageRef in response:\n got: %v\n want: %v", response.RequestMessageRef(), expected)
	}

	if expected := "Edwig:ResponseMessage::6ba7b814-9dad-11d1-0-00c04fd430c8:LOC"; response.ResponseMessageIdentifier() != expected {
		t.Errorf("Wrong ResponseMessageIdentifier in response:\n got: %v\n want: %v", response.ResponseMessageIdentifier(), expected)
	}

	if !response.Status() {
		t.Errorf("Wrong Status in response:\n got: %v\n want: true", response.Status())
	}

	expectedDate := time.Date(1984, time.April, 4, 0, 0, 0, 0, time.UTC)
	if !response.ResponseTimestamp().Equal(expectedDate) {
		t.Errorf("Wrong ResponseTimestamp in response:\n got: %v\n want: %v", response.ResponseTimestamp(), expectedDate)
	}

	if !response.ServiceStartedTime().Equal(expectedDate) {
		t.Errorf("Wrong ServiceStartedTime in response:\n got: %v\n want: %v", response.ServiceStartedTime(), expectedDate)
	}
}

func Test_SIRIHandler_StopMonitoring(t *testing.T) {
	// Generate the request Body
	soapEnvelope := siri.NewSOAPEnvelopeBuffer()
	soapEnvelope.WriteXML(siri.NewSIRIStopMonitoringRequest("Edwig:Message::6ba7b814-9dad-11d1-0-00c04fd430c8:LOC",
		"objectidValue",
		"Edwig",
		model.DefaultClock().Now()).BuildXML())

	server, referential := siriHandler_PrepareServer()
	stopArea := referential.Model().StopAreas().New()
	objectid := model.NewObjectID("objectidKind", "objectidValue")
	stopArea.SetObjectID(objectid)
	stopArea.Save()

	stopVisit := referential.Model().StopVisits().New()
	stopVisit.SetStopAreaId(stopArea.Id())
	stopVisit.SetObjectID(objectid)
	stopVisit.Save()

	responseRecorder := siriHandler_Request(server, soapEnvelope, t)

	// Check the response body is what we expect.
	response, err := siri.NewXMLStopMonitoringResponseFromContent(responseRecorder.Body.Bytes())
	if err != nil {
		t.Fatal(err)
	}

	if expected := "edwig.edwig"; response.Address() != expected {
		t.Errorf("Wrong Address in response:\n got: %v\n want: %v", response.Address(), expected)
	}

	if expected := "Edwig"; response.ProducerRef() != expected {
		t.Errorf("Wrong ProducerRef in response:\n got: %v\n want: %v", response.ProducerRef(), expected)
	}

	if expected := "Edwig:Message::6ba7b814-9dad-11d1-0-00c04fd430c8:LOC"; response.RequestMessageRef() != expected {
		t.Errorf("Wrong RequestMessageRef in response:\n got: %v\n want: %v", response.RequestMessageRef(), expected)
	}

	if expected := "Edwig:ResponseMessage::6ba7b814-9dad-11d1-0-00c04fd430c8:LOC"; response.ResponseMessageIdentifier() != expected {
		t.Errorf("Wrong ResponseMessageIdentifier in response:\n got: %v\n want: %v", response.ResponseMessageIdentifier(), expected)
	}

	expectedDate := time.Date(1984, time.April, 4, 0, 0, 0, 0, time.UTC)
	if !response.ResponseTimestamp().Equal(expectedDate) {
		t.Errorf("Wrong ResponseTimestamp in response:\n got: %v\n want: %v", response.ResponseTimestamp(), expectedDate)
	}
}
<commit_msg>fix test<commit_after>package api

import (
	"net/http"
	"net/http/httptest"
	"testing"
	"time"

	"github.com/af83/edwig/core"
	"github.com/af83/edwig/model"
	"github.com/af83/edwig/siri"
)

func siriHandler_PrepareServer() (*Server, *core.Referential) {
	model.SetDefaultClock(model.NewFakeClock())
	defer model.SetDefaultClock(model.NewRealClock())

	// create a server with a fake clock and fake UUID generator
	server := NewTestServer()

	// Create the default referential with the appropriate connectors
	referential := server.CurrentReferentials().New("default")
	referential.Start()
	referential.Stop()

	partner := referential.Partners().New("partner")
	partner.Settings = map[string]string{
		"remote_url":           "",
		"remote_credential":    "",
		"remote_objectid_kind": "objectidKind",
		"local_credential":     "Edwig",
		"Address":              "edwig.edwig",
	}
	partner.ConnectorTypes = []string{"siri-check-status-client", "siri-stop-monitoring-request-collector"}
	partner.RefreshConnectors()
	siriPartner := core.NewSIRIPartner(partner)
	generator := core.NewFormatMessageIdentifierGenerator("Edwig:ResponseMessage::%v:LOC")
	generator.SetUUIDGenerator(model.NewFakeUUIDGenerator())
	siriPartner.SetMessageIdentifierGenerator(generator)
	partner.Context().SetValue(core.SIRI_PARTNER, siriPartner)

	partner.Save()
	referential.Save()

	return server, referential
}

func siriHandler_Request(server *Server, soapEnvelope *siri.SOAPEnvelopeBuffer, t *testing.T) *httptest.ResponseRecorder {
	model.SetDefaultClock(model.NewFakeClock())
	defer model.SetDefaultClock(model.NewRealClock())

	// Create a request
	request, err := http.NewRequest("POST", "/default/siri", soapEnvelope)
	if err != nil {
		t.Fatal(err)
	}

	// Create a ResponseRecorder
	responseRecorder := httptest.NewRecorder()
	handler := http.HandlerFunc(server.APIHandler)

	// Call ServeHTTP method and pass in our Request and ResponseRecorder.
	handler.ServeHTTP(responseRecorder, request)

	// Check the status code is what we expect.
	if status := responseRecorder.Code; status != http.StatusOK {
		t.Errorf("Handler returned wrong status code:\n got %v\n want %v",
			status, http.StatusOK)
	}

	if contentType := responseRecorder.Header().Get("Content-Type"); contentType != "text/xml" {
		t.Errorf("Handler returned wrong Content-Type:\n got: %v\n want: %v",
			contentType, "text/xml")
	}

	return responseRecorder
}

func Test_SIRIHandler_CheckStatus(t *testing.T) {
	// Generate the request Body
	soapEnvelope := siri.NewSOAPEnvelopeBuffer()
	request, err := siri.NewSIRICheckStatusRequest("Edwig",
		model.DefaultClock().Now(),
		"Edwig:Message::6ba7b814-9dad-11d1-0-00c04fd430c8:LOC").BuildXML()
	if err != nil {
		t.Fatal(err)
	}
	soapEnvelope.WriteXML(request)

	server, _ := siriHandler_PrepareServer()
	responseRecorder := siriHandler_Request(server, soapEnvelope, t)

	// Check the response body is what we expect.
	response, err := siri.NewXMLCheckStatusResponseFromContent(responseRecorder.Body.Bytes())
	if err != nil {
		t.Fatal(err)
	}

	if expected := "edwig.edwig"; response.Address() != expected {
		t.Errorf("Wrong Address in response:\n got: %v\n want: %v", response.Address(), expected)
	}

	if expected := "Edwig"; response.ProducerRef() != expected {
		t.Errorf("Wrong ProducerRef in response:\n got: %v\n want: %v", response.ProducerRef(), expected)
	}

	if expected := "Edwig:Message::6ba7b814-9dad-11d1-0-00c04fd430c8:LOC"; response.RequestMessageRef() != expected {
		t.Errorf("Wrong RequestMessageRef in response:\n got: %v\n want: %v", response.RequestMessageRef(), expected)
	}

	if expected := "Edwig:ResponseMessage::6ba7b814-9dad-11d1-0-00c04fd430c8:LOC"; response.ResponseMessageIdentifier() != expected {
		t.Errorf("Wrong ResponseMessageIdentifier in response:\n got: %v\n want: %v", response.ResponseMessageIdentifier(), expected)
	}

	if !response.Status() {
		t.Errorf("Wrong Status in response:\n got: %v\n want: true", response.Status())
	}

	expectedDate := time.Date(1984, time.April, 4, 0, 0, 0, 0, time.UTC)
	if !response.ResponseTimestamp().Equal(expectedDate) {
		t.Errorf("Wrong ResponseTimestamp in response:\n got: %v\n want: %v", response.ResponseTimestamp(), expectedDate)
	}

	if !response.ServiceStartedTime().Equal(expectedDate) {
		t.Errorf("Wrong ServiceStartedTime in response:\n got: %v\n want: %v", response.ServiceStartedTime(), expectedDate)
	}
}

func Test_SIRIHandler_StopMonitoring(t *testing.T) {
	// Generate the request Body
	soapEnvelope := siri.NewSOAPEnvelopeBuffer()
	request, err := siri.NewSIRIStopMonitoringRequest("Edwig:Message::6ba7b814-9dad-11d1-0-00c04fd430c8:LOC",
		"objectidValue",
		"Edwig",
		model.DefaultClock().Now()).BuildXML()
	if err != nil {
		t.Fatal(err)
	}
	soapEnvelope.WriteXML(request)

	server, referential := siriHandler_PrepareServer()
	stopArea := referential.Model().StopAreas().New()
	objectid := model.NewObjectID("objectidKind", "objectidValue")
	stopArea.SetObjectID(objectid)
	stopArea.Save()

	stopVisit := referential.Model().StopVisits().New()
	stopVisit.SetStopAreaId(stopArea.Id())
	stopVisit.SetObjectID(objectid)
	stopVisit.Save()

	responseRecorder := siriHandler_Request(server, soapEnvelope, t)

	// Check the response body is what we expect.
	response, err := siri.NewXMLStopMonitoringResponseFromContent(responseRecorder.Body.Bytes())
	if err != nil {
		t.Fatal(err)
	}

	if expected := "edwig.edwig"; response.Address() != expected {
		t.Errorf("Wrong Address in response:\n got: %v\n want: %v", response.Address(), expected)
	}

	if expected := "Edwig"; response.ProducerRef() != expected {
		t.Errorf("Wrong ProducerRef in response:\n got: %v\n want: %v", response.ProducerRef(), expected)
	}

	if expected := "Edwig:Message::6ba7b814-9dad-11d1-0-00c04fd430c8:LOC"; response.RequestMessageRef() != expected {
		t.Errorf("Wrong RequestMessageRef in response:\n got: %v\n want: %v", response.RequestMessageRef(), expected)
	}

	if expected := "Edwig:ResponseMessage::6ba7b814-9dad-11d1-0-00c04fd430c8:LOC"; response.ResponseMessageIdentifier() != expected {
		t.Errorf("Wrong ResponseMessageIdentifier in response:\n got: %v\n want: %v", response.ResponseMessageIdentifier(), expected)
	}

	expectedDate := time.Date(1984, time.April, 4, 0, 0, 0, 0, time.UTC)
	if !response.ResponseTimestamp().Equal(expectedDate) {
		t.Errorf("Wrong ResponseTimestamp in response:\n got: %v\n want: %v", response.ResponseTimestamp(), expectedDate)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package federation

import (
	"k8s.io/kubernetes/pkg/api"
	metav1 "k8s.io/kubernetes/pkg/apis/meta/v1"
)

// ServerAddressByClientCIDR helps the client to determine the server address that they should use, depending on the clientCIDR that they match.
type ServerAddressByClientCIDR struct {
	// The CIDR with which clients can match their IP to figure out the server address that they should use.
	ClientCIDR string `json:"clientCIDR" protobuf:"bytes,1,opt,name=clientCIDR"`
	// Address of this server, suitable for a client that matches the above CIDR.
	// This can be a hostname, hostname:port, IP or IP:port.
	ServerAddress string `json:"serverAddress" protobuf:"bytes,2,opt,name=serverAddress"`
}

// ClusterSpec describes the attributes of a kubernetes cluster.
type ClusterSpec struct {
	// A map of client CIDR to server address.
	// This is to help clients reach servers in the most network-efficient way possible.
	// Clients can use the appropriate server address as per the CIDR that they match.
	// In case of multiple matches, clients should use the longest matching CIDR.
	ServerAddressByClientCIDRs []ServerAddressByClientCIDR `json:"serverAddressByClientCIDRs" patchStrategy:"merge" patchMergeKey:"clientCIDR"`
	// Name of the secret containing kubeconfig to access this cluster.
	// The secret is read from the kubernetes cluster that is hosting federation control plane.
	// Admin needs to ensure that the required secret exists. Secret should be in the same namespace where federation control plane is hosted and it should have kubeconfig in its data with key "kubeconfig".
	// This will later be changed to a reference to secret in federation control plane when the federation control plane supports secrets.
	// This can be left empty if the cluster allows insecure access.
	// +optional
	SecretRef *api.LocalObjectReference `json:"secretRef,omitempty"`
}

type ClusterConditionType string

// These are valid conditions of a cluster.
const (
	// ClusterReady means the cluster is ready to accept workloads.
	ClusterReady ClusterConditionType = "Ready"
	// ClusterOffline means the cluster is temporarily down or not reachable
	ClusterOffline ClusterConditionType = "Offline"
)

// ClusterCondition describes current state of a cluster.
type ClusterCondition struct {
	// Type of cluster condition, Complete or Failed.
	Type ClusterConditionType `json:"type"`
	// Status of the condition, one of True, False, Unknown.
	Status api.ConditionStatus `json:"status"`
	// Last time the condition was checked.
	// +optional
	LastProbeTime metav1.Time `json:"lastProbeTime,omitempty"`
	// Last time the condition transit from one status to another.
	// +optional
	LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"`
	// (brief) reason for the condition's last transition.
	// +optional
	Reason string `json:"reason,omitempty"`
	// Human readable message indicating details about last transition.
	// +optional
	Message string `json:"message,omitempty"`
}

// ClusterStatus is information about the current status of a cluster updated by cluster controller peridocally.
type ClusterStatus struct {
	// Conditions is an array of current cluster conditions.
	// +optional
	Conditions []ClusterCondition `json:"conditions,omitempty"`
	// Zones is the list of availability zones in which the nodes of the cluster exist, e.g. 'us-east1-a'.
	// These will always be in the same region.
	// +optional
	Zones []string `json:"zones,omitempty"`
	// Region is the name of the region in which all of the nodes in the cluster exist.  e.g. 'us-east1'.
	// +optional
	Region string `json:"region,omitempty"`
}

// +genclient=true
// +nonNamespaced=true

// Information about a registered cluster in a federated kubernetes setup. Clusters are not namespaced and have unique names in the federation.
type Cluster struct {
	metav1.TypeMeta `json:",inline"`
	// Standard object's metadata.
	// More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#metadata
	// +optional
	api.ObjectMeta `json:"metadata,omitempty"`

	// Spec defines the behavior of the Cluster.
	// +optional
	Spec ClusterSpec `json:"spec,omitempty"`
	// Status describes the current status of a Cluster
	// +optional
	Status ClusterStatus `json:"status,omitempty"`
}

// A list of all the kubernetes clusters registered to the federation
type ClusterList struct {
	metav1.TypeMeta `json:",inline"`
	// Standard list metadata.
	// More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#types-kinds
	// +optional
	metav1.ListMeta `json:"metadata,omitempty"`

	// List of Cluster objects.
	Items []Cluster `json:"items"`
}

// Temporary/alpha structures to support custom replica assignments within FederatedReplicaSet.

// A set of preferences that can be added to federated version of ReplicaSet as a json-serialized annotation.
// The preferences allow the user to express in which culsters he wants to put his replicas within the
// mentiond FederatedReplicaSet.
type FederatedReplicaSetPreferences struct {
	// If set to true then already scheduled and running replicas may be moved to other clusters to
	// in order to bring cluster replicasets towards a desired state. Otherwise, if set to false,
	// up and running replicas will not be moved.
	// +optional
	Rebalance bool `json:"rebalance,omitempty"`

	// A mapping between cluster names and preferences regarding local ReplicaSet in these clusters.
	// "*" (if provided) applies to all clusters if an explicit mapping is not provided. If there is no
	// "*" that clusters without explicit preferences should not have any replicas scheduled.
	// +optional
	Clusters map[string]ClusterReplicaSetPreferences `json:"clusters,omitempty"`
}

// Preferences regarding number of replicas assigned to a cluster replicaset within a federated replicaset.
type ClusterReplicaSetPreferences struct {
	// Minimum number of replicas that should be assigned to this Local ReplicaSet. 0 by default.
	// +optional
	MinReplicas int64 `json:"minReplicas,omitempty"`

	// Maximum number of replicas that should be assigned to this Local ReplicaSet. Unbounded if no value provided (default).
	// +optional
	MaxReplicas *int64 `json:"maxReplicas,omitempty"`

	// A number expressing the preference to put an additional replica to this LocalReplicaSet. 0 by default.
	Weight int64
}
<commit_msg>Remove json serialization annotations from internal types<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package federation

import (
	"k8s.io/kubernetes/pkg/api"
	metav1 "k8s.io/kubernetes/pkg/apis/meta/v1"
)

// ServerAddressByClientCIDR helps the client to determine the server address that they should use, depending on the clientCIDR that they match.
type ServerAddressByClientCIDR struct {
	// The CIDR with which clients can match their IP to figure out the server address that they should use.
	ClientCIDR string
	// Address of this server, suitable for a client that matches the above CIDR.
	// This can be a hostname, hostname:port, IP or IP:port.
	ServerAddress string
}

// ClusterSpec describes the attributes of a kubernetes cluster.
type ClusterSpec struct {
	// A map of client CIDR to server address.
	// This is to help clients reach servers in the most network-efficient way possible.
	// Clients can use the appropriate server address as per the CIDR that they match.
	// In case of multiple matches, clients should use the longest matching CIDR.
	ServerAddressByClientCIDRs []ServerAddressByClientCIDR
	// Name of the secret containing kubeconfig to access this cluster.
	// The secret is read from the kubernetes cluster that is hosting federation control plane.
	// Admin needs to ensure that the required secret exists. Secret should be in the same namespace where federation control plane is hosted and it should have kubeconfig in its data with key "kubeconfig".
	// This will later be changed to a reference to secret in federation control plane when the federation control plane supports secrets.
	// This can be left empty if the cluster allows insecure access.
	// +optional
	SecretRef *api.LocalObjectReference
}

type ClusterConditionType string

// These are valid conditions of a cluster.
const (
	// ClusterReady means the cluster is ready to accept workloads.
	ClusterReady ClusterConditionType = "Ready"
	// ClusterOffline means the cluster is temporarily down or not reachable
	ClusterOffline ClusterConditionType = "Offline"
)

// ClusterCondition describes current state of a cluster.
type ClusterCondition struct {
	// Type of cluster condition, Complete or Failed.
	Type ClusterConditionType
	// Status of the condition, one of True, False, Unknown.
	Status api.ConditionStatus
	// Last time the condition was checked.
	// +optional
	LastProbeTime metav1.Time
	// Last time the condition transit from one status to another.
	// +optional
	LastTransitionTime metav1.Time
	// (brief) reason for the condition's last transition.
	// +optional
	Reason string
	// Human readable message indicating details about last transition.
	// +optional
	Message string
}

// ClusterStatus is information about the current status of a cluster updated by cluster controller peridocally.
type ClusterStatus struct {
	// Conditions is an array of current cluster conditions.
	// +optional
	Conditions []ClusterCondition
	// Zones is the list of availability zones in which the nodes of the cluster exist, e.g. 'us-east1-a'.
	// These will always be in the same region.
	// +optional
	Zones []string
	// Region is the name of the region in which all of the nodes in the cluster exist.  e.g. 'us-east1'.
	// +optional
	Region string
}

// +genclient=true
// +nonNamespaced=true

// Information about a registered cluster in a federated kubernetes setup. Clusters are not namespaced and have unique names in the federation.
type Cluster struct {
	metav1.TypeMeta
	// Standard object's metadata.
	// More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#metadata
	// +optional
	api.ObjectMeta

	// Spec defines the behavior of the Cluster.
	// +optional
	Spec ClusterSpec
	// Status describes the current status of a Cluster
	// +optional
	Status ClusterStatus
}

// A list of all the kubernetes clusters registered to the federation
type ClusterList struct {
	metav1.TypeMeta
	// Standard list metadata.
	// More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#types-kinds
	// +optional
	metav1.ListMeta

	// List of Cluster objects.
	Items []Cluster
}

// Temporary/alpha structures to support custom replica assignments within FederatedReplicaSet.

// A set of preferences that can be added to federated version of ReplicaSet as a json-serialized annotation.
// The preferences allow the user to express in which culsters he wants to put his replicas within the
// mentiond FederatedReplicaSet.
type FederatedReplicaSetPreferences struct {
	// If set to true then already scheduled and running replicas may be moved to other clusters to
	// in order to bring cluster replicasets towards a desired state. Otherwise, if set to false,
	// up and running replicas will not be moved.
	// +optional
	Rebalance bool

	// A mapping between cluster names and preferences regarding local ReplicaSet in these clusters.
	// "*" (if provided) applies to all clusters if an explicit mapping is not provided. If there is no
	// "*" that clusters without explicit preferences should not have any replicas scheduled.
	// +optional
	Clusters map[string]ClusterReplicaSetPreferences
}

// Preferences regarding number of replicas assigned to a cluster replicaset within a federated replicaset.
type ClusterReplicaSetPreferences struct {
	// Minimum number of replicas that should be assigned to this Local ReplicaSet. 0 by default.
	// +optional
	MinReplicas int64

	// Maximum number of replicas that should be assigned to this Local ReplicaSet. Unbounded if no value provided (default).
	// +optional
	MaxReplicas *int64

	// A number expressing the preference to put an additional replica to this LocalReplicaSet. 0 by default.
	Weight int64
}
<|endoftext|>


<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package apiserver_test

import (
	"time"

	"github.com/juju/loggo"
	gitjujutesting "github.com/juju/testing"
	jc "github.com/juju/testing/checkers"
	"github.com/juju/utils"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/api"
	"github.com/juju/juju/apiserver"
	"github.com/juju/juju/juju/testing"
	"github.com/juju/juju/rpc"
	coretesting "github.com/juju/juju/testing"
)

type pingerSuite struct {
	testing.JujuConnSuite
}

var _ = gc.Suite(&pingerSuite{})

var testPingPeriod = 100 * time.Millisecond

func (s *pingerSuite) TestConnectionBrokenDetection(c *gc.C) {
	s.PatchValue(&api.PingPeriod, testPingPeriod)

	st, _ := s.OpenAPIAsNewMachine(c)

	// Connection still alive
	select {
	case <-time.After(testPingPeriod):
	case <-st.Broken():
		c.Fatalf("connection should be alive still")
	}

	// Close the connection and see if we detect this
	go st.Close()

	// Check it's detected
	select {
	case <-time.After(testPingPeriod + time.Second):
		c.Fatalf("connection not closed as expected")
	case <-st.Broken():
		return
	}
}

func (s *pingerSuite) TestPing(c *gc.C) {
	tw := &loggo.TestWriter{}
	c.Assert(loggo.RegisterWriter("ping-tester", tw, loggo.DEBUG), gc.IsNil)
	defer loggo.RemoveWriter("ping-tester")

	st, _ := s.OpenAPIAsNewMachine(c)
	err := st.Ping()
	c.Assert(err, jc.ErrorIsNil)
	err = st.Close()
	c.Assert(err, jc.ErrorIsNil)
	err = st.Ping()
	c.Assert(err, gc.Equals, rpc.ErrShutdown)

	// Make sure that ping messages have not been logged.
	for _, m := range tw.Log() {
		c.Logf("checking %q", m.Message)
		c.Check(m.Message, gc.Not(gc.Matches), `.*"Request":"Ping".*`)
	}
}

func (s *pingerSuite) TestClientNoNeedToPing(c *gc.C) {
	s.PatchValue(apiserver.MaxClientPingInterval, time.Duration(0))
	st, err := api.Open(s.APIInfo(c), api.DefaultDialOpts())
	c.Assert(err, jc.ErrorIsNil)
	defer st.Close()
	time.Sleep(coretesting.ShortWait)
	err = st.Ping()
	c.Assert(err, jc.ErrorIsNil)
}

func (s *pingerSuite) TestAgentConnectionShutsDownWithNoPing(c *gc.C) {
	s.PatchValue(apiserver.MaxClientPingInterval, time.Duration(0))
	st, _ := s.OpenAPIAsNewMachine(c)
	time.Sleep(coretesting.ShortWait)
	err := st.Ping()
	c.Assert(err, gc.ErrorMatches, "connection is shut down")
}

func (s *pingerSuite) TestAgentConnectionDelaysShutdownWithPing(c *gc.C) {
	// We have to be careful, because Login can take 25ms, so we ping
	// immediately after connecting.
	s.PatchValue(apiserver.MaxClientPingInterval, 50*time.Millisecond)
	st, _ := s.OpenAPIAsNewMachine(c)
	err := st.Ping()
	c.Assert(err, jc.ErrorIsNil)
	// As long as we don't wait too long, the connection stays open
	for i := 0; i < 10; i++ {
		time.Sleep(10 * time.Millisecond)
		err = st.Ping()
		c.Assert(err, jc.ErrorIsNil)
	}
	// However, once we stop pinging for too long, the connection dies
	time.Sleep(75 * time.Millisecond)
	err = st.Ping()
	c.Assert(err, gc.ErrorMatches, "connection is shut down")
}

type mongoPingerSuite struct {
	testing.JujuConnSuite
}

var _ = gc.Suite(&mongoPingerSuite{})

func (s *mongoPingerSuite) SetUpSuite(c *gc.C) {
	s.JujuConnSuite.SetUpSuite(c)
	// We need to set the ping interval before the server is started in test setup.
	restore := gitjujutesting.PatchValue(apiserver.MongoPingInterval, coretesting.ShortWait)
	s.AddSuiteCleanup(func(*gc.C) { restore() })
}

func (s *mongoPingerSuite) TestAgentConnectionsShutDownWhenStateDies(c *gc.C) {
	st, _ := s.OpenAPIAsNewMachine(c)
	err := st.Ping()
	c.Assert(err, jc.ErrorIsNil)
	gitjujutesting.MgoServer.Destroy()

	attempt := utils.AttemptStrategy{
		Total: coretesting.LongWait,
		Delay: coretesting.ShortWait,
	}
	for a := attempt.Start(); a.Next(); {
		if err := st.Ping(); err != nil {
			c.Assert(err, gc.ErrorMatches, "connection is shut down")
			return
		}
	}
	c.Fatalf("timed out waiting for API server to die")
}
<commit_msg>Disable pinger tests on i386<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

// TODO(wallyworld) bug http://pad.lv/1408459
// Re-enable tests for i386 when these tests are fixed to work on that architecture.

// +build !386

package apiserver_test

import (
	"time"

	"github.com/juju/loggo"
	gitjujutesting "github.com/juju/testing"
	jc "github.com/juju/testing/checkers"
	"github.com/juju/utils"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/api"
	"github.com/juju/juju/apiserver"
	"github.com/juju/juju/juju/testing"
	"github.com/juju/juju/rpc"
	coretesting "github.com/juju/juju/testing"
)

type pingerSuite struct {
	testing.JujuConnSuite
}

var _ = gc.Suite(&pingerSuite{})

var testPingPeriod = 100 * time.Millisecond

func (s *pingerSuite) TestConnectionBrokenDetection(c *gc.C) {
	s.PatchValue(&api.PingPeriod, testPingPeriod)

	st, _ := s.OpenAPIAsNewMachine(c)

	// Connection still alive
	select {
	case <-time.After(testPingPeriod):
	case <-st.Broken():
		c.Fatalf("connection should be alive still")
	}

	// Close the connection and see if we detect this
	go st.Close()

	// Check it's detected
	select {
	case <-time.After(testPingPeriod + time.Second):
		c.Fatalf("connection not closed as expected")
	case <-st.Broken():
		return
	}
}

func (s *pingerSuite) TestPing(c *gc.C) {
	tw := &loggo.TestWriter{}
	c.Assert(loggo.RegisterWriter("ping-tester", tw, loggo.DEBUG), gc.IsNil)
	defer loggo.RemoveWriter("ping-tester")

	st, _ := s.OpenAPIAsNewMachine(c)
	err := st.Ping()
	c.Assert(err, jc.ErrorIsNil)
	err = st.Close()
	c.Assert(err, jc.ErrorIsNil)
	err = st.Ping()
	c.Assert(err, gc.Equals, rpc.ErrShutdown)

	// Make sure that ping messages have not been logged.
	for _, m := range tw.Log() {
		c.Logf("checking %q", m.Message)
		c.Check(m.Message, gc.Not(gc.Matches), `.*"Request":"Ping".*`)
	}
}

func (s *pingerSuite) TestClientNoNeedToPing(c *gc.C) {
	s.PatchValue(apiserver.MaxClientPingInterval, time.Duration(0))
	st, err := api.Open(s.APIInfo(c), api.DefaultDialOpts())
	c.Assert(err, jc.ErrorIsNil)
	defer st.Close()
	time.Sleep(coretesting.ShortWait)
	err = st.Ping()
	c.Assert(err, jc.ErrorIsNil)
}

func (s *pingerSuite) TestAgentConnectionShutsDownWithNoPing(c *gc.C) {
	s.PatchValue(apiserver.MaxClientPingInterval, time.Duration(0))
	st, _ := s.OpenAPIAsNewMachine(c)
	time.Sleep(coretesting.ShortWait)
	err := st.Ping()
	c.Assert(err, gc.ErrorMatches, "connection is shut down")
}

func (s *pingerSuite) TestAgentConnectionDelaysShutdownWithPing(c *gc.C) {
	// We have to be careful, because Login can take 25ms, so we ping
	// immediately after connecting.
	s.PatchValue(apiserver.MaxClientPingInterval, 50*time.Millisecond)
	st, _ := s.OpenAPIAsNewMachine(c)
	err := st.Ping()
	c.Assert(err, jc.ErrorIsNil)
	// As long as we don't wait too long, the connection stays open
	for i := 0; i < 10; i++ {
		time.Sleep(10 * time.Millisecond)
		err = st.Ping()
		c.Assert(err, jc.ErrorIsNil)
	}
	// However, once we stop pinging for too long, the connection dies
	time.Sleep(75 * time.Millisecond)
	err = st.Ping()
	c.Assert(err, gc.ErrorMatches, "connection is shut down")
}

type mongoPingerSuite struct {
	testing.JujuConnSuite
}

var _ = gc.Suite(&mongoPingerSuite{})

func (s *mongoPingerSuite) SetUpSuite(c *gc.C) {
	s.JujuConnSuite.SetUpSuite(c)
	// We need to set the ping interval before the server is started in test setup.
	restore := gitjujutesting.PatchValue(apiserver.MongoPingInterval, coretesting.ShortWait)
	s.AddSuiteCleanup(func(*gc.C) { restore() })
}

func (s *mongoPingerSuite) TestAgentConnectionsShutDownWhenStateDies(c *gc.C) {
	st, _ := s.OpenAPIAsNewMachine(c)
	err := st.Ping()
	c.Assert(err, jc.ErrorIsNil)
	gitjujutesting.MgoServer.Destroy()

	attempt := utils.AttemptStrategy{
		Total: coretesting.LongWait,
		Delay: coretesting.ShortWait,
	}
	for a := attempt.Start(); a.Next(); {
		if err := st.Ping(); err != nil {
			c.Assert(err, gc.ErrorMatches, "connection is shut down")
			return
		}
	}
	c.Fatalf("timed out waiting for API server to die")
}
<|endoftext|>


<commit_before>package transport

import (
	"crypto/hmac"
	"crypto/rand"
	"crypto/sha256"
	"crypto/subtle"
	"errors"
	"hash"
	"io"
	"strconv"
	"time"

	pond "github.com/unixninja92/Div-III-Server/protos"
	"github.com/golang/protobuf/proto"
	"golang.org/x/crypto/curve25519"
	"golang.org/x/crypto/nacl/secretbox"
)

// blockSize is the size of the blocks of data that we'll send and receive when
// working in streaming mode. Each block is prefixed by two length bytes (which
// aren't counted in blockSize) and includes secretbox.Overhead bytes of MAC
// tag (which are).
const blockSize = 4096 - 2

type Conn struct {
	conn                     io.ReadWriteCloser
	isServer                 bool
	identity, identityPublic [32]byte
	Peer                     [32]byte

	writeKey, readKey           [32]byte
	writeKeyValid, readKeyValid bool
	writeSequence, readSequence [24]byte

	// readBuffer is used to receive bytes from the network when this Conn
	// is used to stream data.
	readBuffer []byte
	// decryptBuffer is used to store decrypted payloads when this Conn is
	// used to stream data and the caller's buffer isn't large enough to
	// decrypt into directly.
	decryptBuffer []byte
	// readPending aliases into decryptBuffer when a partial decryption had
	// to be returned to a caller because of buffer size limitations.
	readPending []byte

	// writeBuffer is used to hold encrypted payloads when this Conn is
	// used for streaming data.
	writeBuffer []byte
}

func NewServer(conn io.ReadWriteCloser, identity *[32]byte) *Conn {
	c := &Conn{
		conn:     conn,
		isServer: true,
	}
	copy(c.identity[:], identity[:])
	return c
}

func NewClient(conn io.ReadWriteCloser, myIdentity, myIdentityPublic, serverPublic *[32]byte) *Conn {
	c := &Conn{
		conn: conn,
	}
	copy(c.identity[:], myIdentity[:])
	copy(c.identityPublic[:], myIdentityPublic[:])
	copy(c.Peer[:], serverPublic[:])
	return c
}

func incSequence(seq *[24]byte) {
	n := uint32(1)

	for i := 0; i < 8; i++ {
		n += uint32(seq[i])
		seq[i] = byte(n)
		n >>= 8
	}
}

type deadlineable interface {
	SetDeadline(time.Time)
}

func (c *Conn) SetDeadline(t time.Time) {
	if d, ok := c.conn.(deadlineable); ok {
		d.SetDeadline(t)
	}
}

func (c *Conn) Read(out []byte) (n int, err error) {
	if len(c.readPending) > 0 {
		n = copy(out, c.readPending)
		c.readPending = c.readPending[n:]
		return
	}

	if c.readBuffer == nil {
		c.readBuffer = make([]byte, blockSize+2)
	}

	if _, err := io.ReadFull(c.conn, c.readBuffer[:2]); err != nil {
		return 0, err
	}
	n = int(c.readBuffer[0]) | int(c.readBuffer[1])<<8
	if n > len(c.readBuffer) {
		return 0, errors.New("transport: peer's message too large for Read")
	}
	if _, err := io.ReadFull(c.conn, c.readBuffer[:n]); err != nil {
		return 0, err
	}

	var ok bool
	if len(out) >= n-secretbox.Overhead {
		// We can decrypt directly into the output buffer.
		out, ok = secretbox.Open(out[:0], c.readBuffer[:n], &c.readSequence, &c.readKey)
		n = len(out)
	} else {
		// We need to decrypt into a side buffer and copy a prefix of
		// the result into the caller's buffer.
		c.decryptBuffer, ok = secretbox.Open(c.decryptBuffer[:0], c.readBuffer[:n], &c.readSequence, &c.readKey)
		n = copy(out, c.decryptBuffer)
		c.readPending = c.decryptBuffer[n:]
	}
	incSequence(&c.readSequence)
	if !ok {
		c.readPending = c.readPending[:0]
		return 0, errors.New("transport: bad MAC")
	}

	return
}

func (c *Conn) Write(buf []byte) (n int, err error) {
	if c.writeBuffer == nil {
		c.writeBuffer = make([]byte, blockSize+2)
	}

	for len(buf) > 0 {
		m := len(buf)
		if m > blockSize-secretbox.Overhead {
			m = blockSize - secretbox.Overhead
		}
		l := len(secretbox.Seal(c.writeBuffer[2:2], buf[:m], &c.writeSequence, &c.writeKey))
		c.writeBuffer[0] = byte(l)
		c.writeBuffer[1] = byte(l >> 8)
		if _, err = c.conn.Write(c.writeBuffer[:2+l]); err != nil {
			return n, err
		}
		n += m
		buf = buf[m:]
		incSequence(&c.writeSequence)
	}

	return
}

func (c *Conn) ReadProto(out proto.Message) error {
	buf := make([]byte, pond.TransportSize+2+secretbox.Overhead)
	n, err := c.read(buf)
	if err != nil {
		return err
	}
	if n != pond.TransportSize+2 {
		return errors.New("transport: message wrong length")
	}

	n = int(buf[0]) | int(buf[1])<<8
	buf = buf[2:]
	if n > len(buf) {
		return errors.New("transport: corrupt message")
	}
	return proto.Unmarshal(buf[:n], out)
}

func (c *Conn) WriteProto(msg proto.Message) error {
	data, err := proto.Marshal(msg)
	if err != nil {
		return err
	}
	if len(data) > pond.TransportSize {
		return errors.New("transport: message too large")
	}

	buf := make([]byte, pond.TransportSize+2)
	buf[0] = byte(len(data))
	buf[1] = byte(len(data) >> 8)
	copy(buf[2:], data)
	_, err = c.write(buf)
	return err
}

func (c *Conn) Close() (err error) {
	if !c.isServer {
		_, err = c.write(nil)
	}

	if closeErr := c.conn.Close(); err == nil {
		err = closeErr
	}

	return
}

func (c *Conn) WaitForClose() error {
	if !c.isServer {
		panic("non-server waited for connection close")
	}
	n, err := c.read(make([]byte, 128))
	if err != nil {
		return err
	}
	if n != 0 {
		return errors.New("transport: non-close message received when expecting close")
	}

	return nil
}

func (c *Conn) read(data []byte) (n int, err error) {
	var lengthBytes [2]byte

	if _, err := io.ReadFull(c.conn, lengthBytes[:]); err != nil {
		return 0, err
	}

	theirLength := int(lengthBytes[0]) + int(lengthBytes[1])<<8
	if theirLength > len(data) {
		return 0, errors.New("tranport: given buffer too small (" + strconv.Itoa(len(data)) + " vs " + strconv.Itoa(theirLength) + ")")
	}

	data = data[:theirLength]
	if _, err := io.ReadFull(c.conn, data); err != nil {
		return 0, err
	}

	decrypted, err := c.decrypt(data)
	if err != nil {
		return 0, err
	}
	copy(data, decrypted)

	return len(decrypted), nil
}

func (c *Conn) write(data []byte) (n int, err error) {
	encrypted := c.encrypt(data)

	var lengthBytes [2]byte
	lengthBytes[0] = byte(len(encrypted))
	lengthBytes[1] = byte(len(encrypted) >> 8)

	if _, err := c.conn.Write(lengthBytes[:]); err != nil {
		return 0, err
	}
	if _, err := c.conn.Write(encrypted); err != nil {
		return 0, err
	}

	return len(data), nil
}

func (c *Conn) encrypt(data []byte) []byte {
	if !c.writeKeyValid {
		return data
	}

	encrypted := secretbox.Seal(nil, data, &c.writeSequence, &c.writeKey)
	incSequence(&c.writeSequence)
	return encrypted
}

func (c *Conn) decrypt(data []byte) ([]byte, error) {
	if !c.readKeyValid {
		return data, nil
	}

	decrypted, ok := secretbox.Open(nil, data, &c.readSequence, &c.readKey)
	incSequence(&c.readSequence)
	if !ok {
		return nil, errors.New("transport: bad MAC")
	}
	return decrypted, nil
}

var serverKeysMagic = []byte("server keys\x00")
var clientKeysMagic = []byte("client keys\x00")

func (c *Conn) setupKeys(ephemeralShared *[32]byte) {
	var writeMagic, readMagic []byte
	if c.isServer {
		writeMagic, readMagic = serverKeysMagic, clientKeysMagic
	} else {
		writeMagic, readMagic = clientKeysMagic, serverKeysMagic
	}

	h := sha256.New()
	h.Write(writeMagic)
	h.Write(ephemeralShared[:])
	h.Sum(c.writeKey[:0])
	c.writeKeyValid = true

	h.Reset()
	h.Write(readMagic)
	h.Write(ephemeralShared[:])
	h.Sum(c.readKey[:0])
	c.readKeyValid = true
}

var serverProofMagic = []byte("server proof\x00")
var clientProofMagic = []byte("client proof\x00")

var shortMessageError = errors.New("transport: received short handshake message")

func (c *Conn) Handshake() error {
	var ephemeralPrivate, ephemeralPublic, ephemeralShared [32]byte
	if _, err := io.ReadFull(rand.Reader, ephemeralPrivate[:]); err != nil {
		return err
	}
	curve25519.ScalarBaseMult(&ephemeralPublic, &ephemeralPrivate)

	if _, err := c.write(ephemeralPublic[:]); err != nil {
		return err
	}

	var theirEphemeralPublic [32]byte
	if n, err := c.read(theirEphemeralPublic[:]); err != nil || n != len(theirEphemeralPublic) {
		if err == nil {
			err = shortMessageError
		}
		return err
	}

	handshakeHash := sha256.New()
	if c.isServer {
		handshakeHash.Write(theirEphemeralPublic[:])
		handshakeHash.Write(ephemeralPublic[:])
	} else {
		handshakeHash.Write(ephemeralPublic[:])
		handshakeHash.Write(theirEphemeralPublic[:])
	}

	curve25519.ScalarMult(&ephemeralShared, &ephemeralPrivate, &theirEphemeralPublic)
	c.setupKeys(&ephemeralShared)

	if c.isServer {
		return c.handshakeServer(handshakeHash, &theirEphemeralPublic)
	}
	return c.handshakeClient(handshakeHash, &ephemeralPrivate)
}

func (c *Conn) handshakeClient(handshakeHash hash.Hash, ephemeralPrivate *[32]byte) error {
	var ephemeralIdentityShared [32]byte
	curve25519.ScalarMult(&ephemeralIdentityShared, ephemeralPrivate, &c.Peer)

	digest := handshakeHash.Sum(nil)
	h := hmac.New(sha256.New, ephemeralIdentityShared[:])
	h.Write(serverProofMagic)
	h.Write(digest)
	digest = h.Sum(digest[:0])

	digestReceived := make([]byte, len(digest)+secretbox.Overhead)
	n, err := c.read(digestReceived)
	if err != nil {
		return err
	}
	if n != len(digest) {
		return shortMessageError
	}
	digestReceived = digestReceived[:n]

	if subtle.ConstantTimeCompare(digest, digestReceived) != 1 {
		return errors.New("transport: server identity incorrect")
	}

	var identityShared [32]byte
	curve25519.ScalarMult(&identityShared, &c.identity, &c.Peer)

	handshakeHash.Write(digest)
	digest = handshakeHash.Sum(digest[:0])

	h = hmac.New(sha256.New, identityShared[:])
	h.Write(clientProofMagic)
	h.Write(digest)

	finalMessage := make([]byte, 32+sha256.Size)
	copy(finalMessage, c.identityPublic[:])
	h.Sum(finalMessage[32:32])

	if _, err := c.write(finalMessage); err != nil {
		return err
	}

	return nil
}

func (c *Conn) handshakeServer(handshakeHash hash.Hash, theirEphemeralPublic *[32]byte) error {
	var ephemeralIdentityShared [32]byte
	curve25519.ScalarMult(&ephemeralIdentityShared, &c.identity, theirEphemeralPublic)

	digest := handshakeHash.Sum(nil)
	h := hmac.New(sha256.New, ephemeralIdentityShared[:])
	h.Write(serverProofMagic)
	h.Write(digest)
	digest = h.Sum(digest[:0])

	if _, err := c.write(digest); err != nil {
		return err
	}

	handshakeHash.Write(digest)
	digest = handshakeHash.Sum(digest[:0])

	finalMessage := make([]byte, 32+sha256.Size+secretbox.Overhead)
	n, err := c.read(finalMessage)
	if err != nil {
		return err
	}
	if n != 32+sha256.Size {
		return shortMessageError
	}
	finalMessage = finalMessage[:n]

	copy(c.Peer[:], finalMessage[:32])
	var identityShared [32]byte
	curve25519.ScalarMult(&identityShared, &c.identity, &c.Peer)

	h = hmac.New(sha256.New, identityShared[:])
	h.Write(clientProofMagic)
	h.Write(digest)
	digest = h.Sum(digest[:0])

	if subtle.ConstantTimeCompare(digest, finalMessage[32:]) != 1 {
		return errors.New("transport: bad proof from client")
	}

	return nil
}
<commit_msg>Updated magic values<commit_after>package transport

import (
	"crypto/hmac"
	"crypto/rand"
	"crypto/sha256"
	"crypto/subtle"
	"errors"
	"hash"
	"io"
	"strconv"
	"time"

	pond "github.com/unixninja92/Div-III-Server/protos"
	"github.com/golang/protobuf/proto"
	"golang.org/x/crypto/curve25519"
	"golang.org/x/crypto/nacl/secretbox"
)

// blockSize is the size of the blocks of data that we'll send and receive when
// working in streaming mode. Each block is prefixed by two length bytes (which
// aren't counted in blockSize) and includes secretbox.Overhead bytes of MAC
// tag (which are).
const blockSize = 4096 - 2

type Conn struct {
	conn                     io.ReadWriteCloser
	isServer                 bool
	identity, identityPublic [32]byte
	Peer                     [32]byte

	writeKey, readKey           [32]byte
	writeKeyValid, readKeyValid bool
	writeSequence, readSequence [24]byte

	// readBuffer is used to receive bytes from the network when this Conn
	// is used to stream data.
	readBuffer []byte
	// decryptBuffer is used to store decrypted payloads when this Conn is
	// used to stream data and the caller's buffer isn't large enough to
	// decrypt into directly.
	decryptBuffer []byte
	// readPending aliases into decryptBuffer when a partial decryption had
	// to be returned to a caller because of buffer size limitations.
	readPending []byte

	// writeBuffer is used to hold encrypted payloads when this Conn is
	// used for streaming data.
	writeBuffer []byte
}

func NewServer(conn io.ReadWriteCloser, identity *[32]byte) *Conn {
	c := &Conn{
		conn:     conn,
		isServer: true,
	}
	copy(c.identity[:], identity[:])
	return c
}

func NewClient(conn io.ReadWriteCloser, myIdentity, myIdentityPublic, serverPublic *[32]byte) *Conn {
	c := &Conn{
		conn: conn,
	}
	copy(c.identity[:], myIdentity[:])
	copy(c.identityPublic[:], myIdentityPublic[:])
	copy(c.Peer[:], serverPublic[:])
	return c
}

func incSequence(seq *[24]byte) {
	n := uint32(1)

	for i := 0; i < 8; i++ {
		n += uint32(seq[i])
		seq[i] = byte(n)
		n >>= 8
	}
}

type deadlineable interface {
	SetDeadline(time.Time)
}

func (c *Conn) SetDeadline(t time.Time) {
	if d, ok := c.conn.(deadlineable); ok {
		d.SetDeadline(t)
	}
}

func (c *Conn) Read(out []byte) (n int, err error) {
	if len(c.readPending) > 0 {
		n = copy(out, c.readPending)
		c.readPending = c.readPending[n:]
		return
	}

	if c.readBuffer == nil {
		c.readBuffer = make([]byte, blockSize+2)
	}

	if _, err := io.ReadFull(c.conn, c.readBuffer[:2]); err != nil {
		return 0, err
	}
	n = int(c.readBuffer[0]) | int(c.readBuffer[1])<<8
	if n > len(c.readBuffer) {
		return 0, errors.New("transport: peer's message too large for Read")
	}
	if _, err := io.ReadFull(c.conn, c.readBuffer[:n]); err != nil {
		return 0, err
	}

	var ok bool
	if len(out) >= n-secretbox.Overhead {
		// We can decrypt directly into the output buffer.
		out, ok = secretbox.Open(out[:0], c.readBuffer[:n], &c.readSequence, &c.readKey)
		n = len(out)
	} else {
		// We need to decrypt into a side buffer and copy a prefix of
		// the result into the caller's buffer.
		c.decryptBuffer, ok = secretbox.Open(c.decryptBuffer[:0], c.readBuffer[:n], &c.readSequence, &c.readKey)
		n = copy(out, c.decryptBuffer)
		c.readPending = c.decryptBuffer[n:]
	}
	incSequence(&c.readSequence)
	if !ok {
		c.readPending = c.readPending[:0]
		return 0, errors.New("transport: bad MAC")
	}

	return
}

func (c *Conn) Write(buf []byte) (n int, err error) {
	if c.writeBuffer == nil {
		c.writeBuffer = make([]byte, blockSize+2)
	}

	for len(buf) > 0 {
		m := len(buf)
		if m > blockSize-secretbox.Overhead {
			m = blockSize - secretbox.Overhead
		}
		l := len(secretbox.Seal(c.writeBuffer[2:2], buf[:m], &c.writeSequence, &c.writeKey))
		c.writeBuffer[0] = byte(l)
		c.writeBuffer[1] = byte(l >> 8)
		if _, err = c.conn.Write(c.writeBuffer[:2+l]); err != nil {
			return n, err
		}
		n += m
		buf = buf[m:]
		incSequence(&c.writeSequence)
	}

	return
}

func (c *Conn) ReadProto(out proto.Message) error {
	buf := make([]byte, pond.TransportSize+2+secretbox.Overhead)
	n, err := c.read(buf)
	if err != nil {
		return err
	}
	if n != pond.TransportSize+2 {
		return errors.New("transport: message wrong length")
	}

	n = int(buf[0]) | int(buf[1])<<8
	buf = buf[2:]
	if n > len(buf) {
		return errors.New("transport: corrupt message")
	}
	return proto.Unmarshal(buf[:n], out)
}

func (c *Conn) WriteProto(msg proto.Message) error {
	data, err := proto.Marshal(msg)
	if err != nil {
		return err
	}
	if len(data) > pond.TransportSize {
		return errors.New("transport: message too large")
	}

	buf := make([]byte, pond.TransportSize+2)
	buf[0] = byte(len(data))
	buf[1] = byte(len(data) >> 8)
	copy(buf[2:], data)
	_, err = c.write(buf)
	return err
}

func (c *Conn) Close() (err error) {
	if !c.isServer {
		_, err = c.write(nil)
	}

	if closeErr := c.conn.Close(); err == nil {
		err = closeErr
	}

	return
}

func (c *Conn) WaitForClose() error {
	if !c.isServer {
		panic("non-server waited for connection close")
	}
	n, err := c.read(make([]byte, 128))
	if err != nil {
		return err
	}
	if n != 0 {
		return errors.New("transport: non-close message received when expecting close")
	}

	return nil
}

func (c *Conn) read(data []byte) (n int, err error) {
	var lengthBytes [2]byte

	if _, err := io.ReadFull(c.conn, lengthBytes[:]); err != nil {
		return 0, err
	}

	theirLength := int(lengthBytes[0]) + int(lengthBytes[1])<<8
	if theirLength > len(data) {
		return 0, errors.New("tranport: given buffer too small (" + strconv.Itoa(len(data)) + " vs " + strconv.Itoa(theirLength) + ")")
	}

	data = data[:theirLength]
	if _, err := io.ReadFull(c.conn, data); err != nil {
		return 0, err
	}

	decrypted, err := c.decrypt(data)
	if err != nil {
		return 0, err
	}
	copy(data, decrypted)

	return len(decrypted), nil
}

func (c *Conn) write(data []byte) (n int, err error) {
	encrypted := c.encrypt(data)

	var lengthBytes [2]byte
	lengthBytes[0] = byte(len(encrypted))
	lengthBytes[1] = byte(len(encrypted) >> 8)

	if _, err := c.conn.Write(lengthBytes[:]); err != nil {
		return 0, err
	}
	if _, err := c.conn.Write(encrypted); err != nil {
		return 0, err
	}

	return len(data), nil
}

func (c *Conn) encrypt(data []byte) []byte {
	if !c.writeKeyValid {
		return data
	}

	encrypted := secretbox.Seal(nil, data, &c.writeSequence, &c.writeKey)
	incSequence(&c.writeSequence)
	return encrypted
}

func (c *Conn) decrypt(data []byte) ([]byte, error) {
	if !c.readKeyValid {
		return data, nil
	}

	decrypted, ok := secretbox.Open(nil, data, &c.readSequence, &c.readKey)
	incSequence(&c.readSequence)
	if !ok {
		return nil, errors.New("transport: bad MAC")
	}
	return decrypted, nil
}

var serverKeysMagic = []byte("server keys snap")
var clientKeysMagic = []byte("client keys snap")

func (c *Conn) setupKeys(ephemeralShared *[32]byte) {
	var writeMagic, readMagic []byte
	if c.isServer {
		writeMagic, readMagic = serverKeysMagic, clientKeysMagic
	} else {
		writeMagic, readMagic = clientKeysMagic, serverKeysMagic
	}

	h := sha256.New()
	h.Write(writeMagic)
	h.Write(ephemeralShared[:])
	h.Sum(c.writeKey[:0])
	c.writeKeyValid = true

	h.Reset()
	h.Write(readMagic)
	h.Write(ephemeralShared[:])
	h.Sum(c.readKey[:0])
	c.readKeyValid = true
}

var serverProofMagic = []byte("server proof snap")
var clientProofMagic = []byte("client proof snap")

var shortMessageError = errors.New("transport: received short handshake message")

func (c *Conn) Handshake() error {
	var ephemeralPrivate, ephemeralPublic, ephemeralShared [32]byte
	if _, err := io.ReadFull(rand.Reader, ephemeralPrivate[:]); err != nil {
		return err
	}
	curve25519.ScalarBaseMult(&ephemeralPublic, &ephemeralPrivate)

	if _, err := c.write(ephemeralPublic[:]); err != nil {
		return err
	}

	var theirEphemeralPublic [32]byte
	if n, err := c.read(theirEphemeralPublic[:]); err != nil || n != len(theirEphemeralPublic) {
		if err == nil {
			err = shortMessageError
		}
		return err
	}

	handshakeHash := sha256.New()
	if c.isServer {
		handshakeHash.Write(theirEphemeralPublic[:])
		handshakeHash.Write(ephemeralPublic[:])
	} else {
		handshakeHash.Write(ephemeralPublic[:])
		handshakeHash.Write(theirEphemeralPublic[:])
	}

	curve25519.ScalarMult(&ephemeralShared, &ephemeralPrivate, &theirEphemeralPublic)
	c.setupKeys(&ephemeralShared)

	if c.isServer {
		return c.handshakeServer(handshakeHash, &theirEphemeralPublic)
	}
	return c.handshakeClient(handshakeHash, &ephemeralPrivate)
}

func (c *Conn) handshakeClient(handshakeHash hash.Hash, ephemeralPrivate *[32]byte) error {
	var ephemeralIdentityShared [32]byte
	curve25519.ScalarMult(&ephemeralIdentityShared, ephemeralPrivate, &c.Peer)

	digest := handshakeHash.Sum(nil)
	h := hmac.New(sha256.New, ephemeralIdentityShared[:])
	h.Write(serverProofMagic)
	h.Write(digest)
	digest = h.Sum(digest[:0])

	digestReceived := make([]byte, len(digest)+secretbox.Overhead)
	n, err := c.read(digestReceived)
	if err != nil {
		return err
	}
	if n != len(digest) {
		return shortMessageError
	}
	digestReceived = digestReceived[:n]

	if subtle.ConstantTimeCompare(digest, digestReceived) != 1 {
		return errors.New("transport: server identity incorrect")
	}

	var identityShared [32]byte
	curve25519.ScalarMult(&identityShared, &c.identity, &c.Peer)

	handshakeHash.Write(digest)
	digest = handshakeHash.Sum(digest[:0])

	h = hmac.New(sha256.New, identityShared[:])
	h.Write(clientProofMagic)
	h.Write(digest)

	finalMessage := make([]byte, 32+sha256.Size)
	copy(finalMessage, c.identityPublic[:])
	h.Sum(finalMessage[32:32])

	if _, err := c.write(finalMessage); err != nil {
		return err
	}

	return nil
}

func (c *Conn) handshakeServer(handshakeHash hash.Hash, theirEphemeralPublic *[32]byte) error {
	var ephemeralIdentityShared [32]byte
	curve25519.ScalarMult(&ephemeralIdentityShared, &c.identity, theirEphemeralPublic)

	digest := handshakeHash.Sum(nil)
	h := hmac.New(sha256.New, ephemeralIdentityShared[:])
	h.Write(serverProofMagic)
	h.Write(digest)
	digest = h.Sum(digest[:0])

	if _, err := c.write(digest); err != nil {
		return err
	}

	handshakeHash.Write(digest)
	digest = handshakeHash.Sum(digest[:0])

	finalMessage := make([]byte, 32+sha256.Size+secretbox.Overhead)
	n, err := c.read(finalMessage)
	if err != nil {
		return err
	}
	if n != 32+sha256.Size {
		return shortMessageError
	}
	finalMessage = finalMessage[:n]

	copy(c.Peer[:], finalMessage[:32])
	var identityShared [32]byte
	curve25519.ScalarMult(&identityShared, &c.identity, &c.Peer)

	h = hmac.New(sha256.New, identityShared[:])
	h.Write(clientProofMagic)
	h.Write(digest)
	digest = h.Sum(digest[:0])

	if subtle.ConstantTimeCompare(digest, finalMessage[32:]) != 1 {
		return errors.New("transport: bad proof from client")
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Frank Braun <frank@cryptogroup.net>
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

// Package convert implements converters for two-dimensional hierarchical
// ASCII art.
package convert

import (
	"fmt"
	"io"
	"io/ioutil"

	"github.com/frankbraun/asciiart/aa2d"
	"github.com/frankbraun/asciiart/svg"
)

// ASCIIArt2SVG converts two-dimensional ASCII art read from r to a SVG
// written to w.
// xScale denotes the number of pixels to scale each unit on the x-axis to.
// yScale denotes the number of pixels to scale each unit on the y-axis to.
func ASCIIArt2SVG(w io.Writer, r io.Reader, xScale, yScale float64) error {
	aa, err := ioutil.ReadAll(r)
	if err != nil {
		return err
	}
	p := aa2d.NewParser()
	if err := p.SetScale(xScale, yScale); err != nil {
		return err
	}
	grid, err := p.Parse(string(aa))
	if err != nil {
		return err
	}
	if err := svg.Generate(w, grid); err != nil {
		return err
	}
	return nil
}

// ASCIIArt2Txt converts two-dimensional ASCII art read from r to text
// written to w.
func ASCIIArt2Txt(w io.Writer, r io.Reader) error {
	aa, err := ioutil.ReadAll(r)
	if err != nil {
		return err
	}
	err = generateTxt(w, string(aa))
	if err != nil {
		return err
	}
	return nil
}

func generateTxt(w io.Writer, aa string) error {
	p := aa2d.NewParser()
	grid, err := p.Parse(aa)
	if err != nil {
		return err
	}
	var traverse func(w io.Writer, elems []interface{}, indent string)
	traverse = func(w io.Writer, elems []interface{}, indent string) {
		for _, elem := range elems {
			switch t := elem.(type) {
			case *aa2d.Rectangle:
				fmt.Fprintf(w, "%s[rect x=%.2f y=%.2f w=%.2f h=%.2f]\n",
					indent, t.X, t.Y, t.W, t.H)
				traverse(w, t.Elems, indent+"  ")
			case *aa2d.Line:
				fmt.Fprintf(w, "%s[line x1=%.2f y1=%.2f x2=%.2f y2=%.2f]\n",
					indent, t.X1, t.Y1, t.X2, t.Y2)
			case *aa2d.Polyline:
				fmt.Fprintf(w, "%s[polyline", indent)
				for i := 0; i < len(t.X); i++ {
					fmt.Fprintf(w, " x%d=%.2f y%d=%.2f",
						i+1, t.X[i], i+1, t.Y[i])
				}
				fmt.Fprintf(w, "]\n")
			case *aa2d.Polygon:
				fmt.Fprintf(w, "%s[polygon", indent)
				for i := 0; i < len(t.X); i++ {
					fmt.Fprintf(w, " x%d=%.2f y%d=%.2f",
						i+1, t.X[i], i+1, t.Y[i])
				}
				traverse(w, t.Elems, indent+"  ")
			case *aa2d.Textline:
				fmt.Fprintf(w, "%s[text x=%.2f y=%.2f t=%q]",
					indent, t.X, t.Y, t.Text)
			}
		}
	}
	traverse(w, grid.Elems, "")
	return nil
}
<commit_msg>simplify aa2txt<commit_after>// Copyright (c) 2016 Frank Braun <frank@cryptogroup.net>
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

// Package convert implements converters for two-dimensional hierarchical
// ASCII art.
package convert

import (
	"fmt"
	"io"
	"io/ioutil"
	"strconv"
	"strings"

	"github.com/frankbraun/asciiart/aa2d"
	"github.com/frankbraun/asciiart/svg"
)

// ASCIIArt2SVG converts two-dimensional ASCII art read from r to a SVG
// written to w.
// xScale denotes the number of pixels to scale each unit on the x-axis to.
// yScale denotes the number of pixels to scale each unit on the y-axis to.
func ASCIIArt2SVG(w io.Writer, r io.Reader, xScale, yScale float64) error {
	aa, err := ioutil.ReadAll(r)
	if err != nil {
		return err
	}
	p := aa2d.NewParser()
	if err := p.SetScale(xScale, yScale); err != nil {
		return err
	}
	grid, err := p.Parse(string(aa))
	if err != nil {
		return err
	}
	if err := svg.Generate(w, grid); err != nil {
		return err
	}
	return nil
}

// ASCIIArt2Txt converts two-dimensional ASCII art read from r to text
// written to w.
func ASCIIArt2Txt(w io.Writer, r io.Reader) error {
	aa, err := ioutil.ReadAll(r)
	if err != nil {
		return err
	}
	err = generateTxt(w, string(aa))
	if err != nil {
		return err
	}
	return nil
}

func generateTxt(w io.Writer, aa string) error {
	p := aa2d.NewParser()
	p.SetScale(1, 1)
	grid, err := p.Parse(aa)
	if err != nil {
		return err
	}
	fmtFlt := func(f float64) string { return strconv.FormatFloat(f, 'f', -1, 64) }
	var traverse func(w io.Writer, elems []interface{}, indent string)
	traverse = func(w io.Writer, elems []interface{}, indent string) {
		for _, elem := range elems {
			switch t := elem.(type) {
			case *aa2d.Rectangle:
				fmt.Fprintln(w, indent, "rect", t.X, t.Y, t.W, t.H)
				traverse(w, t.Elems, indent+"  ") // recursion
			case *aa2d.Line:
				fmt.Fprintln(w, indent, "line", t.X1, t.Y1, t.X2, t.Y2)
			case *aa2d.Polyline:
				var p []string
				for i := 0; i < len(t.X); i++ {
					p = append(p, fmtFlt(t.X[i]), fmtFlt(t.Y[i]))
				}
				fmt.Fprintln(w, indent, "polyline", strings.Join(p, " "))
			case *aa2d.Polygon:
				var p []string
				for i := 0; i < len(t.X); i++ {
					p = append(p, fmtFlt(t.X[i]), fmtFlt(t.Y[i]))
				}
				fmt.Fprintln(w, indent, "polygon", strings.Join(p, " "))
				traverse(w, t.Elems, indent+"  ") // recursion
			case *aa2d.Textline:
				fmt.Fprintln(w, indent, "textline", t.X, t.Y, t.Text)
			}
		}
	}
	fmt.Fprintln(w, "grid", grid.W, grid.H)
	traverse(w, grid.Elems, " ")
	return nil
}
<|endoftext|>


<commit_before>package convert

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"strings"
	"time"

	"gopkg.in/yaml.v2"

	"github.com/PuerkitoBio/goquery"
	"github.com/dreampuf/evernote-sdk-golang/types"
	"github.com/pkg/errors"
	"github.com/yosssi/gohtml"
)

type frontMatter struct {
	Title     string   `yaml:"title,omitempty"`
	Layout    string   `yaml:"layout,omitempty"`
	Published bool     `yaml:"published"`
	Date      string   `yaml:"date,omitempty"`
	Tags      []string `yaml:"tags,omitempty"`
}

const cacheExtension = ".yml"

// Convert local cache to static files
func Convert(cacheRoot string, noteCacheDirName string, resourceCacheDirName string, jekyllRoot string, postsDirName string, resourcesDirName string, cleanNeeded bool) error {
	jekyllPostsDir := path.Join(jekyllRoot, postsDirName)
	jekyllResourcesDir := path.Join(jekyllRoot, resourcesDirName)
	noteCacheDir := path.Join(cacheRoot, noteCacheDirName)
	resourceCacheDir := path.Join(cacheRoot, resourceCacheDirName)

	notefiles, err := ioutil.ReadDir(noteCacheDir)
	if err != nil {
		return errors.Wrapf(err, "can't get cached notes", noteCacheDir)
	}

	resourceFiles, err := ioutil.ReadDir(resourceCacheDir)
	if err != nil {
		return errors.Wrapf(err, "can't get cached resources %v", resourceCacheDir)
	}

	createDestinations(cleanNeeded, &jekyllPostsDir, &jekyllResourcesDir)

	for _, notefile := range notefiles {
		cachedNote := &types.Note{}
		cachedNotePath := path.Join(noteCacheDir, notefile.Name())
		yamlBytes, err := ioutil.ReadFile(cachedNotePath)
		if err != nil {
			return errors.Wrapf(err, "can't read cached note file %v", cachedNotePath)
		}
		if err := yaml.Unmarshal(yamlBytes, cachedNote); err != nil {
			return errors.Wrapf(err, "can't unmarshal cached note file %v", cachedNotePath)
		}

		created := time.Unix(int64(*cachedNote.Created)/1000, 0)
		created = created.In(time.Local)

		html, err := replaceEvernoteTags(cachedNote.Content, &resourceFiles, &resourcesDirName)
		if err != nil {
			return errors.Wrapf(err, "can't replace evernote tags %v", cachedNotePath)
		}
		*html = strings.Replace(*html, "\u00a0", "\x20", -1)
		*html = gohtml.Format(*html)

		fm := frontMatter{
			Title:     *cachedNote.Title,
			Layout:    "post",
			Published: false,
			Date:      created.Format("2006-01-02 15:04:05 -0700"),
			Tags:      cachedNote.TagNames,
		}

		for i, tag := range fm.Tags {
			if tag == "published" {
				fm.Published = true
				fm.Tags = append(fm.Tags[:i], fm.Tags[i+1:]...)
				break
			}
		}

		for i, tag := range fm.Tags {
			if tag == "page" {
				fm.Layout = "page"
				fm.Tags = append(fm.Tags[:i], fm.Tags[i+1:]...)
				break
			}
		}

		fmyaml, err := yaml.Marshal(fm)
		if err != nil {
			return errors.Wrap(err, "can't create front matter")
		}

		*html = "---\n" + string(fmyaml) + "---\n" + *html

		var noteFileName string
		if cachedNote.Attributes.SourceURL != nil && *cachedNote.Attributes.SourceURL != "" {
			noteFileName = *cachedNote.Attributes.SourceURL
		} else {
			// TODO: need to sanitize title
			noteFileName = *cachedNote.Title
		}

		var notePath string
		if fm.Layout == "page" {
			notePath = path.Join(jekyllRoot, noteFileName+".html")
		} else {
			notePath = path.Join(jekyllPostsDir, created.Format("2006-01-02")+"-"+noteFileName+".html")
		}

		if err := ioutil.WriteFile(notePath, []byte(*html), os.ModePerm); err != nil {
			return errors.Wrapf(err, "can't create note file %v", notePath)
		}

		for _, resourceFile := range resourceFiles {
			copyResourceFile(resourceCacheDir, jekyllResourcesDir, resourceFile.Name())
		}
	}

	return nil
}

func createDestinations(needClean bool, jekyllPostsDir *string, jekyllResourcesDir *string) {
	if needClean {
		os.RemoveAll(*jekyllPostsDir)
		os.RemoveAll(*jekyllResourcesDir)
	}

	os.MkdirAll(*jekyllPostsDir, os.ModePerm)
	os.MkdirAll(*jekyllResourcesDir, os.ModePerm)
}

func replaceEvernoteTags(enml *string, resourceFiles *[]os.FileInfo, jekyllResourcesDirName *string) (*string, error) {
	// FIXME
	// standard library's html parser can't handle unknown self closing tags
	// https://github.com/golang/net/blob/master/html/parse.go#L727-L980
	// so replace en-medias to imgs to parse them
	*enml = strings.Replace(*enml, "<en-media", `<img en-media="true"`, -1)

	reader := bytes.NewReader([]byte(*enml))
	doc, _ := goquery.NewDocumentFromReader(reader)

	doc.Find("en-todo").Each(func(_ int, todo *goquery.Selection) {
		if _, exists := todo.Attr("checked"); exists {
			todo.ReplaceWithHtml(`<input type="checkbox" checked="checked"/>`)
		} else {
			todo.ReplaceWithHtml(`<input type="checkbox"/>`)
		}
	})

	for {
		codeOpen := doc.Find("div:contains(\\`\\`\\`)").First()
		if len(codeOpen.Nodes) == 0 {
			break
		}
		codeClose := codeOpen.NextAllFiltered("div:contains(\\`\\`\\`)").First()

		if len(codeClose.Nodes) == 0 {
			return nil, errors.Errorf("can't find code block end")
		}

		language := strings.Replace(codeOpen.Text(), "```", "", 1)
		codeLines := make([]string, 0, 10)
		codeLines = append(codeLines, `<div>{% highlight `+language+` %}`)

		codeOpen.NextUntilSelection(codeClose).Each(func(_ int, line *goquery.Selection) {
			codeLines = append(codeLines, line.Text())
			line.Remove()
		})

		codeLines = append(codeLines, `{% endhighlight %}</div>`)
		codeOpen.ReplaceWithHtml(strings.Join(codeLines, "\n"))
		codeClose.Remove()
	}

	doc.Find(`div:contains("#")`).Each(func(_ int, div *goquery.Selection) {
		line := div.Text()
		if strings.HasPrefix(line, "##### ") {
			div.ReplaceWithHtml("<h5>" + strings.Replace(line, "##### ", "", 1))
		} else if strings.HasPrefix(line, "#### ") {
			div.ReplaceWithHtml("<h4>" + strings.Replace(line, "#### ", "", 1))
		} else if strings.HasPrefix(line, "### ") {
			div.ReplaceWithHtml("<h3>" + strings.Replace(line, "### ", "", 1))
		} else if strings.HasPrefix(line, "## ") {
			div.ReplaceWithHtml("<h2>" + strings.Replace(line, "## ", "", 1))
		} else if strings.HasPrefix(line, "# ") {
			div.ReplaceWithHtml("<h1>" + strings.Replace(line, "# ", "", 1))
		}
	})

	doc.Find("img[en-media]").Each(func(i int, selection *goquery.Selection) {
		hash, _ := selection.Attr("hash")
		found := false
		for _, resourceFile := range *resourceFiles {
			if !strings.HasPrefix(resourceFile.Name(), hash) {
				continue
			}

			found = true
			lowerName := strings.ToLower(resourceFile.Name())
			url := "{{ site.baseurl }}/" + path.Join(*jekyllResourcesDirName, resourceFile.Name())

			if strings.HasSuffix(lowerName, ".png") || strings.HasSuffix(lowerName, ".jpg") || strings.HasSuffix(lowerName, ".gif") {
				selection.ReplaceWithHtml(fmt.Sprintf(`<img src="%v" />`, url))
			} else if strings.HasSuffix(lowerName, ".mp3") {
				selection.ReplaceWithHtml(fmt.Sprintf(`<audio src="%v" controls="true"/>`, url))
			} else if strings.HasSuffix(lowerName, ".mp4") {
				selection.ReplaceWithHtml(fmt.Sprintf(`<video src="%v" />`, url))
			} else {
				selection.ReplaceWithHtml(fmt.Sprintf(`<a src="%v" />`, url))
			}
		}

		if !found {
			fmt.Printf("can't find resource %v\n", hash)
		}
	})

	innerNoteHTML, _ := doc.Find("en-note").Html()
	return &innerNoteHTML, nil
}

func copyResourceFile(from string, to string, fileName string) error {
	sourcePath := path.Join(from, fileName)
	source, err := os.OpenFile(sourcePath, os.O_RDONLY, os.ModePerm)
	if err != nil {
		return errors.Wrapf(err, "can't open resource cache file %v", sourcePath)
	}
	defer source.Close()

	destPath := path.Join(to, fileName)
	dest, err := os.OpenFile(destPath, os.O_CREATE, os.ModePerm)
	if err != nil {
		return errors.Wrapf(err, "can't create resource file %v", sourcePath)
	}
	defer dest.Close()

	if _, err := io.Copy(dest, source); err != nil {
		return errors.Wrapf(err, "can't copy resource file %v", sourcePath)
	}

	return nil
}
<commit_msg>Better handling of non-breaking spaces<commit_after>package convert

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"strings"
	"time"

	"gopkg.in/yaml.v2"

	"github.com/PuerkitoBio/goquery"
	"github.com/dreampuf/evernote-sdk-golang/types"
	"github.com/pkg/errors"
	"github.com/yosssi/gohtml"
)

type frontMatter struct {
	Title     string   `yaml:"title,omitempty"`
	Layout    string   `yaml:"layout,omitempty"`
	Published bool     `yaml:"published"`
	Date      string   `yaml:"date,omitempty"`
	Tags      []string `yaml:"tags,omitempty"`
}

const cacheExtension = ".yml"

// Convert local cache to static files
func Convert(cacheRoot string, noteCacheDirName string, resourceCacheDirName string, jekyllRoot string, postsDirName string, resourcesDirName string, cleanNeeded bool) error {
	jekyllPostsDir := path.Join(jekyllRoot, postsDirName)
	jekyllResourcesDir := path.Join(jekyllRoot, resourcesDirName)
	noteCacheDir := path.Join(cacheRoot, noteCacheDirName)
	resourceCacheDir := path.Join(cacheRoot, resourceCacheDirName)

	notefiles, err := ioutil.ReadDir(noteCacheDir)
	if err != nil {
		return errors.Wrapf(err, "can't get cached notes", noteCacheDir)
	}

	resourceFiles, err := ioutil.ReadDir(resourceCacheDir)
	if err != nil {
		return errors.Wrapf(err, "can't get cached resources %v", resourceCacheDir)
	}

	createDestinations(cleanNeeded, &jekyllPostsDir, &jekyllResourcesDir)

	for _, notefile := range notefiles {
		cachedNote := &types.Note{}
		cachedNotePath := path.Join(noteCacheDir, notefile.Name())
		yamlBytes, err := ioutil.ReadFile(cachedNotePath)
		if err != nil {
			return errors.Wrapf(err, "can't read cached note file %v", cachedNotePath)
		}
		if err := yaml.Unmarshal(yamlBytes, cachedNote); err != nil {
			return errors.Wrapf(err, "can't unmarshal cached note file %v", cachedNotePath)
		}

		created := time.Unix(int64(*cachedNote.Created)/1000, 0)
		created = created.In(time.Local)

		html, err := replaceEvernoteTags(cachedNote.Content, &resourceFiles, &resourcesDirName)
		if err != nil {
			return errors.Wrapf(err, "can't replace evernote tags %v", cachedNotePath)
		}
		*html = strings.Replace(*html, "\u00a0", " ", -1)
		*html = gohtml.Format(*html)

		fm := frontMatter{
			Title:     *cachedNote.Title,
			Layout:    "post",
			Published: false,
			Date:      created.Format("2006-01-02 15:04:05 -0700"),
			Tags:      cachedNote.TagNames,
		}

		for i, tag := range fm.Tags {
			if tag == "published" {
				fm.Published = true
				fm.Tags = append(fm.Tags[:i], fm.Tags[i+1:]...)
				break
			}
		}

		for i, tag := range fm.Tags {
			if tag == "page" {
				fm.Layout = "page"
				fm.Tags = append(fm.Tags[:i], fm.Tags[i+1:]...)
				break
			}
		}

		fmyaml, err := yaml.Marshal(fm)
		if err != nil {
			return errors.Wrap(err, "can't create front matter")
		}

		*html = "---\n" + string(fmyaml) + "---\n" + *html

		var noteFileName string
		if cachedNote.Attributes.SourceURL != nil && *cachedNote.Attributes.SourceURL != "" {
			noteFileName = *cachedNote.Attributes.SourceURL
		} else {
			// TODO: need to sanitize title
			noteFileName = *cachedNote.Title
		}

		var notePath string
		if fm.Layout == "page" {
			notePath = path.Join(jekyllRoot, noteFileName+".html")
		} else {
			notePath = path.Join(jekyllPostsDir, created.Format("2006-01-02")+"-"+noteFileName+".html")
		}

		if err := ioutil.WriteFile(notePath, []byte(*html), os.ModePerm); err != nil {
			return errors.Wrapf(err, "can't create note file %v", notePath)
		}

		for _, resourceFile := range resourceFiles {
			copyResourceFile(resourceCacheDir, jekyllResourcesDir, resourceFile.Name())
		}
	}

	return nil
}

func createDestinations(needClean bool, jekyllPostsDir *string, jekyllResourcesDir *string) {
	if needClean {
		os.RemoveAll(*jekyllPostsDir)
		os.RemoveAll(*jekyllResourcesDir)
	}

	os.MkdirAll(*jekyllPostsDir, os.ModePerm)
	os.MkdirAll(*jekyllResourcesDir, os.ModePerm)
}

func replaceEvernoteTags(enml *string, resourceFiles *[]os.FileInfo, jekyllResourcesDirName *string) (*string, error) {
	// FIXME
	// standard library's html parser can't handle unknown self closing tags
	// https://github.com/golang/net/blob/master/html/parse.go#L727-L980
	// so replace en-medias to imgs to parse them
	*enml = strings.Replace(*enml, "<en-media", `<img en-media="true"`, -1)

	reader := bytes.NewReader([]byte(*enml))
	doc, _ := goquery.NewDocumentFromReader(reader)

	doc.Find("en-todo").Each(func(_ int, todo *goquery.Selection) {
		if _, exists := todo.Attr("checked"); exists {
			todo.ReplaceWithHtml(`<input type="checkbox" checked="checked"/>`)
		} else {
			todo.ReplaceWithHtml(`<input type="checkbox"/>`)
		}
	})

	for {
		codeOpen := doc.Find("div:contains(\\`\\`\\`)").First()
		if len(codeOpen.Nodes) == 0 {
			break
		}
		codeClose := codeOpen.NextAllFiltered("div:contains(\\`\\`\\`)").First()

		if len(codeClose.Nodes) == 0 {
			return nil, errors.Errorf("can't find code block end")
		}

		language := strings.Replace(codeOpen.Text(), "```", "", 1)
		codeLines := make([]string, 0, 10)
		codeLines = append(codeLines, `<div>{% highlight `+language+` %}`)

		codeOpen.NextUntilSelection(codeClose).Each(func(_ int, line *goquery.Selection) {
			codeLines = append(codeLines, line.Text())
			line.Remove()
		})

		codeLines = append(codeLines, `{% endhighlight %}</div>`)
		codeOpen.ReplaceWithHtml(strings.Join(codeLines, "\n"))
		codeClose.Remove()
	}

	doc.Find(`div:contains("#")`).Each(func(_ int, div *goquery.Selection) {
		line := strings.Replace(div.Text(), "\u00a0", " ", -1)
		if strings.HasPrefix(line, "##### ") {
			div.ReplaceWithHtml("<h5>" + strings.Replace(line, "##### ", "", 1))
		} else if strings.HasPrefix(line, "#### ") {
			div.ReplaceWithHtml("<h4>" + strings.Replace(line, "#### ", "", 1))
		} else if strings.HasPrefix(line, "### ") {
			div.ReplaceWithHtml("<h3>" + strings.Replace(line, "### ", "", 1))
		} else if strings.HasPrefix(line, "## ") {
			div.ReplaceWithHtml("<h2>" + strings.Replace(line, "## ", "", 1))
		} else if strings.HasPrefix(line, "# ") {
			div.ReplaceWithHtml("<h1>" + strings.Replace(line, "# ", "", 1))
		}
	})

	doc.Find("img[en-media]").Each(func(i int, selection *goquery.Selection) {
		hash, _ := selection.Attr("hash")
		found := false
		for _, resourceFile := range *resourceFiles {
			if !strings.HasPrefix(resourceFile.Name(), hash) {
				continue
			}

			found = true
			lowerName := strings.ToLower(resourceFile.Name())
			url := "{{ site.baseurl }}/" + path.Join(*jekyllResourcesDirName, resourceFile.Name())

			if strings.HasSuffix(lowerName, ".png") || strings.HasSuffix(lowerName, ".jpg") || strings.HasSuffix(lowerName, ".gif") {
				selection.ReplaceWithHtml(fmt.Sprintf(`<img src="%v" />`, url))
			} else if strings.HasSuffix(lowerName, ".mp3") {
				selection.ReplaceWithHtml(fmt.Sprintf(`<audio src="%v" controls="true"/>`, url))
			} else if strings.HasSuffix(lowerName, ".mp4") {
				selection.ReplaceWithHtml(fmt.Sprintf(`<video src="%v" />`, url))
			} else {
				selection.ReplaceWithHtml(fmt.Sprintf(`<a src="%v" />`, url))
			}
		}

		if !found {
			fmt.Printf("can't find resource %v\n", hash)
		}
	})

	innerNoteHTML, _ := doc.Find("en-note").Html()
	return &innerNoteHTML, nil
}

func copyResourceFile(from string, to string, fileName string) error {
	sourcePath := path.Join(from, fileName)
	source, err := os.OpenFile(sourcePath, os.O_RDONLY, os.ModePerm)
	if err != nil {
		return errors.Wrapf(err, "can't open resource cache file %v", sourcePath)
	}
	defer source.Close()

	destPath := path.Join(to, fileName)
	dest, err := os.OpenFile(destPath, os.O_CREATE, os.ModePerm)
	if err != nil {
		return errors.Wrapf(err, "can't create resource file %v", sourcePath)
	}
	defer dest.Close()

	if _, err := io.Copy(dest, source); err != nil {
		return errors.Wrapf(err, "can't copy resource file %v", sourcePath)
	}

	return nil
}
<|endoftext|>


<commit_before>package triton

import (
	"fmt"
	"log"
	"sort"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/s3"
)

func listDatesFromRange(start, end time.Time) (dates []time.Time) {
	dates = make([]time.Time, 0, 2)
	current := start
	day, _ := time.ParseDuration("24h")

	if start.After(end) {
		panic("invalid date range")
	}

	dates = append(dates, current)
	for !current.Equal(end) {
		dates = append(dates, current)
		current = current.Add(day)
	}

	return
}

// Sortable list of store archives.
//
// Though archives, when they come out of S3 are lexigraphically sorted, we
// want to just be sure that we're really handling our dates and times
// correctly.
type StoreArchiveList []StoreArchive

func (l StoreArchiveList) Len() int {
	return len(l)
}

func (l StoreArchiveList) Swap(i, j int) {
	l[i], l[j] = l[j], l[i]
}

func (l StoreArchiveList) Less(i, j int) bool {
	if l[i].T != l[j].T {
		return l[i].T.Before(l[j].T)
	} else {
		return l[i].SortValue < l[j].SortValue
	}
}

func NewStoreReader(svc S3Service, bucketName, clientName, streamName string, startDate, endDate time.Time) (Reader, error) {
	allDates := listDatesFromRange(startDate, endDate)
	archives := make(StoreArchiveList, 0, len(allDates))

	for _, date := range allDates {
		dateStr := date.Format("20060102")
		prefix := fmt.Sprintf("%s/%s-", dateStr, streamName)
		if clientName != "" {
			prefix = fmt.Sprintf("%s%s-", prefix, clientName)
		}
		resp, err := svc.ListObjects(&s3.ListObjectsInput{
			Bucket: aws.String(bucketName),
			Prefix: aws.String(prefix),
		})

		if err != nil {
			return nil, err
		}

		for _, o := range resp.Contents {
			log.Println("Opening store archive", *o.Key)
			sa, err := NewStoreArchive(bucketName, *o.Key, svc)
			if err != nil {
				log.Println("Failed to parse contents", *o.Key, err)
				continue
			}

			archives = append(archives, sa)
		}

		// TODO: Would be nice to handle this, limit is 1000
		if *resp.IsTruncated {
			log.Println("WARNING: truncated s3 response")
		}
	}

	foundClientName := clientName
	for _, a := range archives {
		if foundClientName == "" {
			foundClientName = a.ClientName
		}

		if foundClientName != a.ClientName {
			return nil, fmt.Errorf("Multiple clients found: %s and %s", foundClientName, a.ClientName)
		}
	}

	sort.Sort(archives)

	// Convert to a list of Readers... feels like there should be a better way
	// here. Is this what generics are for? Or is there an interface for a list?
	readers := make([]Reader, 0, len(archives))
	for _, a := range archives {
		readers = append(readers, &a)
	}

	return NewSerialReader(readers), nil
}
<commit_msg>Fix date generator<commit_after>package triton

import (
	"fmt"
	"log"
	"sort"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/s3"
)

func listDatesFromRange(start, end time.Time) (dates []time.Time) {
	dates = make([]time.Time, 0, 2)
	current := start
	day, _ := time.ParseDuration("24h")

	if start.After(end) {
		panic("invalid date range")
	}

	dates = append(dates, current)
	for !current.Equal(end) {
		current = current.Add(day)
		dates = append(dates, current)
	}

	return
}

// Sortable list of store archives.
//
// Though archives, when they come out of S3 are lexigraphically sorted, we
// want to just be sure that we're really handling our dates and times
// correctly.
type StoreArchiveList []StoreArchive

func (l StoreArchiveList) Len() int {
	return len(l)
}

func (l StoreArchiveList) Swap(i, j int) {
	l[i], l[j] = l[j], l[i]
}

func (l StoreArchiveList) Less(i, j int) bool {
	if l[i].T != l[j].T {
		return l[i].T.Before(l[j].T)
	} else {
		return l[i].SortValue < l[j].SortValue
	}
}

func NewStoreReader(svc S3Service, bucketName, clientName, streamName string, startDate, endDate time.Time) (Reader, error) {
	allDates := listDatesFromRange(startDate, endDate)
	archives := make(StoreArchiveList, 0, len(allDates))

	for _, date := range allDates {
		dateStr := date.Format("20060102")
		prefix := fmt.Sprintf("%s/%s-", dateStr, streamName)
		if clientName != "" {
			prefix = fmt.Sprintf("%s%s-", prefix, clientName)
		}
		resp, err := svc.ListObjects(&s3.ListObjectsInput{
			Bucket: aws.String(bucketName),
			Prefix: aws.String(prefix),
		})

		if err != nil {
			return nil, err
		}

		for _, o := range resp.Contents {
			log.Println("Opening store archive", *o.Key)
			sa, err := NewStoreArchive(bucketName, *o.Key, svc)
			if err != nil {
				log.Println("Failed to parse contents", *o.Key, err)
				continue
			}

			archives = append(archives, sa)
		}

		// TODO: Would be nice to handle this, limit is 1000
		if *resp.IsTruncated {
			log.Println("WARNING: truncated s3 response")
		}
	}

	foundClientName := clientName
	for _, a := range archives {
		if foundClientName == "" {
			foundClientName = a.ClientName
		}

		if foundClientName != a.ClientName {
			return nil, fmt.Errorf("Multiple clients found: %s and %s", foundClientName, a.ClientName)
		}
	}

	sort.Sort(archives)

	// Convert to a list of Readers... feels like there should be a better way
	// here. Is this what generics are for? Or is there an interface for a list?
	readers := make([]Reader, len(archives))
	for i, a := range archives {
		readers[i] = &a
	}

	return NewSerialReader(readers), nil
}
<|endoftext|>