<commit_before>/*
Copyright 2021 The Ceph-CSI Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cephfs

import (
	"fmt"

	"github.com/ceph/ceph-csi/internal/util"

	"github.com/container-storage-interface/spec/lib/go/csi"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

// validateCreateVolumeRequest validates the Controller CreateVolume request.
func (cs *ControllerServer) validateCreateVolumeRequest(req *csi.CreateVolumeRequest) error {
	if err := cs.Driver.ValidateControllerServiceRequest(
		csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil {
		return fmt.Errorf("invalid CreateVolumeRequest: %w", err)
	}

	if req.GetName() == "" {
		return status.Error(codes.InvalidArgument, "volume Name cannot be empty")
	}

	reqCaps := req.GetVolumeCapabilities()
	if reqCaps == nil {
		return status.Error(codes.InvalidArgument, "volume Capabilities cannot be empty")
	}

	for _, capability := range reqCaps {
		if capability.GetBlock() != nil {
			return status.Error(codes.Unimplemented, "block volume not supported")
		}
	}

	// Allow readonly access mode for volume with content source
	err := util.CheckReadOnlyManyIsSupported(req)
	if err != nil {
		return err
	}

	if req.VolumeContentSource != nil {
		volumeSource := req.VolumeContentSource
		switch volumeSource.Type.(type) {
		case *csi.VolumeContentSource_Snapshot:
			snapshot := req.VolumeContentSource.GetSnapshot()
			// CSI spec requires returning NOT_FOUND when the volumeSource is missing/incorrect.
			if snapshot == nil {
				return status.Error(codes.NotFound, "volume Snapshot cannot be empty")
			}
			if snapshot.GetSnapshotId() == "" {
				return status.Error(codes.NotFound, "volume Snapshot ID cannot be empty")
			}
		case *csi.VolumeContentSource_Volume:
			// CSI spec requires returning NOT_FOUND when the volumeSource is missing/incorrect.
			vol := req.VolumeContentSource.GetVolume()
			if vol == nil {
				return status.Error(codes.NotFound, "volume cannot be empty")
			}
			if vol.GetVolumeId() == "" {
				return status.Error(codes.NotFound, "volume ID cannot be empty")
			}

		default:
			return status.Error(codes.InvalidArgument, "unsupported volume data source")
		}
	}

	return nil
}

// validateDeleteVolumeRequest validates the Controller DeleteVolume request.
func (cs *ControllerServer) validateDeleteVolumeRequest() error {
	if err := cs.Driver.ValidateControllerServiceRequest(
		csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil {
		return fmt.Errorf("invalid DeleteVolumeRequest: %w", err)
	}

	return nil
}

// Controller expand volume request validation.
func (cs *ControllerServer) validateExpandVolumeRequest(req *csi.ControllerExpandVolumeRequest) error {
	if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_EXPAND_VOLUME); err != nil {
		return fmt.Errorf("invalid ExpandVolumeRequest: %w", err)
	}

	if req.GetVolumeId() == "" {
		return status.Error(codes.InvalidArgument, "Volume ID cannot be empty")
	}

	capRange := req.GetCapacityRange()
	if capRange == nil {
		return status.Error(codes.InvalidArgument, "CapacityRange cannot be empty")
	}

	return nil
}
<commit_msg>cephfs: correct comment for validateExpandVolumeRequest<commit_after>/*
Copyright 2021 The Ceph-CSI Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cephfs

import (
	"fmt"

	"github.com/ceph/ceph-csi/internal/util"

	"github.com/container-storage-interface/spec/lib/go/csi"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

// validateCreateVolumeRequest validates the Controller CreateVolume request.
func (cs *ControllerServer) validateCreateVolumeRequest(req *csi.CreateVolumeRequest) error {
	if err := cs.Driver.ValidateControllerServiceRequest(
		csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil {
		return fmt.Errorf("invalid CreateVolumeRequest: %w", err)
	}

	if req.GetName() == "" {
		return status.Error(codes.InvalidArgument, "volume Name cannot be empty")
	}

	reqCaps := req.GetVolumeCapabilities()
	if reqCaps == nil {
		return status.Error(codes.InvalidArgument, "volume Capabilities cannot be empty")
	}

	for _, capability := range reqCaps {
		if capability.GetBlock() != nil {
			return status.Error(codes.Unimplemented, "block volume not supported")
		}
	}

	// Allow readonly access mode for volume with content source
	err := util.CheckReadOnlyManyIsSupported(req)
	if err != nil {
		return err
	}

	if req.VolumeContentSource != nil {
		volumeSource := req.VolumeContentSource
		switch volumeSource.Type.(type) {
		case *csi.VolumeContentSource_Snapshot:
			snapshot := req.VolumeContentSource.GetSnapshot()
			// CSI spec requires returning NOT_FOUND when the volumeSource is missing/incorrect.
			if snapshot == nil {
				return status.Error(codes.NotFound, "volume Snapshot cannot be empty")
			}
			if snapshot.GetSnapshotId() == "" {
				return status.Error(codes.NotFound, "volume Snapshot ID cannot be empty")
			}
		case *csi.VolumeContentSource_Volume:
			// CSI spec requires returning NOT_FOUND when the volumeSource is missing/incorrect.
			vol := req.VolumeContentSource.GetVolume()
			if vol == nil {
				return status.Error(codes.NotFound, "volume cannot be empty")
			}
			if vol.GetVolumeId() == "" {
				return status.Error(codes.NotFound, "volume ID cannot be empty")
			}

		default:
			return status.Error(codes.InvalidArgument, "unsupported volume data source")
		}
	}

	return nil
}

// validateDeleteVolumeRequest validates the Controller DeleteVolume request.
func (cs *ControllerServer) validateDeleteVolumeRequest() error {
	if err := cs.Driver.ValidateControllerServiceRequest(
		csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil {
		return fmt.Errorf("invalid DeleteVolumeRequest: %w", err)
	}

	return nil
}

// validateExpandVolumeRequest validates the Controller ExpandVolume request.
func (cs *ControllerServer) validateExpandVolumeRequest(req *csi.ControllerExpandVolumeRequest) error {
	if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_EXPAND_VOLUME); err != nil {
		return fmt.Errorf("invalid ExpandVolumeRequest: %w", err)
	}

	if req.GetVolumeId() == "" {
		return status.Error(codes.InvalidArgument, "Volume ID cannot be empty")
	}

	capRange := req.GetCapacityRange()
	if capRange == nil {
		return status.Error(codes.InvalidArgument, "CapacityRange cannot be empty")
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package generate

import (
	"context"
	"fmt"
	"net/http"
	"sort"
	"strings"
	"time"

	"go.opencensus.io/trace"
	"go.uber.org/zap"

	"github.com/google/exposure-notifications-server/internal/pb"
	publishdb "github.com/google/exposure-notifications-server/internal/publish/database"
	publishmodel "github.com/google/exposure-notifications-server/internal/publish/model"
	"github.com/google/exposure-notifications-server/internal/serverenv"
	"github.com/google/exposure-notifications-server/internal/verification"
	verifyapi "github.com/google/exposure-notifications-server/pkg/api/v1"
	"github.com/google/exposure-notifications-server/pkg/logging"
	"github.com/google/exposure-notifications-server/pkg/util"
)

func NewHandler(ctx context.Context, config *Config, env *serverenv.ServerEnv) (http.Handler, error) {
	logger := logging.FromContext(ctx).Named("generate")

	if env.Database() == nil {
		return nil, fmt.Errorf("missing database in server environment")
	}

	transformer, err := publishmodel.NewTransformer(config)
	if err != nil {
		return nil, fmt.Errorf("model.NewTransformer: %w", err)
	}
	logger.Debugw("max keys per upload", "value", config.MaxKeysOnPublish)
	logger.Debugw("max interval start age", "value", config.MaxIntervalAge)
	logger.Debugw("truncate window", "value", config.TruncateWindow)

	return &generateHandler{
		serverenv:   env,
		transformer: transformer,
		config:      config,
		database:    publishdb.New(env.Database()),
		logger:      logger,
	}, nil
}

type generateHandler struct {
	config      *Config
	serverenv   *serverenv.ServerEnv
	transformer *publishmodel.Transformer
	database    *publishdb.PublishDB
	logger      *zap.SugaredLogger
}

func (h *generateHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	ctx, span := trace.StartSpan(r.Context(), "(*generate.generateHandler).ServeHTTP")
	defer span.End()

	logger := h.logger.Named("ServeHTTP")

	regionStr := h.config.DefaultRegion
	if v := r.URL.Query().Get("region"); v != "" {
		regionStr = v
	}
	regions := strings.Split(regionStr, ",")

	logger.Debugw("generating data", "regions", regions)

	if err := h.generate(ctx, regions); err != nil {
		logger.Errorw("failed to generate", "error", err)
		span.SetStatus(trace.Status{Code: trace.StatusCodeInternal, Message: err.Error()})
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprint(w, err.Error())
		return
	}

	w.WriteHeader(http.StatusOK)
	fmt.Fprint(w, "successfully generated exposure keys")
}

func (h *generateHandler) generate(ctx context.Context, regions []string) error {
	logger := h.logger.Named("generate")

	if h.config.KeysPerExposure < 2 {
		return fmt.Errorf("number of keys to publish must be at least 2")
	}

	batchTime := time.Now().UTC()
	for i := 0; i < h.config.NumExposures; i++ {
		logger.Debugf("generating exposure %d of %d", i+1, h.config.NumExposures)

		publish := &verifyapi.Publish{
			Keys:              util.GenerateExposureKeys(h.config.KeysPerExposure, 0, false),
			HealthAuthorityID: "generated.data",
		}

		if h.config.SimulateSameDayRelease {
			sort.Slice(publish.Keys, func(i int, j int) bool {
				return publish.Keys[i].IntervalNumber < publish.Keys[j].IntervalNumber
			})

			lastKey := &publish.Keys[len(publish.Keys)-1]
			newLastDayKey, err := util.RandomExposureKey(lastKey.IntervalNumber, 144, lastKey.TransmissionRisk)
			if err != nil {
				return fmt.Errorf("failed to simulate same day key release: %w", err)
			}

			lastKey.IntervalCount = lastKey.IntervalCount / 2
			publish.Keys = append(publish.Keys, newLastDayKey)
		}

		val, err := util.RandomInt(100)
		if err != nil {
			return fmt.Errorf("failed to decide revised key status: %w", err)
		}
		generateRevisedKeys := val <= h.config.ChanceOfKeyRevision

		reportType := verifyapi.ReportTypeClinical
		if !generateRevisedKeys {
			reportType, err = util.RandomReportType()
			if err != nil {
				return fmt.Errorf("failed to generate report type: %w", err)
			}
		}

		intervalIdx, err := util.RandomInt(len(publish.Keys) - 1)
		if err != nil {
			return fmt.Errorf("failed to generate symptom onset interval: %w", err)
		}

		claims := verification.VerifiedClaims{
			ReportType:           reportType,
			SymptomOnsetInterval: uint32(publish.Keys[intervalIdx].IntervalNumber),
		}

		exposures, err := h.transformer.TransformPublish(ctx, publish, regions, &claims, batchTime)
		if err != nil {
			return fmt.Errorf("failed to transform generated exposures: %w", err)
		}

		n, err := h.database.InsertAndReviseExposures(ctx, exposures, nil, true)
		if err != nil {
			return fmt.Errorf("failed to write exposure record: %w", err)
		}
		logger.Debugw("generated exposures", "num", n)

		if generateRevisedKeys {
			revisedReportType, err := util.RandomRevisedReportType()
			if err != nil {
				return fmt.Errorf("failed to generate revised report type: %w", err)
			}

			claims.ReportType = revisedReportType
			batchTime = batchTime.Add(h.config.KeyRevisionDelay)

			exposures, err := h.transformer.TransformPublish(ctx, publish, regions, &claims, batchTime)
			if err != nil {
				return fmt.Errorf("failed to transform generated exposures: %w", err)
			}

			// Build the revision token
			var token pb.RevisionTokenData
			for _, e := range exposures {
				token.RevisableKeys = append(token.RevisableKeys, &pb.RevisableKey{
					TemporaryExposureKey: e.ExposureKey,
					IntervalNumber:       e.IntervalNumber,
					IntervalCount:        e.IntervalCount,
				})
			}

			n, err := h.database.InsertAndReviseExposures(ctx, exposures, &token, true)
			if err != nil {
				return fmt.Errorf("failed to revise exposure record: %w", err)
			}
			logger.Debugw("revised exposures", "num", n)
		}
	}

	return nil
}
<commit_msg>Add comment (#972)<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package generate

import (
	"context"
	"fmt"
	"net/http"
	"sort"
	"strings"
	"time"

	"go.opencensus.io/trace"
	"go.uber.org/zap"

	"github.com/google/exposure-notifications-server/internal/pb"
	publishdb "github.com/google/exposure-notifications-server/internal/publish/database"
	publishmodel "github.com/google/exposure-notifications-server/internal/publish/model"
	"github.com/google/exposure-notifications-server/internal/serverenv"
	"github.com/google/exposure-notifications-server/internal/verification"
	verifyapi "github.com/google/exposure-notifications-server/pkg/api/v1"
	"github.com/google/exposure-notifications-server/pkg/logging"
	"github.com/google/exposure-notifications-server/pkg/util"
)

func NewHandler(ctx context.Context, config *Config, env *serverenv.ServerEnv) (http.Handler, error) {
	logger := logging.FromContext(ctx).Named("generate")

	if env.Database() == nil {
		return nil, fmt.Errorf("missing database in server environment")
	}

	transformer, err := publishmodel.NewTransformer(config)
	if err != nil {
		return nil, fmt.Errorf("model.NewTransformer: %w", err)
	}
	logger.Debugw("max keys per upload", "value", config.MaxKeysOnPublish)
	logger.Debugw("max interval start age", "value", config.MaxIntervalAge)
	logger.Debugw("truncate window", "value", config.TruncateWindow)

	return &generateHandler{
		serverenv:   env,
		transformer: transformer,
		config:      config,
		database:    publishdb.New(env.Database()),
		logger:      logger,
	}, nil
}

type generateHandler struct {
	config      *Config
	serverenv   *serverenv.ServerEnv
	transformer *publishmodel.Transformer
	database    *publishdb.PublishDB
	logger      *zap.SugaredLogger
}

func (h *generateHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	ctx, span := trace.StartSpan(r.Context(), "(*generate.generateHandler).ServeHTTP")
	defer span.End()

	logger := h.logger.Named("ServeHTTP")

	regionStr := h.config.DefaultRegion
	if v := r.URL.Query().Get("region"); v != "" {
		regionStr = v
	}
	regions := strings.Split(regionStr, ",")

	logger.Debugw("generating data", "regions", regions)

	if err := h.generate(ctx, regions); err != nil {
		logger.Errorw("failed to generate", "error", err)
		span.SetStatus(trace.Status{Code: trace.StatusCodeInternal, Message: err.Error()})
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprint(w, err.Error())
		return
	}

	w.WriteHeader(http.StatusOK)
	fmt.Fprint(w, "successfully generated exposure keys")
}

func (h *generateHandler) generate(ctx context.Context, regions []string) error {
	logger := h.logger.Named("generate")

	// We require at least 2 keys because revision only revises a subset of keys,
	// and that subset selects a random sample from (0-len(keys)], and rand panics
	// if you try to generate a random number between 0 and 0 :).
	if h.config.KeysPerExposure < 2 {
		return fmt.Errorf("number of keys to publish must be at least 2")
	}

	batchTime := time.Now().UTC()
	for i := 0; i < h.config.NumExposures; i++ {
		logger.Debugf("generating exposure %d of %d", i+1, h.config.NumExposures)

		publish := &verifyapi.Publish{
			Keys:              util.GenerateExposureKeys(h.config.KeysPerExposure, 0, false),
			HealthAuthorityID: "generated.data",
		}

		if h.config.SimulateSameDayRelease {
			sort.Slice(publish.Keys, func(i int, j int) bool {
				return publish.Keys[i].IntervalNumber < publish.Keys[j].IntervalNumber
			})

			lastKey := &publish.Keys[len(publish.Keys)-1]
			newLastDayKey, err := util.RandomExposureKey(lastKey.IntervalNumber, 144, lastKey.TransmissionRisk)
			if err != nil {
				return fmt.Errorf("failed to simulate same day key release: %w", err)
			}

			lastKey.IntervalCount = lastKey.IntervalCount / 2
			publish.Keys = append(publish.Keys, newLastDayKey)
		}

		val, err := util.RandomInt(100)
		if err != nil {
			return fmt.Errorf("failed to decide revised key status: %w", err)
		}
		generateRevisedKeys := val <= h.config.ChanceOfKeyRevision

		reportType := verifyapi.ReportTypeClinical
		if !generateRevisedKeys {
			reportType, err = util.RandomReportType()
			if err != nil {
				return fmt.Errorf("failed to generate report type: %w", err)
			}
		}

		intervalIdx, err := util.RandomInt(len(publish.Keys) - 1)
		if err != nil {
			return fmt.Errorf("failed to generate symptom onset interval: %w", err)
		}

		claims := verification.VerifiedClaims{
			ReportType:           reportType,
			SymptomOnsetInterval: uint32(publish.Keys[intervalIdx].IntervalNumber),
		}

		exposures, err := h.transformer.TransformPublish(ctx, publish, regions, &claims, batchTime)
		if err != nil {
			return fmt.Errorf("failed to transform generated exposures: %w", err)
		}

		n, err := h.database.InsertAndReviseExposures(ctx, exposures, nil, true)
		if err != nil {
			return fmt.Errorf("failed to write exposure record: %w", err)
		}
		logger.Debugw("generated exposures", "num", n)

		if generateRevisedKeys {
			revisedReportType, err := util.RandomRevisedReportType()
			if err != nil {
				return fmt.Errorf("failed to generate revised report type: %w", err)
			}

			claims.ReportType = revisedReportType
			batchTime = batchTime.Add(h.config.KeyRevisionDelay)

			exposures, err := h.transformer.TransformPublish(ctx, publish, regions, &claims, batchTime)
			if err != nil {
				return fmt.Errorf("failed to transform generated exposures: %w", err)
			}

			// Build the revision token
			var token pb.RevisionTokenData
			for _, e := range exposures {
				token.RevisableKeys = append(token.RevisableKeys, &pb.RevisableKey{
					TemporaryExposureKey: e.ExposureKey,
					IntervalNumber:       e.IntervalNumber,
					IntervalCount:        e.IntervalCount,
				})
			}

			n, err := h.database.InsertAndReviseExposures(ctx, exposures, &token, true)
			if err != nil {
				return fmt.Errorf("failed to revise exposure record: %w", err)
			}
			logger.Debugw("revised exposures", "num", n)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2016 Jacques Supcik <jacques.supcik@hefr.ch>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//   _   _                     _                             _        _
//  | |_| |__  _   _ _ __ ___ (_) ___         ___ __ _ _ __ | |_ __ _(_)_ __
//  | __| '_ \| | | | '_ ` _ \| |/ _ \ _____ / __/ _` | '_ \| __/ _` | | '_ \
//  | |_| | | | |_| | | | | | | | (_) |_____| (_| (_| | |_) | || (_| | | | | |
//   \__|_| |_|\__, |_| |_| |_|_|\___/       \___\__,_| .__/ \__\__,_|_|_| |_|
//             |___/                                  |_|
//

// Frontend
package main

import (
	"crypto/hmac"
	"crypto/sha1"
	"encoding/base64"
	"flag"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/gorilla/mux"
	"github.com/gorilla/sessions"
	"github.com/kidstuff/mongostore"
	"gopkg.in/mgo.v2"
	"html/template"
	"net/http"
	"path/filepath"
)

const (
	dbName     = "thymio_captain"
	sessionC   = "sessions"
	maxAge     = 24 * 3600
	sessionKey = "session-key"
	root       = "internal_pages"
)

var (
	database       *mgo.Session
	store          *mongostore.MongoStore
	adminSecretKey *string
	startSecretKey *string
	templates      = make(map[string]*template.Template)
)

func initSession(w http.ResponseWriter, r *http.Request) (vars map[string]string, session *sessions.Session, err error) {
	database.Refresh()
	vars = mux.Vars(r)
	session, err = store.Get(r, sessionKey)
	if err != nil {
		log.Error(err.Error())
		http.Error(w, "Session Error", 500)
	} else {
		w.Header().Set("Cache-Control", "max-age=0, no-cache, no-store")
		w.Header().Set("Pragma", "no-cache")
	}
	return
}

func isValidToken(token string, key string) bool {
	t, err := base64.RawURLEncoding.DecodeString(token)
	if err != nil {
		return false
	}

	if len(t) != 40 { // TODO: replace MAGIC
		log.Infof("Invalid token length: %d", len(t))
		return false
	}
	data := t[0:20] // TODO: replace MAGIC
	sig := t[20:40] // TODO: replace MAGIC
	mac := hmac.New(sha1.New, []byte(key))
	mac.Write(data)
	log.Debugf("HMAC: %v", mac.Sum(nil))
	log.Debugf("EXP : %v", sig)
	return hmac.Equal(mac.Sum(nil), sig)
}

func CardLogin(w http.ResponseWriter, r *http.Request) {
	vars, session, err := initSession(w, r)
	if err != nil {
		return
	}
	if isValidToken(vars["CardId"], *adminSecretKey) {
		log.Debugf("Valid Card Login: %v", vars["CardId"])
		session.Values["admin"] = "1"
		sessions.Save(r, w)
		err = templates["login-ok.html"].Execute(w, nil)
	} else {
		log.Infof("Bad Card Login: %v", vars["CardId"])
		session.Values["admin"] = "0"
		sessions.Save(r, w)
		err = templates["login-failed.html"].Execute(w, nil)
	}
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func Logout(w http.ResponseWriter, r *http.Request) {
	_, session, err := initSession(w, r)
	if err != nil {
		return
	}

	log.Debug("Logout")
	session.Values["admin"] = "0"
	sessions.Save(r, w)

	err = templates["logout.html"].Execute(w, nil)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func Index(w http.ResponseWriter, r *http.Request) {
	err := templates["index.html"].Execute(w, nil)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func Help(w http.ResponseWriter, r *http.Request) {
	err := templates["help.html"].Execute(w, nil)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func About(w http.ResponseWriter, r *http.Request) {
	err := templates["about.html"].Execute(w, nil)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func notFound(w http.ResponseWriter, r *http.Request) {
	err := templates["404.html"].Execute(w, nil)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func Debug(w http.ResponseWriter, r *http.Request) {
	_, session, err := initSession(w, r)
	if err != nil {
		return
	}

	log.Debug("Debug page")
	tmpl, err := template.ParseFiles(root + "/debug.html")
	if err != nil {
		log.Infof("Error 1 %s", err)
	}
	s := fmt.Sprintf("%v", session.Values)
	if err != nil {
		log.Infof("Error 2 %s", err)
	}
	err = tmpl.Execute(w, struct{ Session string }{string(s)})
}

func Start(w http.ResponseWriter, r *http.Request) {
	vars, session, err := initSession(w, r)
	if err != nil {
		return
	}

	if *startSecretKey == "" || isValidToken(vars["CardId"], *startSecretKey) {
		log.Debugf("Valid Start page: %v", vars["CardId"])
		session.Values["cardId"] = vars["CardId"]
		sessions.Save(r, w)

		if session.Values["admin"] == "1" {
			http.ServeFile(w, r, root+"/admin.html")
		} else {
			http.ServeFile(w, r, root+"/public.html")
		}
	} else {
		log.Infof("Bad Start page: %v", vars["CardId"])
		err = templates["bad-cards.html"].Execute(w, nil)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
	}
}

func main() {
	var port = flag.Int("port", 8080, "port")
	var debug = flag.Bool("debug", false, "run in debug mode")
	var domain = flag.String("domain", "thymio.tk", "Domain name (for the cookie)")
	var mongoServer = flag.String("mongo-server", "localhost", "MongoDB server URL")
	var cookieSecretKey = flag.String("cookie-secret-key", "not-so-secret", "Secret key (for secure cookies)")
	adminSecretKey = flag.String("admin-secret-key", "change-me", "Secret key (for admin card-login)")
	startSecretKey = flag.String("start-secret-key", "", "Secret key (for start ID)")

	flag.Parse()

	if *debug {
		log.SetLevel(log.DebugLevel)
		log.Debug("Debug mode")
	} else {
		log.SetLevel(log.InfoLevel)
	}

	if *startSecretKey == "" {
		log.Warn("Running without start id validation")
	} else {
		log.Info("Start id validation enabled")
	}

	var err error
	database, err = mgo.Dial(*mongoServer)
	if err != nil {
		log.Fatal(err)
	}
	store = mongostore.NewMongoStore(
		database.DB(dbName).C(sessionC),
		maxAge, true, []byte(*cookieSecretKey))

	store.Options.Domain = *domain

	tpls := template.Must(template.ParseGlob("internal_pages/templates/*")) // TODO: Replace Magic
	nameList, err := filepath.Glob("internal_pages/*.html")
	if err != nil {
		panic(err)
	}
	for _, name := range nameList {
		log.Debugf("Reading %s", name)
		key := filepath.Base(name)
		t, _ := tpls.Clone()
		templates[key] = t
		_, err = templates[key].ParseFiles(name)
		if err != nil {
			panic(err)
		}
	}

	r := mux.NewRouter()
	r.HandleFunc("/start/{CardId}", Start)
	r.HandleFunc("/cardlogin/{CardId}", CardLogin)
	r.HandleFunc("/logout", Logout)
	r.HandleFunc("/debug", Debug)

	r.HandleFunc("/", Index)
	r.HandleFunc("/about", About)
	r.HandleFunc("/help", Help)

	r.NotFoundHandler = http.HandlerFunc(notFound)

	http.Handle("/img/", http.StripPrefix("/img/", http.FileServer(http.Dir("img"))))
	http.Handle("/css/", http.StripPrefix("/css/", http.FileServer(http.Dir("css"))))
	http.Handle("/vendor/", http.StripPrefix("/vendor/", http.FileServer(http.Dir("vendor"))))

	http.Handle("/", r)
	log.Infof("Ready, listening on port %d", *port)
	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil))
}
<commit_msg>Add debug<commit_after>// Copyright 2016 Jacques Supcik <jacques.supcik@hefr.ch>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//   _   _                     _                             _        _
//  | |_| |__  _   _ _ __ ___ (_) ___         ___ __ _ _ __ | |_ __ _(_)_ __
//  | __| '_ \| | | | '_ ` _ \| |/ _ \ _____ / __/ _` | '_ \| __/ _` | | '_ \
//  | |_| | | | |_| | | | | | | | (_) |_____| (_| (_| | |_) | || (_| | | | | |
//   \__|_| |_|\__, |_| |_| |_|_|\___/       \___\__,_| .__/ \__\__,_|_|_| |_|
//             |___/                                  |_|
//

// Frontend
package main

import (
	"crypto/hmac"
	"crypto/sha1"
	"encoding/base64"
	"flag"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/gorilla/mux"
	"github.com/gorilla/sessions"
	"github.com/kidstuff/mongostore"
	"gopkg.in/mgo.v2"
	"html/template"
	"net/http"
	"path/filepath"
)

const (
	dbName        = "thymio_captain"
	sessionC      = "sessions"
	maxAge        = 24 * 3600
	sessionKey    = "session-key"
	root          = "internal_pages"
	TOKEN_RND_LEN = 20
	TOKEN_SIG_LEN = 20
)

var (
	database       *mgo.Session
	store          *mongostore.MongoStore
	adminSecretKey *string
	startSecretKey *string
	templates      = make(map[string]*template.Template)
)

func initSession(w http.ResponseWriter, r *http.Request) (vars map[string]string, session *sessions.Session, err error) {
	database.Refresh()
	vars = mux.Vars(r)
	session, err = store.Get(r, sessionKey)
	log.Debugf("Session ID = %v", session.ID)
	if err != nil {
		log.Error(err.Error())
		http.Error(w, "Session Error", 500)
	} else {
		w.Header().Set("Cache-Control", "max-age=0, no-cache, no-store")
		w.Header().Set("Pragma", "no-cache")
	}
	return
}

func isValidToken(token string, key string) bool {
	t, err := base64.RawURLEncoding.DecodeString(token)
	if err != nil {
		return false
	}

	if len(t) != TOKEN_RND_LEN+TOKEN_SIG_LEN {
		log.Infof("Invalid token length: %d", len(t))
		return false
	}
	data := t[0:TOKEN_RND_LEN]
	sig := t[TOKEN_RND_LEN : TOKEN_RND_LEN+TOKEN_SIG_LEN]
	mac := hmac.New(sha1.New, []byte(key))
	mac.Write(data)
	log.Debugf("HMAC: %v", mac.Sum(nil))
	log.Debugf("EXP : %v", sig)
	return hmac.Equal(mac.Sum(nil), sig)
}

func CardLogin(w http.ResponseWriter, r *http.Request) {
	vars, session, err := initSession(w, r)
	if err != nil {
		return
	}
	if isValidToken(vars["CardId"], *adminSecretKey) {
		log.Debugf("Valid Card Login: %v", vars["CardId"])
		session.Values["admin"] = "1"
		sessions.Save(r, w)
		err = templates["login-ok.html"].Execute(w, nil)
	} else {
		log.Infof("Bad Card Login: %v", vars["CardId"])
		session.Values["admin"] = "0"
		sessions.Save(r, w)
		err = templates["login-failed.html"].Execute(w, nil)
	}
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func Logout(w http.ResponseWriter, r *http.Request) {
	_, session, err := initSession(w, r)
	if err != nil {
		return
	}

	log.Debug("Logout")
	session.Values["admin"] = "0"
	sessions.Save(r, w)

	err = templates["logout.html"].Execute(w, nil)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func Index(w http.ResponseWriter, r *http.Request) {
	err := templates["index.html"].Execute(w, nil)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func Help(w http.ResponseWriter, r *http.Request) {
	err := templates["help.html"].Execute(w, nil)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func About(w http.ResponseWriter, r *http.Request) {
	err := templates["about.html"].Execute(w, nil)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func notFound(w http.ResponseWriter, r *http.Request) {
	err := templates["404.html"].Execute(w, nil)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func Debug(w http.ResponseWriter, r *http.Request) {
	_, session, err := initSession(w, r)
	if err != nil {
		return
	}

	log.Debug("Debug page")
	tmpl, err := template.ParseFiles(root + "/debug.html")
	if err != nil {
		log.Infof("Error 1 %s", err)
	}
	s := fmt.Sprintf("%v", session.Values)
	if err != nil {
		log.Infof("Error 2 %s", err)
	}
	err = tmpl.Execute(w, struct{ Session string }{string(s)})
}

func Start(w http.ResponseWriter, r *http.Request) {
	vars, session, err := initSession(w, r)
	if err != nil {
		return
	}

	if *startSecretKey == "" || isValidToken(vars["CardId"], *startSecretKey) {
		log.Debugf("Valid Start page: %v", vars["CardId"])
		session.Values["cardId"] = vars["CardId"]
		sessions.Save(r, w)

		if session.Values["admin"] == "1" {
			log.Debug("Sending Admin UI")
			http.ServeFile(w, r, root+"/admin.html")
		} else {
			log.Debug("Sending User UI")
			http.ServeFile(w, r, root+"/public.html")
		}
	} else {
		log.Infof("Bad Start page: %v", vars["CardId"])
		err = templates["bad-cards.html"].Execute(w, nil)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
	}
}

func main() {
	var port = flag.Int("port", 8080, "port")
	var debug = flag.Bool("debug", false, "run in debug mode")
	var domain = flag.String("domain", "thymio.tk", "Domain name (for the cookie)")
	var mongoServer = flag.String("mongo-server", "localhost", "MongoDB server URL")
	var cookieSecretKey = flag.String("cookie-secret-key", "not-so-secret", "Secret key (for secure cookies)")
	adminSecretKey = flag.String("admin-secret-key", "change-me", "Secret key (for admin card-login)")
	startSecretKey = flag.String("start-secret-key", "", "Secret key (for start ID)")

	flag.Parse()

	if *debug {
		log.SetLevel(log.DebugLevel)
		log.Debug("Debug mode")
	} else {
		log.SetLevel(log.InfoLevel)
	}

	if *startSecretKey == "" {
		log.Warn("Running without start id validation")
	} else {
		log.Info("Start id validation enabled")
	}

	var err error
	database, err = mgo.Dial(*mongoServer)
	if err != nil {
		log.Fatal(err)
	}
	store = mongostore.NewMongoStore(
		database.DB(dbName).C(sessionC),
		maxAge, true, []byte(*cookieSecretKey))

	store.Options.Domain = *domain

	tpls := template.Must(template.ParseGlob("internal_pages/templates/*"))
	nameList, err := filepath.Glob("internal_pages/*.html")
	if err != nil {
		panic(err)
	}
	for _, name := range nameList {
		log.Debugf("Reading %s", name)
		key := filepath.Base(name)
		t, _ := tpls.Clone()
		templates[key] = t
		_, err = templates[key].ParseFiles(name)
		if err != nil {
			panic(err)
		}
	}

	r := mux.NewRouter()
	r.HandleFunc("/start/{CardId}", Start)
	r.HandleFunc("/cardlogin/{CardId}", CardLogin)
	r.HandleFunc("/logout", Logout)
	r.HandleFunc("/debug", Debug)

	r.HandleFunc("/", Index)
	r.HandleFunc("/about", About)
	r.HandleFunc("/help", Help)

	r.NotFoundHandler = http.HandlerFunc(notFound)

	http.Handle("/img/", http.StripPrefix("/img/", http.FileServer(http.Dir("img"))))
	http.Handle("/css/", http.StripPrefix("/css/", http.FileServer(http.Dir("css"))))
	http.Handle("/vendor/", http.StripPrefix("/vendor/", http.FileServer(http.Dir("vendor"))))

	http.Handle("/", r)
	log.Infof("Ready, listening on port %d", *port)
	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *port), nil))
}
<|endoftext|>


<commit_before>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ledcmd

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/golang/protobuf/jsonpb"
	"go.chromium.org/luci/auth"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"

	"go.chromium.org/luci/led/job"
)

// EditRecipeBundleOpts are user-provided options for the recipe bundling
// process.
type EditRecipeBundleOpts struct {
	// Path on disk to the repo to extract the recipes from. May be a subdirectory
	// of the repo, as long as `git rev-parse --show-toplevel` can find the root
	// of the repository.
	//
	// If empty, uses the current working directory.
	RepoDir string

	// Overrides is a mapping of recipe project id (e.g. "recipe_engine") to
	// a local path to a checkout of that repo (e.g. "/path/to/recipes-py.git").
	//
	// When the bundle is created, this local repo will be used instead of the
	// pinned version of this recipe project id. This is helpful for preparing
	// bundles which have code changes in multiple recipe repos.
	Overrides map[string]string

	// DebugSleep is the amount of time to wait after the recipe completes
	// execution (either success or failure). This is injected into the generated
	// recipe bundle as a 'sleep X' command after the invocation of the recipe
	// itself.
	DebugSleep time.Duration

	// PropertyOnly determines whether to pass the recipe bundle's CAS reference
	// as a property and preserve the executable and payload of the input job
	// rather than overwriting it.
	PropertyOnly bool
}

const (
	// RecipeDirectory is a very unfortunate constant which is here for
	// a combination of reasons:
	//   1) swarming doesn't allow you to 'checkout' an isolate relative to any
	//      path in the task (other than the task root). This means that
	//      whatever value we pick for EditRecipeBundle must be used EVERYWHERE
	//      the isolated hash is used.
	//   2) Currently the 'recipe_engine/led' module will blindly take the
	//      isolated input and 'inject' it into further uses of led. This module
	//      currently doesn't specify the checkout dir, relying on kitchen's
	//      default value of (you guessed it) "kitchen-checkout".
	//
	// In order to fix this (and it will need to be fixed for bbagent support):
	//   * The 'recipe_engine/led' module needs to accept 'checkout-dir' as
	//     a parameter in its input properties.
	//   * led needs to start passing the checkout dir to the led module's input
	//     properties.
	//   * `led edit` needs a way to manipulate the checkout directory in a job
	//   * The 'recipe_engine/led' module needs to set this in the job
	//     alongside the isolate hash when it's doing the injection.
	//
	// For now, we just hard-code it.
	//
	// TODO(crbug.com/1072117): Fix this, it's weird.
	RecipeDirectory = "kitchen-checkout"

	// A property that should be set to a boolean value. If true,
	// edit-recipe-bundle will set the "led_cas_recipe_bundle" property
	// instead of overwriting the build's payload.
	LEDBuilderIsBootstrappedProperty = "led_builder_is_bootstrapped"

	// In PropertyOnly mode or if the "led_builder_is_bootstrapped" property
	// of the build is true, this property will be set with the CAS digest
	// of the executable of the recipe bundle.
	CASRecipeBundleProperty = "led_cas_recipe_bundle"
)

// EditRecipeBundle overrides the recipe bundle in the given job with one
// located on disk.
//
// It isolates the recipes from the repository in the given working directory
// into the UserPayload under the directory "kitchen-checkout/". If there's an
// existing directory in the UserPayload at that location, it will be removed.
func EditRecipeBundle(ctx context.Context, authOpts auth.Options, jd *job.Definition, opts *EditRecipeBundleOpts) error {
	if jd.GetBuildbucket() == nil {
		return errors.New("ledcmd.EditRecipeBundle is only available for Buildbucket tasks")
	}

	if opts == nil {
		opts = &EditRecipeBundleOpts{}
	}

	recipesPy, err := findRecipesPy(ctx, opts.RepoDir)
	if err != nil {
		return err
	}
	logging.Debugf(ctx, "using recipes.py: %q", recipesPy)

	extraProperties := make(map[string]string)
	setRecipeBundleProperty := opts.PropertyOnly || jd.GetBuildbucket().GetBbagentArgs().GetBuild().GetInput().GetProperties().GetFields()[LEDBuilderIsBootstrappedProperty].GetBoolValue()
	if setRecipeBundleProperty {
		// In property-only mode, we want to leave the original payload as is
		// and just upload the recipe bundle as a brand new independent CAS
		// archive for the job's executable to download.
		bundlePath, err := ioutil.TempDir("", "led-recipe-bundle")
		if err != nil {
			return errors.Annotate(err, "creating temporary recipe bundle directory").Err()
		}
		if err := opts.prepBundle(ctx, opts.RepoDir, recipesPy, bundlePath); err != nil {
			return err
		}
		logging.Infof(ctx, "isolating recipes")
		casClient, err := newCASClient(ctx, authOpts, jd)
		if err != nil {
			return err
		}
		casRef, err := uploadToCas(ctx, casClient, bundlePath)
		if err != nil {
			return err
		}
		m := &jsonpb.Marshaler{OrigName: true}
		jsonCASRef, err := m.MarshalToString(casRef)
		if err != nil {
			return errors.Annotate(err, "encoding CAS user payload").Err()
		}
		extraProperties[CASRecipeBundleProperty] = jsonCASRef
	} else {
		if err := EditIsolated(ctx, authOpts, jd, func(ctx context.Context, dir string) error {
			bundlePath := filepath.Join(dir, RecipeDirectory)
			// Remove existing bundled recipes, if any. Ignore the error.
			os.RemoveAll(bundlePath)
			if err := opts.prepBundle(ctx, opts.RepoDir, recipesPy, bundlePath); err != nil {
				return err
			}
			logging.Infof(ctx, "isolating recipes")
			return nil
		}); err != nil {
			return err
		}
	}

	return jd.HighLevelEdit(func(je job.HighLevelEditor) {
		if setRecipeBundleProperty {
			je.Properties(extraProperties, false)
		} else {
			je.TaskPayloadSource("", "")
			je.TaskPayloadPath(RecipeDirectory)
		}
		if opts.DebugSleep != 0 {
			je.Env(map[string]string{
				"RECIPES_DEBUG_SLEEP": fmt.Sprintf("%f", opts.DebugSleep.Seconds()),
			})
		}
	})
}

func logCmd(ctx context.Context, inDir string, arg0 string, args ...string) *exec.Cmd {
	ret := exec.CommandContext(ctx, arg0, args...)
	ret.Dir = inDir
	logging.Debugf(ctx, "Running (from %q) - %s %v", inDir, arg0, args)
	return ret
}

func cmdErr(cmd *exec.Cmd, err error, reason string) error {
	if err == nil {
		return nil
	}
	var outErr string
	if ee, ok := err.(*exec.ExitError); ok {
		outErr = strings.TrimSpace(string(ee.Stderr))
		if len(outErr) > 128 {
			outErr = outErr[:128] + "..."
		}
	} else {
		outErr = err.Error()
	}
	return errors.Annotate(err, "running %q: %s: %s", strings.Join(cmd.Args, " "), reason, outErr).Err()
}

func appendText(path, fmtStr string, items ...interface{}) error {
	file, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0666)
	if err != nil {
		return err
	}
	defer file.Close()
	_, err = fmt.Fprintf(file, fmtStr, items...)
	return err
}

func (opts *EditRecipeBundleOpts) prepBundle(ctx context.Context, inDir, recipesPy, toDirectory string) error {
	logging.Infof(ctx, "bundling recipes")
	args := []string{
		recipesPy,
	}
	if logging.GetLevel(ctx) < logging.Info {
		args = append(args, "-v")
	}
	for projID, path := range opts.Overrides {
		args = append(args, "-O", fmt.Sprintf("%s=%s", projID, path))
	}
	args = append(args, "bundle", "--destination", filepath.Join(toDirectory))
	cmd := logCmd(ctx, inDir, "python", args...)
	if logging.GetLevel(ctx) < logging.Info {
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
	}
	return cmdErr(cmd, cmd.Run(), "creating bundle")
}

// findRecipesPy locates the current repo's `recipes.py`. It does this by:
//   * invoking git to find the repo root
//   * loading the recipes.cfg at infra/config/recipes.cfg
//   * stat'ing the recipes.py implied by the recipes_path in that cfg file.
//
// Failure will return an error.
//
// On success, the absolute path to recipes.py is returned.
func findRecipesPy(ctx context.Context, inDir string) (string, error) {
	cmd := logCmd(ctx, inDir, "git", "rev-parse", "--show-toplevel")
	out, err := cmd.Output()
	if err = cmdErr(cmd, err, "finding git repo"); err != nil {
		return "", err
	}

	repoRoot := strings.TrimSpace(string(out))

	pth := filepath.Join(repoRoot, "infra", "config", "recipes.cfg")
	switch st, err := os.Stat(pth); {
	case err != nil:
		return "", errors.Annotate(err, "reading recipes.cfg").Err()

	case !st.Mode().IsRegular():
		return "", errors.Reason("%q is not a regular file", pth).Err()
	}

	type recipesJSON struct {
		RecipesPath string `json:"recipes_path"`
	}
	rj := &recipesJSON{}

	f, err := os.Open(pth)
	if err != nil {
		return "", errors.Reason("reading recipes.cfg: %q", pth).Err()
	}
	defer f.Close()

	if err := json.NewDecoder(f).Decode(rj); err != nil {
		return "", errors.Reason("parsing recipes.cfg: %q", pth).Err()
	}

	return filepath.Join(
		repoRoot, filepath.FromSlash(rj.RecipesPath), "recipes.py"), nil
}
<commit_msg>[led] Use LookPath to prefer python3 if available when bundling recipes.<commit_after>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ledcmd

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/golang/protobuf/jsonpb"
	"go.chromium.org/luci/auth"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"

	"go.chromium.org/luci/led/job"
)

// EditRecipeBundleOpts are user-provided options for the recipe bundling
// process.
type EditRecipeBundleOpts struct {
	// Path on disk to the repo to extract the recipes from. May be a subdirectory
	// of the repo, as long as `git rev-parse --show-toplevel` can find the root
	// of the repository.
	//
	// If empty, uses the current working directory.
	RepoDir string

	// Overrides is a mapping of recipe project id (e.g. "recipe_engine") to
	// a local path to a checkout of that repo (e.g. "/path/to/recipes-py.git").
	//
	// When the bundle is created, this local repo will be used instead of the
	// pinned version of this recipe project id. This is helpful for preparing
	// bundles which have code changes in multiple recipe repos.
	Overrides map[string]string

	// DebugSleep is the amount of time to wait after the recipe completes
	// execution (either success or failure). This is injected into the generated
	// recipe bundle as a 'sleep X' command after the invocation of the recipe
	// itself.
	DebugSleep time.Duration

	// PropertyOnly determines whether to pass the recipe bundle's CAS reference
	// as a property and preserve the executable and payload of the input job
	// rather than overwriting it.
	PropertyOnly bool
}

const (
	// RecipeDirectory is a very unfortunate constant which is here for
	// a combination of reasons:
	//   1) swarming doesn't allow you to 'checkout' an isolate relative to any
	//      path in the task (other than the task root). This means that
	//      whatever value we pick for EditRecipeBundle must be used EVERYWHERE
	//      the isolated hash is used.
	//   2) Currently the 'recipe_engine/led' module will blindly take the
	//      isolated input and 'inject' it into further uses of led. This module
	//      currently doesn't specify the checkout dir, relying on kitchen's
	//      default value of (you guessed it) "kitchen-checkout".
	//
	// In order to fix this (and it will need to be fixed for bbagent support):
	//   * The 'recipe_engine/led' module needs to accept 'checkout-dir' as
	//     a parameter in its input properties.
	//   * led needs to start passing the checkout dir to the led module's input
	//     properties.
	//   * `led edit` needs a way to manipulate the checkout directory in a job
	//   * The 'recipe_engine/led' module needs to set this in the job
	//     alongside the isolate hash when it's doing the injection.
	//
	// For now, we just hard-code it.
	//
	// TODO(crbug.com/1072117): Fix this, it's weird.
	RecipeDirectory = "kitchen-checkout"

	// A property that should be set to a boolean value. If true,
	// edit-recipe-bundle will set the "led_cas_recipe_bundle" property
	// instead of overwriting the build's payload.
	LEDBuilderIsBootstrappedProperty = "led_builder_is_bootstrapped"

	// In PropertyOnly mode or if the "led_builder_is_bootstrapped" property
	// of the build is true, this property will be set with the CAS digest
	// of the executable of the recipe bundle.
	CASRecipeBundleProperty = "led_cas_recipe_bundle"
)

// EditRecipeBundle overrides the recipe bundle in the given job with one
// located on disk.
//
// It isolates the recipes from the repository in the given working directory
// into the UserPayload under the directory "kitchen-checkout/". If there's an
// existing directory in the UserPayload at that location, it will be removed.
func EditRecipeBundle(ctx context.Context, authOpts auth.Options, jd *job.Definition, opts *EditRecipeBundleOpts) error {
	if jd.GetBuildbucket() == nil {
		return errors.New("ledcmd.EditRecipeBundle is only available for Buildbucket tasks")
	}

	if opts == nil {
		opts = &EditRecipeBundleOpts{}
	}

	recipesPy, err := findRecipesPy(ctx, opts.RepoDir)
	if err != nil {
		return err
	}
	logging.Debugf(ctx, "using recipes.py: %q", recipesPy)

	extraProperties := make(map[string]string)
	setRecipeBundleProperty := opts.PropertyOnly || jd.GetBuildbucket().GetBbagentArgs().GetBuild().GetInput().GetProperties().GetFields()[LEDBuilderIsBootstrappedProperty].GetBoolValue()
	if setRecipeBundleProperty {
		// In property-only mode, we want to leave the original payload as is
		// and just upload the recipe bundle as a brand new independent CAS
		// archive for the job's executable to download.
		bundlePath, err := ioutil.TempDir("", "led-recipe-bundle")
		if err != nil {
			return errors.Annotate(err, "creating temporary recipe bundle directory").Err()
		}
		if err := opts.prepBundle(ctx, opts.RepoDir, recipesPy, bundlePath); err != nil {
			return err
		}
		logging.Infof(ctx, "isolating recipes")
		casClient, err := newCASClient(ctx, authOpts, jd)
		if err != nil {
			return err
		}
		casRef, err := uploadToCas(ctx, casClient, bundlePath)
		if err != nil {
			return err
		}
		m := &jsonpb.Marshaler{OrigName: true}
		jsonCASRef, err := m.MarshalToString(casRef)
		if err != nil {
			return errors.Annotate(err, "encoding CAS user payload").Err()
		}
		extraProperties[CASRecipeBundleProperty] = jsonCASRef
	} else {
		if err := EditIsolated(ctx, authOpts, jd, func(ctx context.Context, dir string) error {
			bundlePath := filepath.Join(dir, RecipeDirectory)
			// Remove existing bundled recipes, if any. Ignore the error.
			os.RemoveAll(bundlePath)
			if err := opts.prepBundle(ctx, opts.RepoDir, recipesPy, bundlePath); err != nil {
				return err
			}
			logging.Infof(ctx, "isolating recipes")
			return nil
		}); err != nil {
			return err
		}
	}

	return jd.HighLevelEdit(func(je job.HighLevelEditor) {
		if setRecipeBundleProperty {
			je.Properties(extraProperties, false)
		} else {
			je.TaskPayloadSource("", "")
			je.TaskPayloadPath(RecipeDirectory)
		}
		if opts.DebugSleep != 0 {
			je.Env(map[string]string{
				"RECIPES_DEBUG_SLEEP": fmt.Sprintf("%f", opts.DebugSleep.Seconds()),
			})
		}
	})
}

func logCmd(ctx context.Context, inDir string, arg0 string, args ...string) *exec.Cmd {
	ret := exec.CommandContext(ctx, arg0, args...)
	ret.Dir = inDir
	logging.Debugf(ctx, "Running (from %q) - %s %v", inDir, arg0, args)
	return ret
}

func cmdErr(cmd *exec.Cmd, err error, reason string) error {
	if err == nil {
		return nil
	}
	var outErr string
	if ee, ok := err.(*exec.ExitError); ok {
		outErr = strings.TrimSpace(string(ee.Stderr))
		if len(outErr) > 128 {
			outErr = outErr[:128] + "..."
		}
	} else {
		outErr = err.Error()
	}
	return errors.Annotate(err, "running %q: %s: %s", strings.Join(cmd.Args, " "), reason, outErr).Err()
}

func appendText(path, fmtStr string, items ...interface{}) error {
	file, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0666)
	if err != nil {
		return err
	}
	defer file.Close()
	_, err = fmt.Fprintf(file, fmtStr, items...)
	return err
}

func (opts *EditRecipeBundleOpts) prepBundle(ctx context.Context, inDir, recipesPy, toDirectory string) error {
	logging.Infof(ctx, "bundling recipes")
	args := []string{
		recipesPy,
	}
	if logging.GetLevel(ctx) < logging.Info {
		args = append(args, "-v")
	}
	for projID, path := range opts.Overrides {
		args = append(args, "-O", fmt.Sprintf("%s=%s", projID, path))
	}
	args = append(args, "bundle", "--destination", filepath.Join(toDirectory))

	// Always prefer python3 to python
	python, err := exec.LookPath("python3")
	if err != nil {
		python, err = exec.LookPath("python")
	}
	if err != nil {
		return errors.Annotate(err, "unable to find python3 or python in $PATH").Err()
	}

	cmd := logCmd(ctx, inDir, python, args...)
	if logging.GetLevel(ctx) < logging.Info {
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
	}
	return cmdErr(cmd, cmd.Run(), "creating bundle")
}

// findRecipesPy locates the current repo's `recipes.py`. It does this by:
//   * invoking git to find the repo root
//   * loading the recipes.cfg at infra/config/recipes.cfg
//   * stat'ing the recipes.py implied by the recipes_path in that cfg file.
//
// Failure will return an error.
//
// On success, the absolute path to recipes.py is returned.
func findRecipesPy(ctx context.Context, inDir string) (string, error) {
	cmd := logCmd(ctx, inDir, "git", "rev-parse", "--show-toplevel")
	out, err := cmd.Output()
	if err = cmdErr(cmd, err, "finding git repo"); err != nil {
		return "", err
	}

	repoRoot := strings.TrimSpace(string(out))

	pth := filepath.Join(repoRoot, "infra", "config", "recipes.cfg")
	switch st, err := os.Stat(pth); {
	case err != nil:
		return "", errors.Annotate(err, "reading recipes.cfg").Err()

	case !st.Mode().IsRegular():
		return "", errors.Reason("%q is not a regular file", pth).Err()
	}

	type recipesJSON struct {
		RecipesPath string `json:"recipes_path"`
	}
	rj := &recipesJSON{}

	f, err := os.Open(pth)
	if err != nil {
		return "", errors.Reason("reading recipes.cfg: %q", pth).Err()
	}
	defer f.Close()

	if err := json.NewDecoder(f).Decode(rj); err != nil {
		return "", errors.Reason("parsing recipes.cfg: %q", pth).Err()
	}

	return filepath.Join(
		repoRoot, filepath.FromSlash(rj.RecipesPath), "recipes.py"), nil
}
<|endoftext|>


<commit_before>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ledcmd

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/golang/protobuf/jsonpb"
	"go.chromium.org/luci/auth"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"

	"go.chromium.org/luci/led/job"
)

// EditRecipeBundleOpts are user-provided options for the recipe bundling
// process.
type EditRecipeBundleOpts struct {
	// Path on disk to the repo to extract the recipes from. May be a subdirectory
	// of the repo, as long as `git rev-parse --show-toplevel` can find the root
	// of the repository.
	//
	// If empty, uses the current working directory.
	RepoDir string

	// Overrides is a mapping of recipe project id (e.g. "recipe_engine") to
	// a local path to a checkout of that repo (e.g. "/path/to/recipes-py.git").
	//
	// When the bundle is created, this local repo will be used instead of the
	// pinned version of this recipe project id. This is helpful for preparing
	// bundles which have code changes in multiple recipe repos.
	Overrides map[string]string

	// DebugSleep is the amount of time to wait after the recipe completes
	// execution (either success or failure). This is injected into the generated
	// recipe bundle as a 'sleep X' command after the invocation of the recipe
	// itself.
	DebugSleep time.Duration

	// PropertyOnly determines whether to pass the recipe bundle's CAS reference
	// as a property and preserve the executable and payload of the input job
	// rather than overwriting it.
	PropertyOnly bool
}

const (
	// RecipeDirectory is a very unfortunate constant which is here for
	// a combination of reasons:
	//   1) swarming doesn't allow you to 'checkout' an isolate relative to any
	//      path in the task (other than the task root). This means that
	//      whatever value we pick for EditRecipeBundle must be used EVERYWHERE
	//      the isolated hash is used.
	//   2) Currently the 'recipe_engine/led' module will blindly take the
	//      isolated input and 'inject' it into further uses of led. This module
	//      currently doesn't specify the checkout dir, relying on kitchen's
	//      default value of (you guessed it) "kitchen-checkout".
	//
	// In order to fix this (and it will need to be fixed for bbagent support):
	//   * The 'recipe_engine/led' module needs to accept 'checkout-dir' as
	//     a parameter in its input properties.
	//   * led needs to start passing the checkout dir to the led module's input
	//     properties.
	//   * `led edit` needs a way to manipulate the checkout directory in a job
	//   * The 'recipe_engine/led' module needs to set this in the job
	//     alongside the isolate hash when it's doing the injection.
	//
	// For now, we just hard-code it.
	//
	// TODO(crbug.com/1072117): Fix this, it's weird.
	RecipeDirectory = "kitchen-checkout"

	// A property that should be set to a boolean value. If true,
	// edit-recipe-bundle will set the "led_cas_recipe_bundle" property
	// instead of overwriting the build's payload.
	LEDBuilderIsBootstrappedProperty = "led_builder_is_bootstrapped"

	// In PropertyOnly mode or if the "led_builder_is_bootstrapped" property
	// of the build is true, this property will be set with the CAS digest
	// of the executable of the recipe bundle.
	CASRecipeBundleProperty = "led_cas_recipe_bundle"
)

// EditRecipeBundle overrides the recipe bundle in the given job with one
// located on disk.
//
// It isolates the recipes from the repository in the given working directory
// into the UserPayload under the directory "kitchen-checkout/". If there's an
// existing directory in the UserPayload at that location, it will be removed.
func EditRecipeBundle(ctx context.Context, authOpts auth.Options, jd *job.Definition, opts *EditRecipeBundleOpts) error {
	if jd.GetBuildbucket() == nil {
		return errors.New("ledcmd.EditRecipeBundle is only available for Buildbucket tasks")
	}

	if opts == nil {
		opts = &EditRecipeBundleOpts{}
	}

	recipesPy, err := findRecipesPy(ctx, opts.RepoDir)
	if err != nil {
		return err
	}
	logging.Debugf(ctx, "using recipes.py: %q", recipesPy)

	extraProperties := make(map[string]string)
	setRecipeBundleProperty := opts.PropertyOnly || jd.GetBuildbucket().GetBbagentArgs().GetBuild().GetInput().GetProperties().GetFields()[LEDBuilderIsBootstrappedProperty].GetBoolValue()
	if setRecipeBundleProperty {
		// In property-only mode, we want to leave the original payload as is
		// and just upload the recipe bundle as a brand new independent CAS
		// archive for the job's executable to download.
		bundlePath, err := ioutil.TempDir("", "led-recipe-bundle")
		if err != nil {
			return errors.Annotate(err, "creating temporary recipe bundle directory").Err()
		}
		if err := opts.prepBundle(ctx, opts.RepoDir, recipesPy, bundlePath); err != nil {
			return err
		}
		logging.Infof(ctx, "isolating recipes")
		casClient, err := newCASClient(ctx, authOpts, jd)
		if err != nil {
			return err
		}
		casRef, err := uploadToCas(ctx, casClient, bundlePath)
		if err != nil {
			return err
		}
		m := &jsonpb.Marshaler{OrigName: true}
		jsonCASRef, err := m.MarshalToString(casRef)
		if err != nil {
			return errors.Annotate(err, "encoding CAS user payload").Err()
		}
		extraProperties[CASRecipeBundleProperty] = jsonCASRef
	} else {
		if err := EditIsolated(ctx, authOpts, jd, func(ctx context.Context, dir string) error {
			bundlePath := filepath.Join(dir, RecipeDirectory)
			// Remove existing bundled recipes, if any. Ignore the error.
			os.RemoveAll(bundlePath)
			if err := opts.prepBundle(ctx, opts.RepoDir, recipesPy, bundlePath); err != nil {
				return err
			}
			logging.Infof(ctx, "isolating recipes")
			return nil
		}); err != nil {
			return err
		}
	}

	return jd.HighLevelEdit(func(je job.HighLevelEditor) {
		if setRecipeBundleProperty {
			je.Properties(extraProperties, false)
		}
		if opts.DebugSleep != 0 {
			je.Env(map[string]string{
				"RECIPES_DEBUG_SLEEP": fmt.Sprintf("%f", opts.DebugSleep.Seconds()),
			})
		}
	})
}

func logCmd(ctx context.Context, inDir string, arg0 string, args ...string) *exec.Cmd {
	ret := exec.CommandContext(ctx, arg0, args...)
	ret.Dir = inDir
	logging.Debugf(ctx, "Running (from %q) - %s %v", inDir, arg0, args)
	return ret
}

func cmdErr(cmd *exec.Cmd, err error, reason string) error {
	if err == nil {
		return nil
	}
	var outErr string
	if ee, ok := err.(*exec.ExitError); ok {
		outErr = strings.TrimSpace(string(ee.Stderr))
		if len(outErr) > 128 {
			outErr = outErr[:128] + "..."
		}
	} else {
		outErr = err.Error()
	}
	return errors.Annotate(err, "running %q: %s: %s", strings.Join(cmd.Args, " "), reason, outErr).Err()
}

func appendText(path, fmtStr string, items ...interface{}) error {
	file, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0666)
	if err != nil {
		return err
	}
	defer file.Close()
	_, err = fmt.Fprintf(file, fmtStr, items...)
	return err
}

func (opts *EditRecipeBundleOpts) prepBundle(ctx context.Context, inDir, recipesPy, toDirectory string) error {
	logging.Infof(ctx, "bundling recipes")
	args := []string{
		recipesPy,
	}
	if logging.GetLevel(ctx) < logging.Info {
		args = append(args, "-v")
	}
	for projID, path := range opts.Overrides {
		args = append(args, "-O", fmt.Sprintf("%s=%s", projID, path))
	}
	args = append(args, "bundle", "--destination", filepath.Join(toDirectory))

	// Always prefer python3 to python
	python, err := exec.LookPath("python3")
	if err != nil {
		python, err = exec.LookPath("python")
	}
	if err != nil {
		return errors.Annotate(err, "unable to find python3 or python in $PATH").Err()
	}

	cmd := logCmd(ctx, inDir, python, args...)
	if logging.GetLevel(ctx) < logging.Info {
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
	}
	return cmdErr(cmd, cmd.Run(), "creating bundle")
}

// findRecipesPy locates the current repo's `recipes.py`. It does this by:
//   - invoking git to find the repo root
//   - loading the recipes.cfg at infra/config/recipes.cfg
//   - stat'ing the recipes.py implied by the recipes_path in that cfg file.
//
// Failure will return an error.
//
// On success, the absolute path to recipes.py is returned.
func findRecipesPy(ctx context.Context, inDir string) (string, error) {
	cmd := logCmd(ctx, inDir, "git", "rev-parse", "--show-toplevel")
	out, err := cmd.Output()
	if err = cmdErr(cmd, err, "finding git repo"); err != nil {
		return "", err
	}

	repoRoot := strings.TrimSpace(string(out))

	pth := filepath.Join(repoRoot, "infra", "config", "recipes.cfg")
	switch st, err := os.Stat(pth); {
	case err != nil:
		return "", errors.Annotate(err, "reading recipes.cfg").Err()

	case !st.Mode().IsRegular():
		return "", errors.Reason("%q is not a regular file", pth).Err()
	}

	type recipesJSON struct {
		RecipesPath string `json:"recipes_path"`
	}
	rj := &recipesJSON{}

	f, err := os.Open(pth)
	if err != nil {
		return "", errors.Reason("reading recipes.cfg: %q", pth).Err()
	}
	defer f.Close()

	if err := json.NewDecoder(f).Decode(rj); err != nil {
		return "", errors.Reason("parsing recipes.cfg: %q", pth).Err()
	}

	return filepath.Join(
		repoRoot, filepath.FromSlash(rj.RecipesPath), "recipes.py"), nil
}
<commit_msg>[led] do not put the recipe bundle in "kitchen-checkout" dir if bbagent will do the package download in the led build<commit_after>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ledcmd

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/golang/protobuf/jsonpb"
	"go.chromium.org/luci/auth"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"

	"go.chromium.org/luci/led/job"
)

// EditRecipeBundleOpts are user-provided options for the recipe bundling
// process.
type EditRecipeBundleOpts struct {
	// Path on disk to the repo to extract the recipes from. May be a subdirectory
	// of the repo, as long as `git rev-parse --show-toplevel` can find the root
	// of the repository.
	//
	// If empty, uses the current working directory.
	RepoDir string

	// Overrides is a mapping of recipe project id (e.g. "recipe_engine") to
	// a local path to a checkout of that repo (e.g. "/path/to/recipes-py.git").
	//
	// When the bundle is created, this local repo will be used instead of the
	// pinned version of this recipe project id. This is helpful for preparing
	// bundles which have code changes in multiple recipe repos.
	Overrides map[string]string

	// DebugSleep is the amount of time to wait after the recipe completes
	// execution (either success or failure). This is injected into the generated
	// recipe bundle as a 'sleep X' command after the invocation of the recipe
	// itself.
	DebugSleep time.Duration

	// PropertyOnly determines whether to pass the recipe bundle's CAS reference
	// as a property and preserve the executable and payload of the input job
	// rather than overwriting it.
	PropertyOnly bool
}

const (
	// RecipeDirectory is a very unfortunate constant which is here for
	// a combination of reasons:
	//   1) swarming doesn't allow you to 'checkout' an isolate relative to any
	//      path in the task (other than the task root). This means that
	//      whatever value we pick for EditRecipeBundle must be used EVERYWHERE
	//      the isolated hash is used.
	//   2) Currently the 'recipe_engine/led' module will blindly take the
	//      isolated input and 'inject' it into further uses of led. This module
	//      currently doesn't specify the checkout dir, relying on kitchen's
	//      default value of (you guessed it) "kitchen-checkout".
	//
	// In order to fix this (and it will need to be fixed for bbagent support):
	//   * The 'recipe_engine/led' module needs to accept 'checkout-dir' as
	//     a parameter in its input properties.
	//   * led needs to start passing the checkout dir to the led module's input
	//     properties.
	//   * `led edit` needs a way to manipulate the checkout directory in a job
	//   * The 'recipe_engine/led' module needs to set this in the job
	//     alongside the isolate hash when it's doing the injection.
	//
	// For now, we just hard-code it.
	//
	// TODO(crbug.com/1072117): Fix this, it's weird.
	RecipeDirectory = "kitchen-checkout"

	// A property that should be set to a boolean value. If true,
	// edit-recipe-bundle will set the "led_cas_recipe_bundle" property
	// instead of overwriting the build's payload.
	LEDBuilderIsBootstrappedProperty = "led_builder_is_bootstrapped"

	// In PropertyOnly mode or if the "led_builder_is_bootstrapped" property
	// of the build is true, this property will be set with the CAS digest
	// of the executable of the recipe bundle.
	CASRecipeBundleProperty = "led_cas_recipe_bundle"
)

// EditRecipeBundle overrides the recipe bundle in the given job with one
// located on disk.
//
// It isolates the recipes from the repository in the given working directory
// into the UserPayload under the directory "kitchen-checkout/". If there's an
// existing directory in the UserPayload at that location, it will be removed.
func EditRecipeBundle(ctx context.Context, authOpts auth.Options, jd *job.Definition, opts *EditRecipeBundleOpts) error {
	if jd.GetBuildbucket() == nil {
		return errors.New("ledcmd.EditRecipeBundle is only available for Buildbucket tasks")
	}

	if opts == nil {
		opts = &EditRecipeBundleOpts{}
	}

	recipesPy, err := findRecipesPy(ctx, opts.RepoDir)
	if err != nil {
		return err
	}
	logging.Debugf(ctx, "using recipes.py: %q", recipesPy)

	extraProperties := make(map[string]string)
	setRecipeBundleProperty := opts.PropertyOnly || jd.GetBuildbucket().GetBbagentArgs().GetBuild().GetInput().GetProperties().GetFields()[LEDBuilderIsBootstrappedProperty].GetBoolValue()
	if setRecipeBundleProperty {
		// In property-only mode, we want to leave the original payload as is
		// and just upload the recipe bundle as a brand new independent CAS
		// archive for the job's executable to download.
		bundlePath, err := ioutil.TempDir("", "led-recipe-bundle")
		if err != nil {
			return errors.Annotate(err, "creating temporary recipe bundle directory").Err()
		}
		if err := opts.prepBundle(ctx, opts.RepoDir, recipesPy, bundlePath); err != nil {
			return err
		}
		logging.Infof(ctx, "isolating recipes")
		casClient, err := newCASClient(ctx, authOpts, jd)
		if err != nil {
			return err
		}
		casRef, err := uploadToCas(ctx, casClient, bundlePath)
		if err != nil {
			return err
		}
		m := &jsonpb.Marshaler{OrigName: true}
		jsonCASRef, err := m.MarshalToString(casRef)
		if err != nil {
			return errors.Annotate(err, "encoding CAS user payload").Err()
		}
		extraProperties[CASRecipeBundleProperty] = jsonCASRef
	} else {
		if err := EditIsolated(ctx, authOpts, jd, func(ctx context.Context, dir string) error {
			bundlePath := dir
			if !jd.GetBuildbucket().BbagentDownloadCIPDPkgs() {
				bundlePath = filepath.Join(dir, RecipeDirectory)
			}
			// Remove existing bundled recipes, if any. Ignore the error.
			os.RemoveAll(bundlePath)
			if err := opts.prepBundle(ctx, opts.RepoDir, recipesPy, bundlePath); err != nil {
				return err
			}
			logging.Infof(ctx, "isolating recipes")
			return nil
		}); err != nil {
			return err
		}
	}

	return jd.HighLevelEdit(func(je job.HighLevelEditor) {
		if setRecipeBundleProperty {
			je.Properties(extraProperties, false)
		}
		if opts.DebugSleep != 0 {
			je.Env(map[string]string{
				"RECIPES_DEBUG_SLEEP": fmt.Sprintf("%f", opts.DebugSleep.Seconds()),
			})
		}
	})
}

func logCmd(ctx context.Context, inDir string, arg0 string, args ...string) *exec.Cmd {
	ret := exec.CommandContext(ctx, arg0, args...)
	ret.Dir = inDir
	logging.Debugf(ctx, "Running (from %q) - %s %v", inDir, arg0, args)
	return ret
}

func cmdErr(cmd *exec.Cmd, err error, reason string) error {
	if err == nil {
		return nil
	}
	var outErr string
	if ee, ok := err.(*exec.ExitError); ok {
		outErr = strings.TrimSpace(string(ee.Stderr))
		if len(outErr) > 128 {
			outErr = outErr[:128] + "..."
		}
	} else {
		outErr = err.Error()
	}
	return errors.Annotate(err, "running %q: %s: %s", strings.Join(cmd.Args, " "), reason, outErr).Err()
}

func appendText(path, fmtStr string, items ...interface{}) error {
	file, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0666)
	if err != nil {
		return err
	}
	defer file.Close()
	_, err = fmt.Fprintf(file, fmtStr, items...)
	return err
}

func (opts *EditRecipeBundleOpts) prepBundle(ctx context.Context, inDir, recipesPy, toDirectory string) error {
	logging.Infof(ctx, "bundling recipes")
	args := []string{
		recipesPy,
	}
	if logging.GetLevel(ctx) < logging.Info {
		args = append(args, "-v")
	}
	for projID, path := range opts.Overrides {
		args = append(args, "-O", fmt.Sprintf("%s=%s", projID, path))
	}
	args = append(args, "bundle", "--destination", filepath.Join(toDirectory))

	// Always prefer python3 to python
	python, err := exec.LookPath("python3")
	if err != nil {
		python, err = exec.LookPath("python")
	}
	if err != nil {
		return errors.Annotate(err, "unable to find python3 or python in $PATH").Err()
	}

	cmd := logCmd(ctx, inDir, python, args...)
	if logging.GetLevel(ctx) < logging.Info {
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
	}
	return cmdErr(cmd, cmd.Run(), "creating bundle")
}

// findRecipesPy locates the current repo's `recipes.py`. It does this by:
//   - invoking git to find the repo root
//   - loading the recipes.cfg at infra/config/recipes.cfg
//   - stat'ing the recipes.py implied by the recipes_path in that cfg file.
//
// Failure will return an error.
//
// On success, the absolute path to recipes.py is returned.
func findRecipesPy(ctx context.Context, inDir string) (string, error) {
	cmd := logCmd(ctx, inDir, "git", "rev-parse", "--show-toplevel")
	out, err := cmd.Output()
	if err = cmdErr(cmd, err, "finding git repo"); err != nil {
		return "", err
	}

	repoRoot := strings.TrimSpace(string(out))

	pth := filepath.Join(repoRoot, "infra", "config", "recipes.cfg")
	switch st, err := os.Stat(pth); {
	case err != nil:
		return "", errors.Annotate(err, "reading recipes.cfg").Err()

	case !st.Mode().IsRegular():
		return "", errors.Reason("%q is not a regular file", pth).Err()
	}

	type recipesJSON struct {
		RecipesPath string `json:"recipes_path"`
	}
	rj := &recipesJSON{}

	f, err := os.Open(pth)
	if err != nil {
		return "", errors.Reason("reading recipes.cfg: %q", pth).Err()
	}
	defer f.Close()

	if err := json.NewDecoder(f).Decode(rj); err != nil {
		return "", errors.Reason("parsing recipes.cfg: %q", pth).Err()
	}

	return filepath.Join(
		repoRoot, filepath.FromSlash(rj.RecipesPath), "recipes.py"), nil
}
<|endoftext|>


<commit_before>package resource

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"testing"

	"github.com/hashicorp/terraform/config"
	"github.com/hashicorp/terraform/terraform"
)

const TestEnvVar = "TF_ACC"

// TestCheckFunc is the callback type used with acceptance tests to check
// the state of a resource. The state passed in is the latest state known,
// or in the case of being after a destroy, it is the last known state when
// it was created.
type TestCheckFunc func(*terraform.State) error

// TestCase is a single acceptance test case used to test the apply/destroy
// lifecycle of a resource in a specific configuration.
//
// When the destroy plan is executed, the config from the last TestStep
// is used to plan it.
type TestCase struct {
	// PreCheck, if non-nil, will be called before any test steps are
	// executed. It will only be executed in the case that the steps
	// would run, so it can be used for some validation before running
	// acceptance tests, such as verifying that keys are setup.
	PreCheck func()

	// Provider is the ResourceProvider that will be under test.
	Providers map[string]terraform.ResourceProvider

	// CheckDestroy is called after the resource is finally destroyed
	// to allow the tester to test that the resource is truly gone.
	CheckDestroy TestCheckFunc

	// Steps are the apply sequences done within the context of the
	// same state. Each step can have its own check to verify correctness.
	Steps []TestStep
}

// TestStep is a single apply sequence of a test, done within the
// context of a state.
//
// Multiple TestSteps can be sequenced in a Test to allow testing
// potentially complex update logic. In general, simply create/destroy
// tests will only need one step.
type TestStep struct {
	// Config a string of the configuration to give to Terraform.
	Config string

	// Check is called after the Config is applied. Use this step to
	// make your own API calls to check the status of things, and to
	// inspect the format of the ResourceState itself.
	//
	// If an error is returned, the test will fail. In this case, a
	// destroy plan will still be attempted.
	//
	// If this is nil, no check is done on this step.
	Check TestCheckFunc

	// Destroy will create a destroy plan if set to true.
	Destroy bool
}

// Test performs an acceptance test on a resource.
//
// Tests are not run unless an environmental variable "TF_ACC" is
// set to some non-empty value. This is to avoid test cases surprising
// a user by creating real resources.
//
// Tests will fail unless the verbose flag (`go test -v`, or explicitly
// the "-test.v" flag) is set. Because some acceptance tests take quite
// long, we require the verbose flag so users are able to see progress
// output.
func Test(t TestT, c TestCase) {
	// We only run acceptance tests if an env var is set because they're
	// slow and generally require some outside configuration.
	if os.Getenv(TestEnvVar) == "" {
		t.Skip(fmt.Sprintf(
			"Acceptance tests skipped unless env '%s' set",
			TestEnvVar))
		return
	}

	// We require verbose mode so that the user knows what is going on.
	if !testTesting && !testing.Verbose() {
		t.Fatal("Acceptance tests must be run with the -v flag on tests")
		return
	}

	// Run the PreCheck if we have it
	if c.PreCheck != nil {
		c.PreCheck()
	}

	// Build our context options that we can
	ctxProviders := make(map[string]terraform.ResourceProviderFactory)
	for k, p := range c.Providers {
		ctxProviders[k] = terraform.ResourceProviderFactoryFixed(p)
	}
	opts := terraform.ContextOpts{Providers: ctxProviders}

	// A single state variable to track the lifecycle, starting with no state
	var state *terraform.State

	// Go through each step and run it
	for i, step := range c.Steps {
		var err error
		log.Printf("[WARN] Test: Executing step %d", i)
		state, err = testStep(opts, state, step)
		if err != nil {
			t.Error(fmt.Sprintf(
				"Step %d error: %s", i, err))
			break
		}
	}

	// If we have a state, then run the destroy
	if state != nil {
		destroyStep := TestStep{
			Config:  c.Steps[len(c.Steps)-1].Config,
			Check:   c.CheckDestroy,
			Destroy: true,
		}

		log.Printf("[WARN] Test: Executing destroy step")
		state, err := testStep(opts, state, destroyStep)
		if err != nil {
			t.Error(fmt.Sprintf(
				"Error destroying resource! WARNING: Dangling resources\n"+
					"may exist. The full state and error is shown below.\n\n"+
					"Error: %s\n\nState: %s",
				err,
				state))
		}
	} else {
		log.Printf("[WARN] Skipping destroy test since there is no state.")
	}
}

func testStep(
	opts terraform.ContextOpts,
	state *terraform.State,
	step TestStep) (*terraform.State, error) {
	// Write the configuration
	cfgF, err := ioutil.TempFile("", "tf-test")
	if err != nil {
		return state, fmt.Errorf(
			"Error creating temporary file for config: %s", err)
	}
	cfgPath := cfgF.Name() + ".tf"
	cfgF.Close()
	os.Remove(cfgF.Name())

	cfgF, err = os.Create(cfgPath)
	if err != nil {
		return state, fmt.Errorf(
			"Error creating temporary file for config: %s", err)
	}
	defer os.Remove(cfgPath)

	_, err = io.Copy(cfgF, strings.NewReader(step.Config))
	cfgF.Close()
	if err != nil {
		return state, fmt.Errorf(
			"Error creating temporary file for config: %s", err)
	}

	// Parse the configuration
	config, err := config.Load(cfgPath)
	if err != nil {
		return state, fmt.Errorf(
			"Error parsing configuration: %s", err)
	}

	// Build the context
	opts.Config = config
	opts.State = state
	ctx := terraform.NewContext(&opts)
	if ws, es := ctx.Validate(); len(ws) > 0 || len(es) > 0 {
		return state, fmt.Errorf(
			"Configuration is invalid.\n\nWarnings: %#v\n\nErrors: %#v",
			ws, es)
	}

	// Refresh!
	state, err = ctx.Refresh()
	if err != nil {
		return state, fmt.Errorf(
			"Error refreshing: %s", err)
	}

	// Plan!
	if _, err := ctx.Plan(&terraform.PlanOpts{Destroy: step.Destroy}); err != nil {
		return state, fmt.Errorf(
			"Error planning: %s", err)
	}

	// Apply!
	state, err = ctx.Apply()
	if err != nil {
		return state, fmt.Errorf("Error applying: %s", err)
	}

	// Check! Excitement!
	if step.Check != nil {
		if err = step.Check(state); err != nil {
			err = fmt.Errorf("Check failed: %s", err)
		}
	}

	return state, err
}

// ComposeTestCheckFunc lets you compose multiple TestCheckFuncs into
// a single TestCheckFunc.
//
// As a user testing their provider, this lets you decompose your checks
// into smaller pieces more easily.
func ComposeTestCheckFunc(fs ...TestCheckFunc) TestCheckFunc {
	return func(s *terraform.State) error {
		for _, f := range fs {
			if err := f(s); err != nil {
				return err
			}
		}

		return nil
	}
}

func TestCheckResourceAttr(name, key, value string) TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.Resources[name]
		if !ok {
			return fmt.Errorf("Not found: %s", name)
		}

		if rs.Attributes[key] != value {
			return fmt.Errorf(
				"%s: Attribute '%s' expected %#v, got %#v",
				name,
				key,
				value,
				rs.Attributes[key])
		}

		return nil
	}
}

// TestT is the interface used to handle the test lifecycle of a test.
//
// Users should just use a *testing.T object, which implements this.
type TestT interface {
	Error(args ...interface{})
	Fatal(args ...interface{})
	Skip(args ...interface{})
}

// This is set to true by unit tests to alter some behavior
var testTesting = false
<commit_msg>helper/resource: better error if config doesn't validate<commit_after>package resource

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"testing"

	"github.com/hashicorp/terraform/config"
	"github.com/hashicorp/terraform/terraform"
)

const TestEnvVar = "TF_ACC"

// TestCheckFunc is the callback type used with acceptance tests to check
// the state of a resource. The state passed in is the latest state known,
// or in the case of being after a destroy, it is the last known state when
// it was created.
type TestCheckFunc func(*terraform.State) error

// TestCase is a single acceptance test case used to test the apply/destroy
// lifecycle of a resource in a specific configuration.
//
// When the destroy plan is executed, the config from the last TestStep
// is used to plan it.
type TestCase struct {
	// PreCheck, if non-nil, will be called before any test steps are
	// executed. It will only be executed in the case that the steps
	// would run, so it can be used for some validation before running
	// acceptance tests, such as verifying that keys are setup.
	PreCheck func()

	// Provider is the ResourceProvider that will be under test.
	Providers map[string]terraform.ResourceProvider

	// CheckDestroy is called after the resource is finally destroyed
	// to allow the tester to test that the resource is truly gone.
	CheckDestroy TestCheckFunc

	// Steps are the apply sequences done within the context of the
	// same state. Each step can have its own check to verify correctness.
	Steps []TestStep
}

// TestStep is a single apply sequence of a test, done within the
// context of a state.
//
// Multiple TestSteps can be sequenced in a Test to allow testing
// potentially complex update logic. In general, simply create/destroy
// tests will only need one step.
type TestStep struct {
	// Config a string of the configuration to give to Terraform.
	Config string

	// Check is called after the Config is applied. Use this step to
	// make your own API calls to check the status of things, and to
	// inspect the format of the ResourceState itself.
	//
	// If an error is returned, the test will fail. In this case, a
	// destroy plan will still be attempted.
	//
	// If this is nil, no check is done on this step.
	Check TestCheckFunc

	// Destroy will create a destroy plan if set to true.
	Destroy bool
}

// Test performs an acceptance test on a resource.
//
// Tests are not run unless an environmental variable "TF_ACC" is
// set to some non-empty value. This is to avoid test cases surprising
// a user by creating real resources.
//
// Tests will fail unless the verbose flag (`go test -v`, or explicitly
// the "-test.v" flag) is set. Because some acceptance tests take quite
// long, we require the verbose flag so users are able to see progress
// output.
func Test(t TestT, c TestCase) {
	// We only run acceptance tests if an env var is set because they're
	// slow and generally require some outside configuration.
	if os.Getenv(TestEnvVar) == "" {
		t.Skip(fmt.Sprintf(
			"Acceptance tests skipped unless env '%s' set",
			TestEnvVar))
		return
	}

	// We require verbose mode so that the user knows what is going on.
	if !testTesting && !testing.Verbose() {
		t.Fatal("Acceptance tests must be run with the -v flag on tests")
		return
	}

	// Run the PreCheck if we have it
	if c.PreCheck != nil {
		c.PreCheck()
	}

	// Build our context options that we can
	ctxProviders := make(map[string]terraform.ResourceProviderFactory)
	for k, p := range c.Providers {
		ctxProviders[k] = terraform.ResourceProviderFactoryFixed(p)
	}
	opts := terraform.ContextOpts{Providers: ctxProviders}

	// A single state variable to track the lifecycle, starting with no state
	var state *terraform.State

	// Go through each step and run it
	for i, step := range c.Steps {
		var err error
		log.Printf("[WARN] Test: Executing step %d", i)
		state, err = testStep(opts, state, step)
		if err != nil {
			t.Error(fmt.Sprintf(
				"Step %d error: %s", i, err))
			break
		}
	}

	// If we have a state, then run the destroy
	if state != nil {
		destroyStep := TestStep{
			Config:  c.Steps[len(c.Steps)-1].Config,
			Check:   c.CheckDestroy,
			Destroy: true,
		}

		log.Printf("[WARN] Test: Executing destroy step")
		state, err := testStep(opts, state, destroyStep)
		if err != nil {
			t.Error(fmt.Sprintf(
				"Error destroying resource! WARNING: Dangling resources\n"+
					"may exist. The full state and error is shown below.\n\n"+
					"Error: %s\n\nState: %s",
				err,
				state))
		}
	} else {
		log.Printf("[WARN] Skipping destroy test since there is no state.")
	}
}

func testStep(
	opts terraform.ContextOpts,
	state *terraform.State,
	step TestStep) (*terraform.State, error) {
	// Write the configuration
	cfgF, err := ioutil.TempFile("", "tf-test")
	if err != nil {
		return state, fmt.Errorf(
			"Error creating temporary file for config: %s", err)
	}
	cfgPath := cfgF.Name() + ".tf"
	cfgF.Close()
	os.Remove(cfgF.Name())

	cfgF, err = os.Create(cfgPath)
	if err != nil {
		return state, fmt.Errorf(
			"Error creating temporary file for config: %s", err)
	}
	defer os.Remove(cfgPath)

	_, err = io.Copy(cfgF, strings.NewReader(step.Config))
	cfgF.Close()
	if err != nil {
		return state, fmt.Errorf(
			"Error creating temporary file for config: %s", err)
	}

	// Parse the configuration
	config, err := config.Load(cfgPath)
	if err != nil {
		return state, fmt.Errorf(
			"Error parsing configuration: %s", err)
	}

	// Build the context
	opts.Config = config
	opts.State = state
	ctx := terraform.NewContext(&opts)
	if ws, es := ctx.Validate(); len(ws) > 0 || len(es) > 0 {
		estrs := make([]string, len(es))
		for i, e := range es {
			estrs[i] = e.Error()
		}
		return state, fmt.Errorf(
			"Configuration is invalid.\n\nWarnings: %#v\n\nErrors: %#v",
			ws, estrs)
	}

	// Refresh!
	state, err = ctx.Refresh()
	if err != nil {
		return state, fmt.Errorf(
			"Error refreshing: %s", err)
	}

	// Plan!
	if _, err := ctx.Plan(&terraform.PlanOpts{Destroy: step.Destroy}); err != nil {
		return state, fmt.Errorf(
			"Error planning: %s", err)
	}

	// Apply!
	state, err = ctx.Apply()
	if err != nil {
		return state, fmt.Errorf("Error applying: %s", err)
	}

	// Check! Excitement!
	if step.Check != nil {
		if err = step.Check(state); err != nil {
			err = fmt.Errorf("Check failed: %s", err)
		}
	}

	return state, err
}

// ComposeTestCheckFunc lets you compose multiple TestCheckFuncs into
// a single TestCheckFunc.
//
// As a user testing their provider, this lets you decompose your checks
// into smaller pieces more easily.
func ComposeTestCheckFunc(fs ...TestCheckFunc) TestCheckFunc {
	return func(s *terraform.State) error {
		for _, f := range fs {
			if err := f(s); err != nil {
				return err
			}
		}

		return nil
	}
}

func TestCheckResourceAttr(name, key, value string) TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.Resources[name]
		if !ok {
			return fmt.Errorf("Not found: %s", name)
		}

		if rs.Attributes[key] != value {
			return fmt.Errorf(
				"%s: Attribute '%s' expected %#v, got %#v",
				name,
				key,
				value,
				rs.Attributes[key])
		}

		return nil
	}
}

// TestT is the interface used to handle the test lifecycle of a test.
//
// Users should just use a *testing.T object, which implements this.
type TestT interface {
	Error(args ...interface{})
	Fatal(args ...interface{})
	Skip(args ...interface{})
}

// This is set to true by unit tests to alter some behavior
var testTesting = false
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime"
	"mime/multipart"
	"net/http"
	"strings"
)

type configuration struct {
	Endpoint string `json:"endpoint"`
	Port     string `json:"port"`
}

func loadConfig(path string) configuration {
	file, err := ioutil.ReadFile(path)
	if err != nil {
		log.Fatal("Config File Missing. ", err)
	}
	var conf configuration
	err = json.Unmarshal(file, &conf)
	if err != nil {
		log.Fatal("Config Parse Error: ", err)
	}
	return conf
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "%s", "Hello World")
}

func getBoundary(value string, contentType string) (string, *strings.Reader) {
	body := strings.NewReader(value)
	bodySplit := strings.Split(string(value), contentType)
	scanner := bufio.NewScanner(strings.NewReader(bodySplit[1]))
	var lines []string
	for scanner.Scan() {
		lines = append(lines, scanner.Text())
		break
	}
	boundary := lines[0][9:]
	return boundary, body
}

func inboundHandler(w http.ResponseWriter, r *http.Request) {
	mediaType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
	if err != nil {
		log.Fatal(err)
	}
	if strings.HasPrefix(mediaType, "multipart/") {
		mr := multipart.NewReader(r.Body, params["boundary"])
		parsedEmail := make(map[string]string)
		emailHeader := make(map[string]string)
		//TODO: this needs to be a struct
		binaryFiles := make(map[string][]byte)
		rawEmail := make(map[string]string)
		rawFiles := make(map[string]string)
		for {
			p, err := mr.NextPart()
			// We have found an attachment, binary data
			if err == nil && p.FileName() != "" {
				//contentType := p.Header.Get("Content-Type")
				contents, err := ioutil.ReadAll(p)
				if err != nil {
					log.Fatal(err)
				}
				binaryFiles[p.FileName()] = contents
			}
			if err == io.EOF {
				// We have finished parsing
				for key, value := range parsedEmail {
					fmt.Println("Key:", key, " Value:", value)
				}
				for key, value := range emailHeader {
					fmt.Println("eKey:", key, " eValue:", value)
				}
				for key, value := range binaryFiles {
					fmt.Println("bKey:", key, " bValue:", value)
				}
				for key, value := range rawEmail {
					fmt.Println("rKey:", key, " rValue:", value)
				}
				for key, value := range rawFiles {
					fmt.Println("rfKey:", key, " rfValue:", value)
				}
				w.WriteHeader(http.StatusOK)
				return
			}
			if err != nil {
				log.Fatal(err)
			}
			value, err := ioutil.ReadAll(p)
			if err != nil {
				log.Fatal(err)
			}
			header := p.Header.Get("Content-Disposition")

			if strings.Contains(header, "filename") != true {
				header = header[17 : len(header)-1]
				parsedEmail[header] = string(value)
			} else {
				header = header[11:]
				f := strings.Split(header, "=")
				parsedEmail[f[1][1:len(f[1])-11]] = f[2][1 : len(f[2])-1]
			}
			if header == "headers" {
				s := strings.Split(string(value), "\n")
				var a []string
				for _, v := range s {
					t := strings.Split(string(v), ": ")
					a = append(a, t...)
				}
				for i := 0; i < len(a)-1; i += 2 {
					emailHeader[a[i]] = a[i+1]
				}
			}
			delete(parsedEmail, "headers")

			// We have a raw message
			if header == "email" {
				boundary, body := getBoundary(string(value), "Content-Type: multipart/mixed; ")
				raw := multipart.NewReader(body, boundary)
				for {
					// TODO: function1
					q, err := raw.NextPart()
					if err == io.EOF {
						// We have finished parsing
						break
					}
					v, err := ioutil.ReadAll(q)
					if err != nil {
						log.Fatal(err)
					}
					header := q.Header.Get("Content-Type")
					//--end

					// Parse the headers
					if strings.Contains(header, "multipart/alternative") {
						boundary, body := getBoundary(string(value), "Content-Type: multipart/alternative; ")
						raw := multipart.NewReader(body, boundary)
						for {
							// TODO: function1
							q, err := raw.NextPart()
							if err == io.EOF {
								// We have finished parsing
								break
							}
							v, err := ioutil.ReadAll(q)
							if err != nil {
								log.Fatal(err)
							}
							header := q.Header.Get("Content-Type")
							//--end
							rawEmail[header] = string(v)
						}
					} else {
						// Get the attachments, base64 encoded
						rawFiles[header] = string(v)
					}
				}
			}
			delete(parsedEmail, "email")
		}
	}
}

func main() {
	conf := loadConfig("./conf.json")
	http.HandleFunc("/", indexHandler)
	http.HandleFunc(conf.Endpoint, inboundHandler)
	http.ListenAndServe(conf.Port, nil)
}
<commit_msg>Inbound Parse data parser complete<commit_after>package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime"
	"mime/multipart"
	"net/http"
	"strings"
)

type configuration struct {
	Endpoint string `json:"endpoint"`
	Port     string `json:"port"`
}

func loadConfig(path string) configuration {
	file, err := ioutil.ReadFile(path)
	if err != nil {
		log.Fatal("Config File Missing. ", err)
	}
	var conf configuration
	err = json.Unmarshal(file, &conf)
	if err != nil {
		log.Fatal("Config Parse Error: ", err)
	}
	return conf
}

func indexHandler(response http.ResponseWriter, request *http.Request) {
	fmt.Fprintf(response, "%s", "Hello World")
}

func getBoundary(value string, contentType string) (string, *strings.Reader) {
	body := strings.NewReader(value)
	bodySplit := strings.Split(string(value), contentType)
	scanner := bufio.NewScanner(strings.NewReader(bodySplit[1]))
	var lines []string
	for scanner.Scan() {
		lines = append(lines, scanner.Text())
		break
	}
	boundary := lines[0][9:]
	return boundary, body
}

func inboundHandler(response http.ResponseWriter, request *http.Request) {
	mediaType, params, err := mime.ParseMediaType(request.Header.Get("Content-Type"))
	if err != nil {
		log.Fatal(err)
	}
	if strings.HasPrefix(mediaType, "multipart/") {
		mr := multipart.NewReader(request.Body, params["boundary"])
		parsedEmail := make(map[string]string)
		emailHeader := make(map[string]string)
		binaryFiles := make(map[string][]byte)
		parsedRawEmail := make(map[string]string)
		rawFiles := make(map[string]string)
		for {
			p, err := mr.NextPart()
			// We have found an attachment with binary data
			if err == nil && p.FileName() != "" {
				contents, err := ioutil.ReadAll(p)
				if err != nil {
					log.Fatal(err)
				}
				binaryFiles[p.FileName()] = contents
			}
			if err == io.EOF {
				// We have finished parsing, do something with the parsed data
				for key, value := range parsedEmail {
					fmt.Println("Key:", key, " Value:", value)
				}
				for key, value := range emailHeader {
					fmt.Println("eKey:", key, " eValue:", value)
				}
				for key, value := range binaryFiles {
					fmt.Println("bKey:", key, " bValue:", value)
				}
				for key, value := range parsedRawEmail {
					fmt.Println("rKey:", key, " rValue:", value)
				}
				for key, value := range rawFiles {
					fmt.Println("rfKey:", key, " rfValue:", value)
				}
				// SendGrid needs a 200 OK response to stop POSTing
				response.WriteHeader(http.StatusOK)
				return
			}
			if err != nil {
				log.Fatal(err)
			}
			value, err := ioutil.ReadAll(p)
			if err != nil {
				log.Fatal(err)
			}
			header := p.Header.Get("Content-Disposition")
			if strings.Contains(header, "filename") != true {
				header = header[17 : len(header)-1]
				parsedEmail[header] = string(value)
			} else {
				header = header[11:]
				f := strings.Split(header, "=")
				parsedEmail[f[1][1:len(f[1])-11]] = f[2][1 : len(f[2])-1]
			}
			if header == "headers" {
				s := strings.Split(string(value), "\n")
				var a []string
				for _, v := range s {
					t := strings.Split(string(v), ": ")
					a = append(a, t...)
				}
				for i := 0; i < len(a)-1; i += 2 {
					emailHeader[a[i]] = a[i+1]
				}
			}
			// Since we have parsed the headers, we can delete the original
			delete(parsedEmail, "headers")

			// We have a raw message
			if header == "email" {
				boundary, body := getBoundary(string(value), "Content-Type: multipart/mixed; ")
				raw := multipart.NewReader(body, boundary)
				for {
					next, err := raw.NextPart()
					if err == io.EOF {
						// We have finished parsing
						break
					}
					value, err := ioutil.ReadAll(next)
					if err != nil {
						log.Fatal(err)
					}
					header := next.Header.Get("Content-Type")

					// Parse the headers
					if strings.Contains(header, "multipart/alternative") {
						boundary, body := getBoundary(string(value), "Content-Type: multipart/alternative; ")
						raw := multipart.NewReader(body, boundary)
						for {
							next, err := raw.NextPart()
							if err == io.EOF {
								// We have finished parsing
								break
							}
							value, err := ioutil.ReadAll(next)
							if err != nil {
								log.Fatal(err)
							}
							header := next.Header.Get("Content-Type")
							parsedRawEmail[header] = string(value)
						}
					} else {
						// It's a base64 encoded attachment
						rawFiles[header] = string(value)
					}
				}
			}
			// Since we've parsed this header, we can delete the original
			delete(parsedEmail, "email")
		}
	}
}

func main() {
	conf := loadConfig("./conf.json")
	http.HandleFunc("/", indexHandler)
	http.HandleFunc(conf.Endpoint, inboundHandler)
	http.ListenAndServe(conf.Port, nil)
}
<|endoftext|>