<commit_before>package main

import (
  "log"
  "os"
  "path/filepath"
  "time"
)

type ProspectorResume struct {
  files  map[string]*FileState
  resave  chan *FileState
}

type ProspectorInfo struct {
  fileinfo  os.FileInfo /* the file info */
  harvester chan int64 /* the harvester will send an event with its offset when it closes */
  last_seen uint32 /* int number of the last iterations in which we saw this file */
}

type Prospector struct {
  FileConfig FileConfig
  fileinfo   map[string]ProspectorInfo
  iteration  uint32
}

func (p *Prospector) Prospect(resumelist *ProspectorResume, registrar_chan chan []*FileEvent, output chan *FileEvent) {
  p.fileinfo = make(map[string]ProspectorInfo)

  // Handle any "-" (stdin) paths
  for i, path := range p.FileConfig.Paths {
    if path == "-" {
      // Offset and Initial never get used when path is "-"
      harvester := Harvester{Path: path, FileConfig: p.FileConfig}
      go harvester.Harvest(registrar_chan, output)

      // Remove it from the file list
      p.FileConfig.Paths = append(p.FileConfig.Paths[:i], p.FileConfig.Paths[i+1:]...)
    }
  }

  // Now let's do one quick scan to pick up new files - flag true so new files obey from-beginning
  for _, path := range p.FileConfig.Paths {
    p.scan(path, registrar_chan, output, resumelist)
  }

  // This signals we finished considering the previous state
  event := &FileState{
    Source: nil,
  }
  resumelist.resave <- event

  for {
    for _, path := range p.FileConfig.Paths {
      // Scan - flag false so new files always start at beginning
      p.scan(path, registrar_chan, output, nil)
    }

    // Defer next scan for a bit.
    time.Sleep(10 * time.Second) // Make this tunable

    // Clear out files that disappeared
    for file, lastinfo := range p.fileinfo {
      if lastinfo.last_seen < p.iteration {
        log.Printf("No longer tracking file that hasn't been seen for a while: %s\n", file)
        delete(p.fileinfo, file)
      }
    }

    p.iteration++ // Overflow is allowed
  }
} /* Prospect */

func (p *Prospector) scan(path string, registrar_chan chan []*FileEvent, output chan *FileEvent, resumelist *ProspectorResume) {
  //log.Printf("Prospecting %s\n", path)

  // Evaluate the path as a wildcards/shell glob
  matches, err := filepath.Glob(path)
  if err != nil {
    log.Printf("glob(%s) failed: %v\n", path, err)
    return
  }

  // To keep the old inode/dev reference if we see a file has renamed, in case it was also renamed prior
  missingfiles := make(map[string]os.FileInfo)

  // Check any matched files to see if we need to start a harvester
  for _, file := range matches {
    // Stat the file, following any symlinks.
    info, err := os.Stat(file)
    // TODO(sissel): check err
    if err != nil {
      log.Printf("stat(%s) failed: %s\n", file, err)
      continue
    }

    if info.IsDir() {
      log.Printf("Skipping directory: %s\n", file)
      continue
    }

    // Check the current info against p.fileinfo[file]
    lastinfo, is_known := p.fileinfo[file]
    newinfo := lastinfo

    // Conditions for starting a new harvester:
    // - file path hasn't been seen before
    // - the file's inode or device changed
    if !is_known {
      // Create a new prospector info with the stat info for comparison
      newinfo = ProspectorInfo{fileinfo: info, harvester: make(chan int64, 1), last_seen: p.iteration}

      if time.Since(info.ModTime()) > p.FileConfig.deadtime {
        // Call the calculator - it will process resume state if there is one
        offset, is_resuming := p.calculate_resume(file, info, resumelist)

        // Are we resuming a dead file? We have to resume even if dead so we catch any old updates to the file
        // This is safe as the harvester, once it hits the EOF and a timeout, will stop harvesting
        // Once we detect changes again we can resume another harvester again - this keeps number of go routines to a minimum
        if is_resuming {
          log.Printf("Resuming harvester on a previously harvested file: %s\n", file)
          harvester := &Harvester{Path: file, FileConfig: p.FileConfig, Offset: offset, FinishChan: newinfo.harvester}
          go harvester.Harvest(registrar_chan, output)
        } else {
          // Old file, skip it, but push offset of 0 so we obey from_beginning if this file changes and needs picking up
          log.Printf("Skipping file (older than dead time of %v): %s\n", p.FileConfig.deadtime, file)
          newinfo.harvester <- 0
        }
      } else if previous := is_file_renamed(file, info, p.fileinfo, missingfiles); previous != "" {
        // This file was simply renamed (known inode+dev) - link the same harvester channel as the old file
        log.Printf("File rename was detected: %s -> %s\n", previous, file)

        newinfo.harvester = p.fileinfo[previous].harvester
      } else {
        var initial bool = (resumelist != nil)

        // Call the calculator - it will process resume state if there is one
        offset, is_resuming := p.calculate_resume(file, info, resumelist)

        // Are we resuming a file or is this a completely new file?
        if is_resuming {
          log.Printf("Resuming harvester on a previously harvested file: %s\n", file)

          // By setting initial to false we ensure that offset is always obeyed, even on first scan, which is necessary for resume
          initial = false
        } else {
          log.Printf("Launching harvester on new file: %s\n", file)
        }

        // Launch the harvester - if initial is true it means ignore offset and choose end if this is first scan, and choose beginning if subsequence scan
        // This ensures we always pick up new logs from start - and only skip to end if we've just started up
        harvester := &Harvester{Path: file, FileConfig: p.FileConfig, Offset: offset, FinishChan: newinfo.harvester, Initial: initial}
        go harvester.Harvest(registrar_chan, output)
      }
    } else {
      // Update the fileinfo information used for future comparisons, and the last_seen counter
      newinfo.fileinfo = info
      newinfo.last_seen = p.iteration

      if !os.SameFile(lastinfo.fileinfo, info) {
        if previous := is_file_renamed(file, info, p.fileinfo, missingfiles); previous != "" {
          // This file was renamed from another file we know - link the same harvester channel as the old file
          log.Printf("File rename was detected: %s -> %s\n", previous, file)

          newinfo.harvester = p.fileinfo[previous].harvester
        } else {
          // File is not the same file we saw previously, it must have rotated and is a new file
          log.Printf("Launching harvester on rotated file: %s\n", file)

          // Forget about the previous harvester and let it continue on the old file - so start a new channel to use with the new harvester
          newinfo.harvester = make(chan int64, 1)

          // Start a harvester on the path
          harvester := &Harvester{Path: file, FileConfig: p.FileConfig, FinishChan: newinfo.harvester, Initial: (resumelist != nil)}
          go harvester.Harvest(registrar_chan, output)
        }

        // Keep the old file in missingfiles so we don't rescan it if it was renamed and we've not yet reached the new filename
        // We only need to keep it for the remainder of this iteration then we can assume it was deleted and forget about it
        missingfiles[file] = lastinfo.fileinfo
      } else if len(newinfo.harvester) != 0 && time.Since(info.ModTime()) < p.FileConfig.deadtime {
        // NOTE(driskell): If dead time is less than the prospector interval, this stops working
        // Resume harvesting of an old file we've stopped harvesting from
        log.Printf("Resuming harvester on an old file that was just modified: %s\n", file)

        // Start a harvester on the path; an old file was just modified and it doesn't have a harvester
        // The offset to continue from will be stored in the harvester channel - so take that to use and also clear the channel
        // Don't pass initial, we are resuming based on last offset
        harvester := &Harvester{Path: file, FileConfig: p.FileConfig, Offset: <-newinfo.harvester, FinishChan: newinfo.harvester}
        go harvester.Harvest(registrar_chan, output)
      }
    }

    // Track the stat data for this file for later comparison to check for
    // rotation/etc
    p.fileinfo[file] = newinfo
  } // for each file matched by the glob
}

func (p *Prospector) calculate_resume(file string, info os.FileInfo, resumelist *ProspectorResume) (int64, bool) {
  if resumelist != nil {
    last_state, is_found := resumelist.files[file]

    if is_found && is_file_same(file, info, last_state) {
      // We're resuming - throw the last state back downstream so we resave it
      // And return the offset - also force harvest in case the file is old and we're about to skip it
      resumelist.resave <- last_state
      return last_state.Offset, true
    }

    if previous := is_file_renamed_resumelist(file, info, resumelist.files); previous != "" {
      // File has rotated between shutdown and startup
      // We return last state downstream, with a modified event source with the new file name
      // And return the offset - also force harvest in case the file is old and we're about to skip it
      log.Printf("Detected rename of a previously harvested file: %s -> %s\n", previous, file)
      event := resumelist.files[previous]
      event.Source = &file
      resumelist.resave <- event
      return event.Offset, true
    }

    if is_found {
      log.Printf("Not resuming rotated file: %s\n", file)
    }
  }

  // New file so just start from an automatic position if initial scan, or the beginning if subsequent scans
  // The caller will know which to do
  return 0, false
}
<commit_msg>Tweak logging to remove any confusion<commit_after>package main

import (
  "log"
  "os"
  "path/filepath"
  "time"
)

type ProspectorResume struct {
  files  map[string]*FileState
  resave  chan *FileState
}

type ProspectorInfo struct {
  fileinfo  os.FileInfo /* the file info */
  harvester chan int64 /* the harvester will send an event with its offset when it closes */
  last_seen uint32 /* int number of the last iterations in which we saw this file */
}

type Prospector struct {
  FileConfig FileConfig
  fileinfo   map[string]ProspectorInfo
  iteration  uint32
}

func (p *Prospector) Prospect(resumelist *ProspectorResume, registrar_chan chan []*FileEvent, output chan *FileEvent) {
  p.fileinfo = make(map[string]ProspectorInfo)

  // Handle any "-" (stdin) paths
  for i, path := range p.FileConfig.Paths {
    if path == "-" {
      // Offset and Initial never get used when path is "-"
      harvester := Harvester{Path: path, FileConfig: p.FileConfig}
      go harvester.Harvest(registrar_chan, output)

      // Remove it from the file list
      p.FileConfig.Paths = append(p.FileConfig.Paths[:i], p.FileConfig.Paths[i+1:]...)
    }
  }

  // Now let's do one quick scan to pick up new files - flag true so new files obey from-beginning
  for _, path := range p.FileConfig.Paths {
    p.scan(path, registrar_chan, output, resumelist)
  }

  // This signals we finished considering the previous state
  event := &FileState{
    Source: nil,
  }
  resumelist.resave <- event

  for {
    for _, path := range p.FileConfig.Paths {
      // Scan - flag false so new files always start at beginning
      p.scan(path, registrar_chan, output, nil)
    }

    // Defer next scan for a bit.
    time.Sleep(10 * time.Second) // Make this tunable

    // Clear out files that disappeared
    for file, lastinfo := range p.fileinfo {
      if lastinfo.last_seen < p.iteration {
        delete(p.fileinfo, file)
      }
    }

    p.iteration++ // Overflow is allowed
  }
} /* Prospect */

func (p *Prospector) scan(path string, registrar_chan chan []*FileEvent, output chan *FileEvent, resumelist *ProspectorResume) {
  //log.Printf("Prospecting %s\n", path)

  // Evaluate the path as a wildcards/shell glob
  matches, err := filepath.Glob(path)
  if err != nil {
    log.Printf("glob(%s) failed: %v\n", path, err)
    return
  }

  // To keep the old inode/dev reference if we see a file has renamed, in case it was also renamed prior
  missingfiles := make(map[string]os.FileInfo)

  // Check any matched files to see if we need to start a harvester
  for _, file := range matches {
    // Stat the file, following any symlinks.
    info, err := os.Stat(file)
    // TODO(sissel): check err
    if err != nil {
      log.Printf("stat(%s) failed: %s\n", file, err)
      continue
    }

    if info.IsDir() {
      log.Printf("Skipping directory: %s\n", file)
      continue
    }

    // Check the current info against p.fileinfo[file]
    lastinfo, is_known := p.fileinfo[file]
    newinfo := lastinfo

    // Conditions for starting a new harvester:
    // - file path hasn't been seen before
    // - the file's inode or device changed
    if !is_known {
      // Create a new prospector info with the stat info for comparison
      newinfo = ProspectorInfo{fileinfo: info, harvester: make(chan int64, 1), last_seen: p.iteration}

      if time.Since(info.ModTime()) > p.FileConfig.deadtime {
        // Call the calculator - it will process resume state if there is one
        offset, is_resuming := p.calculate_resume(file, info, resumelist)

        // Are we resuming a dead file? We have to resume even if dead so we catch any old updates to the file
        // This is safe as the harvester, once it hits the EOF and a timeout, will stop harvesting
        // Once we detect changes again we can resume another harvester again - this keeps number of go routines to a minimum
        if is_resuming {
          log.Printf("Resuming harvester on a previously harvested file: %s\n", file)
          harvester := &Harvester{Path: file, FileConfig: p.FileConfig, Offset: offset, FinishChan: newinfo.harvester}
          go harvester.Harvest(registrar_chan, output)
        } else {
          // Old file, skip it, but push offset of 0 so we obey from_beginning if this file changes and needs picking up
          log.Printf("Skipping file (older than dead time of %v): %s\n", p.FileConfig.deadtime, file)
          newinfo.harvester <- 0
        }
      } else if previous := is_file_renamed(file, info, p.fileinfo, missingfiles); previous != "" {
        // This file was simply renamed (known inode+dev) - link the same harvester channel as the old file
        log.Printf("File rename was detected: %s -> %s\n", previous, file)

        newinfo.harvester = p.fileinfo[previous].harvester
      } else {
        var initial bool = (resumelist != nil)

        // Call the calculator - it will process resume state if there is one
        offset, is_resuming := p.calculate_resume(file, info, resumelist)

        // Are we resuming a file or is this a completely new file?
        if is_resuming {
          log.Printf("Resuming harvester on a previously harvested file: %s\n", file)

          // By setting initial to false we ensure that offset is always obeyed, even on first scan, which is necessary for resume
          initial = false
        } else {
          log.Printf("Launching harvester on new file: %s\n", file)
        }

        // Launch the harvester - if initial is true it means ignore offset and choose end if this is first scan, and choose beginning if subsequence scan
        // This ensures we always pick up new logs from start - and only skip to end if we've just started up
        harvester := &Harvester{Path: file, FileConfig: p.FileConfig, Offset: offset, FinishChan: newinfo.harvester, Initial: initial}
        go harvester.Harvest(registrar_chan, output)
      }
    } else {
      // Update the fileinfo information used for future comparisons, and the last_seen counter
      newinfo.fileinfo = info
      newinfo.last_seen = p.iteration

      if !os.SameFile(lastinfo.fileinfo, info) {
        if previous := is_file_renamed(file, info, p.fileinfo, missingfiles); previous != "" {
          // This file was renamed from another file we know - link the same harvester channel as the old file
          log.Printf("File rename was detected: %s -> %s\n", previous, file)

          newinfo.harvester = p.fileinfo[previous].harvester
        } else {
          // File is not the same file we saw previously, it must have rotated and is a new file
          log.Printf("Launching harvester on rotated file: %s\n", file)

          // Forget about the previous harvester and let it continue on the old file - so start a new channel to use with the new harvester
          newinfo.harvester = make(chan int64, 1)

          // Start a harvester on the path
          harvester := &Harvester{Path: file, FileConfig: p.FileConfig, FinishChan: newinfo.harvester, Initial: (resumelist != nil)}
          go harvester.Harvest(registrar_chan, output)
        }

        // Keep the old file in missingfiles so we don't rescan it if it was renamed and we've not yet reached the new filename
        // We only need to keep it for the remainder of this iteration then we can assume it was deleted and forget about it
        missingfiles[file] = lastinfo.fileinfo
      } else if len(newinfo.harvester) != 0 && time.Since(info.ModTime()) < p.FileConfig.deadtime {
        // NOTE(driskell): If dead time is less than the prospector interval, this stops working
        // Resume harvesting of an old file we've stopped harvesting from
        log.Printf("Resuming harvester on an old file that was just modified: %s\n", file)

        // Start a harvester on the path; an old file was just modified and it doesn't have a harvester
        // The offset to continue from will be stored in the harvester channel - so take that to use and also clear the channel
        // Don't pass initial, we are resuming based on last offset
        harvester := &Harvester{Path: file, FileConfig: p.FileConfig, Offset: <-newinfo.harvester, FinishChan: newinfo.harvester}
        go harvester.Harvest(registrar_chan, output)
      }
    }

    // Track the stat data for this file for later comparison to check for
    // rotation/etc
    p.fileinfo[file] = newinfo
  } // for each file matched by the glob
}

func (p *Prospector) calculate_resume(file string, info os.FileInfo, resumelist *ProspectorResume) (int64, bool) {
  if resumelist != nil {
    last_state, is_found := resumelist.files[file]

    if is_found && is_file_same(file, info, last_state) {
      // We're resuming - throw the last state back downstream so we resave it
      // And return the offset - also force harvest in case the file is old and we're about to skip it
      resumelist.resave <- last_state
      return last_state.Offset, true
    }

    if previous := is_file_renamed_resumelist(file, info, resumelist.files); previous != "" {
      // File has rotated between shutdown and startup
      // We return last state downstream, with a modified event source with the new file name
      // And return the offset - also force harvest in case the file is old and we're about to skip it
      log.Printf("Detected rename of a previously harvested file: %s -> %s\n", previous, file)
      event := resumelist.files[previous]
      event.Source = &file
      resumelist.resave <- event
      return event.Offset, true
    }

    if is_found {
      log.Printf("Not resuming rotated file: %s\n", file)
    }
  }

  // New file so just start from an automatic position if initial scan, or the beginning if subsequent scans
  // The caller will know which to do
  return 0, false
}
<|endoftext|>


<commit_before>package bitsgo

import (
	"bytes"
	"crypto/sha1"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/petergtz/bitsgo/logger"
)

type MetricsService interface {
	SendTimingMetric(name string, duration time.Duration)
}

type StateForbiddenError struct {
	error
}

func NewStateForbiddenError() *StateForbiddenError {
	return &StateForbiddenError{fmt.Errorf("StateForbiddenError")}
}

type Updater interface {
	NotifyProcessingUpload(guid string) error
	NotifyUploadSucceeded(guid string, sha1 string, sha2 string) error
	NotifyUploadFailed(guid string, e error) error
}

type NullUpdater struct{}

func (u *NullUpdater) NotifyProcessingUpload(guid string) error                          { return nil }
func (u *NullUpdater) NotifyUploadSucceeded(guid string, sha1 string, sha2 string) error { return nil }
func (u *NullUpdater) NotifyUploadFailed(guid string, e error) error                     { return nil }

type ResourceHandler struct {
	blobstore        Blobstore
	resourceType     string
	metricsService   MetricsService
	maxBodySizeLimit uint64
	updater          Updater
}

type responseBody struct {
	Guid      string    `json:"guid"`
	State     string    `json:"state"`
	Type      string    `json:"type"`
	CreatedAt time.Time `json:"created_at"`
}

func NewResourceHandler(blobstore Blobstore, resourceType string, metricsService MetricsService, maxBodySizeLimit uint64) *ResourceHandler {
	return &ResourceHandler{
		blobstore:        blobstore,
		resourceType:     resourceType,
		metricsService:   metricsService,
		maxBodySizeLimit: maxBodySizeLimit,
		updater:          &NullUpdater{},
	}
}

func NewResourceHandlerWithUpdater(blobstore Blobstore, updater Updater, resourceType string, metricsService MetricsService, maxBodySizeLimit uint64) *ResourceHandler {
	return &ResourceHandler{
		blobstore:        blobstore,
		resourceType:     resourceType,
		metricsService:   metricsService,
		maxBodySizeLimit: maxBodySizeLimit,
		updater:          updater,
	}
}

// TODO: instead of params, we could use `identifier string` to make the interface more type-safe.
//       Here and in the other methods.
func (handler *ResourceHandler) AddOrReplaceWithDigestInHeader(responseWriter http.ResponseWriter, request *http.Request, params map[string]string) {
	if !HandleBodySizeLimits(responseWriter, request, handler.maxBodySizeLimit) {
		return
	}
	logger.From(request).Debugw("Octet-Stream")

	digest := request.Header.Get("Digest")
	if digest == "" {
		badRequest(responseWriter, "No Digest header")
		return
	}
	parts := strings.Split(digest, "=")
	if len(parts) != 2 {
		badRequest(responseWriter, "Digest must have format sha256=value, but is "+digest)
		return
	}
	alg, value := strings.ToLower(parts[0]), parts[1]
	if alg != "sha256" {
		badRequest(responseWriter, "Digest must have format sha256=value, but is "+digest)
		return
	}
	if value == "" {
		badRequest(responseWriter, "Digest must have format sha256=value. Value cannot be empty")
		return
	}

	// TODO this can cause an out of memory panic. Should be smart about writing big files to disk instead.
	content, e := ioutil.ReadAll(request.Body)
	if e != nil {
		internalServerError(responseWriter, e)
		return
	}
	e = handler.blobstore.Put(params["identifier"]+"/"+value, bytes.NewReader(content))
	// TODO use Clock instead:
	writeResponseBasedOn("", e, responseWriter, http.StatusCreated, nil, &responseBody{Guid: params["identifier"], State: "READY", Type: "bits", CreatedAt: time.Now()}, "")
}

// TODO: instead of params, we could use `identifier string` to make the interface more type-safe.
//       Here and in the other methods.
func (handler *ResourceHandler) AddOrReplace(responseWriter http.ResponseWriter, request *http.Request, params map[string]string) {
	if !HandleBodySizeLimits(responseWriter, request, handler.maxBodySizeLimit) {
		return
	}
	if strings.Contains(request.Header.Get("Content-Type"), "multipart/form-data") {
		logger.From(request).Debugw("Multipart upload")
		handler.uploadMultipart(responseWriter, request, params["identifier"])
	} else {
		logger.From(request).Debugw("Copy source guid")
		handler.copySourceGuid(responseWriter, request, params["identifier"])
	}
}

func (handler *ResourceHandler) uploadMultipart(responseWriter http.ResponseWriter, request *http.Request, identifier string) {
	file, _, e := request.FormFile(handler.resourceType)
	if e == http.ErrMissingFile {
		file, _, e = request.FormFile("bits")
	}
	if e == http.ErrMissingFile {
		badRequest(responseWriter, "Could not retrieve form parameter '%s' or 'bits", handler.resourceType)
		return
	}
	if e != nil {
		internalServerError(responseWriter, e)
		return
	}
	defer file.Close()

	e = handler.updater.NotifyProcessingUpload(identifier)
	if handleNotificationError(e, responseWriter) {
		return
	}
	buffer, e := ioutil.ReadAll(file)
	if e != nil {
		internalServerError(responseWriter, e)
		return
	}

	startTime := time.Now()
	e = handler.blobstore.Put(identifier, bytes.NewReader(buffer))
	handler.metricsService.SendTimingMetric(handler.resourceType+"-cp_to_blobstore-time", time.Since(startTime))

	if e != nil {
		logger.From(request).Infow("Failed to upload to blobstore.", "error", e)
		notifyErr := handler.updater.NotifyUploadFailed(identifier, e)
		if notifyErr != nil {
			logger.From(request).Errorw("Failed to notifying CC about failed upload.", "error", notifyErr)
		}
		if _, noSpaceLeft := e.(*NoSpaceLeftError); noSpaceLeft {
			responseWriter.WriteHeader(http.StatusInsufficientStorage)
			return
		}
		internalServerError(responseWriter, e)
		return
	}
	sha1, sha256 := sha1.Sum(buffer), sha256.Sum256(buffer)
	e = handler.updater.NotifyUploadSucceeded(identifier, hex.EncodeToString(sha1[:]), hex.EncodeToString(sha256[:]))
	if e != nil {
		internalServerError(responseWriter, e)
		return
	}

	writeResponseBasedOn("", nil, responseWriter, http.StatusCreated, nil, &responseBody{Guid: identifier, State: "READY", Type: "bits", CreatedAt: time.Now()}, "")
}

func handleNotificationError(e error, responseWriter http.ResponseWriter) (wasError bool) {
	switch e.(type) {
	case *StateForbiddenError:
		responseWriter.WriteHeader(http.StatusBadRequest)
		fprintDescriptionAndCodeAsJSON(responseWriter, "290008", "Cannot update an existing package.")
		return true
	case *NotFoundError:
		responseWriter.WriteHeader(http.StatusNotFound)
		fprintDescriptionAndCodeAsJSON(responseWriter, "10010", e.Error())
		return true
	case error:
		internalServerError(responseWriter, e)
		return true
	}
	return false
}

func (handler *ResourceHandler) copySourceGuid(responseWriter http.ResponseWriter, request *http.Request, identifier string) {
	sourceGuid := sourceGuidFrom(request.Body, responseWriter)
	if sourceGuid == "" {
		return // response is already handled in sourceGuidFrom
	}
	e := handler.blobstore.Copy(sourceGuid, identifier)
	// TODO use Clock instead:
	writeResponseBasedOn("", e, responseWriter, http.StatusCreated, nil, &responseBody{Guid: identifier, State: "READY", Type: "bits", CreatedAt: time.Now()}, "")
}

func sourceGuidFrom(body io.ReadCloser, responseWriter http.ResponseWriter) string {
	content, e := ioutil.ReadAll(body)
	if e != nil {
		internalServerError(responseWriter, e)
		return ""
	}
	var payload struct {
		SourceGuid string `json:"source_guid"`
	}
	e = json.Unmarshal(content, &payload)
	if e != nil {
		badRequest(responseWriter, "Body must be valid JSON when request is not multipart/form-data. %+v", e)
		return ""
	}
	return payload.SourceGuid
}

func (handler *ResourceHandler) HeadOrRedirectAsGet(responseWriter http.ResponseWriter, request *http.Request, params map[string]string) {
	redirectLocation, e := handler.blobstore.HeadOrRedirectAsGet(params["identifier"])
	writeResponseBasedOn(redirectLocation, e, responseWriter, http.StatusOK, nil, nil, "")
}

func (handler *ResourceHandler) Get(responseWriter http.ResponseWriter, request *http.Request, params map[string]string) {
	body, redirectLocation, e := handler.blobstore.GetOrRedirect(params["identifier"])
	writeResponseBasedOn(redirectLocation, e, responseWriter, http.StatusOK, body, nil, request.Header.Get("If-None-Modify"))
}

func (handler *ResourceHandler) Delete(responseWriter http.ResponseWriter, request *http.Request, params map[string]string) {
	// TODO nothing should be S3 specific here
	// this check is needed, because S3 does not return a NotFound on a Delete request:
	exists, e := handler.blobstore.Exists(params["identifier"])
	if e != nil {
		internalServerError(responseWriter, e)
		return
	}
	if !exists {
		responseWriter.WriteHeader(http.StatusNotFound)
		return
	}
	e = handler.blobstore.Delete(params["identifier"])
	writeResponseBasedOn("", e, responseWriter, http.StatusNoContent, nil, nil, "")
}

func (handler *ResourceHandler) DeleteDir(responseWriter http.ResponseWriter, request *http.Request, params map[string]string) {
	e := handler.blobstore.DeleteDir(params["identifier"])
	switch e.(type) {
	case *NotFoundError:
		responseWriter.WriteHeader(http.StatusNoContent)
		return
	}
	writeResponseBasedOn("", e, responseWriter, http.StatusNoContent, nil, nil, "")
}

var emptyReader = ioutil.NopCloser(bytes.NewReader(nil))

func writeResponseBasedOn(redirectLocation string, e error, responseWriter http.ResponseWriter, statusCode int, body io.ReadCloser, jsonBody *responseBody, ifNoneModify string) {
	switch e.(type) {
	case *NotFoundError:
		http.NotFound(responseWriter, nil)
		return
	case *NoSpaceLeftError:
		http.Error(responseWriter, "Request Entity Too Large", http.StatusInsufficientStorage)
		return
	case error:
		internalServerError(responseWriter, e)
		return
	}
	if redirectLocation != "" {
		http.Redirect(responseWriter, nil, redirectLocation, http.StatusFound)
		return
	}
	if body != nil {
		defer body.Close()
		var buffer bytes.Buffer
		sha := sha1.New()
		_, e := io.Copy(io.MultiWriter(&buffer, sha), body)
		if e != nil {
			internalServerError(responseWriter, e)
			return
		}
		eTag := hex.EncodeToString(sha.Sum(nil))
		logger.Log.Debugw("Cache check", "if-none-modify", ifNoneModify, "etag", eTag)
		responseWriter.Header().Set("ETag", eTag)
		if ifNoneModify == eTag {
			responseWriter.WriteHeader(http.StatusNotModified)
			return
		}
		responseWriter.WriteHeader(statusCode)
		io.Copy(responseWriter, &buffer)
		return
	}
	if jsonBody != nil {
		respBody, marshallingErr := json.Marshal(jsonBody)
		if marshallingErr != nil {
			internalServerError(responseWriter, marshallingErr)
			return
		}
		responseWriter.WriteHeader(statusCode)
		responseWriter.Write(respBody)
		return
	}
	responseWriter.WriteHeader(statusCode)
}

func internalServerError(responseWriter http.ResponseWriter, e error) {
	logger.Log.Errorw("Internal Server Error.", "error", fmt.Sprintf("%+v", e))
	http.Error(responseWriter, "Internal Server Error.", http.StatusInternalServerError)
}

func badRequest(responseWriter http.ResponseWriter, message string, args ...interface{}) {
	responseBody := fmt.Sprintf(message, args...)
	logger.Log.Infow("Bad request", "body", responseBody)
	responseWriter.WriteHeader(http.StatusBadRequest)
	fprintDescriptionAndCodeAsJSON(responseWriter, "290003", message, args...)
}

func fprintDescriptionAndCodeAsJSON(responseWriter http.ResponseWriter, code string, description string, a ...interface{}) {
	fmt.Fprintf(responseWriter, `{"description":"%v","code":%v}`, fmt.Sprintf(description, a...), code)
}
<commit_msg>Handle StatusInsufficientStorage correctly for PUT requests<commit_after>package bitsgo

import (
	"bytes"
	"crypto/sha1"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/petergtz/bitsgo/logger"
)

type MetricsService interface {
	SendTimingMetric(name string, duration time.Duration)
}

type StateForbiddenError struct {
	error
}

func NewStateForbiddenError() *StateForbiddenError {
	return &StateForbiddenError{fmt.Errorf("StateForbiddenError")}
}

type Updater interface {
	NotifyProcessingUpload(guid string) error
	NotifyUploadSucceeded(guid string, sha1 string, sha2 string) error
	NotifyUploadFailed(guid string, e error) error
}

type NullUpdater struct{}

func (u *NullUpdater) NotifyProcessingUpload(guid string) error                          { return nil }
func (u *NullUpdater) NotifyUploadSucceeded(guid string, sha1 string, sha2 string) error { return nil }
func (u *NullUpdater) NotifyUploadFailed(guid string, e error) error                     { return nil }

type ResourceHandler struct {
	blobstore        Blobstore
	resourceType     string
	metricsService   MetricsService
	maxBodySizeLimit uint64
	updater          Updater
}

type responseBody struct {
	Guid      string    `json:"guid"`
	State     string    `json:"state"`
	Type      string    `json:"type"`
	CreatedAt time.Time `json:"created_at"`
}

func NewResourceHandler(blobstore Blobstore, resourceType string, metricsService MetricsService, maxBodySizeLimit uint64) *ResourceHandler {
	return &ResourceHandler{
		blobstore:        blobstore,
		resourceType:     resourceType,
		metricsService:   metricsService,
		maxBodySizeLimit: maxBodySizeLimit,
		updater:          &NullUpdater{},
	}
}

func NewResourceHandlerWithUpdater(blobstore Blobstore, updater Updater, resourceType string, metricsService MetricsService, maxBodySizeLimit uint64) *ResourceHandler {
	return &ResourceHandler{
		blobstore:        blobstore,
		resourceType:     resourceType,
		metricsService:   metricsService,
		maxBodySizeLimit: maxBodySizeLimit,
		updater:          updater,
	}
}

// TODO: instead of params, we could use `identifier string` to make the interface more type-safe.
//       Here and in the other methods.
func (handler *ResourceHandler) AddOrReplaceWithDigestInHeader(responseWriter http.ResponseWriter, request *http.Request, params map[string]string) {
	if !HandleBodySizeLimits(responseWriter, request, handler.maxBodySizeLimit) {
		return
	}
	logger.From(request).Debugw("Octet-Stream")

	digest := request.Header.Get("Digest")
	if digest == "" {
		badRequest(responseWriter, "No Digest header")
		return
	}
	parts := strings.Split(digest, "=")
	if len(parts) != 2 {
		badRequest(responseWriter, "Digest must have format sha256=value, but is "+digest)
		return
	}
	alg, value := strings.ToLower(parts[0]), parts[1]
	if alg != "sha256" {
		badRequest(responseWriter, "Digest must have format sha256=value, but is "+digest)
		return
	}
	if value == "" {
		badRequest(responseWriter, "Digest must have format sha256=value. Value cannot be empty")
		return
	}

	// TODO this can cause an out of memory panic. Should be smart about writing big files to disk instead.
	content, e := ioutil.ReadAll(request.Body)
	if e != nil {
		internalServerError(responseWriter, e)
		return
	}
	e = handler.blobstore.Put(params["identifier"]+"/"+value, bytes.NewReader(content))
	// TODO use Clock instead:
	writeResponseBasedOn("", e, responseWriter, http.StatusCreated, nil, &responseBody{Guid: params["identifier"], State: "READY", Type: "bits", CreatedAt: time.Now()}, "")
}

// TODO: instead of params, we could use `identifier string` to make the interface more type-safe.
//       Here and in the other methods.
func (handler *ResourceHandler) AddOrReplace(responseWriter http.ResponseWriter, request *http.Request, params map[string]string) {
	if !HandleBodySizeLimits(responseWriter, request, handler.maxBodySizeLimit) {
		return
	}
	if strings.Contains(request.Header.Get("Content-Type"), "multipart/form-data") {
		logger.From(request).Debugw("Multipart upload")
		handler.uploadMultipart(responseWriter, request, params["identifier"])
	} else {
		logger.From(request).Debugw("Copy source guid")
		handler.copySourceGuid(responseWriter, request, params["identifier"])
	}
}

func (handler *ResourceHandler) uploadMultipart(responseWriter http.ResponseWriter, request *http.Request, identifier string) {
	file, _, e := request.FormFile(handler.resourceType)
	if e == http.ErrMissingFile {
		file, _, e = request.FormFile("bits")
	}
	if e == http.ErrMissingFile {
		badRequest(responseWriter, "Could not retrieve form parameter '%s' or 'bits", handler.resourceType)
		return
	}
	if e != nil {
		internalServerError(responseWriter, e)
		return
	}
	defer file.Close()

	e = handler.updater.NotifyProcessingUpload(identifier)
	if handleNotificationError(e, responseWriter) {
		return
	}
	buffer, e := ioutil.ReadAll(file)
	if e != nil {
		internalServerError(responseWriter, e)
		return
	}

	startTime := time.Now()
	e = handler.blobstore.Put(identifier, bytes.NewReader(buffer))
	handler.metricsService.SendTimingMetric(handler.resourceType+"-cp_to_blobstore-time", time.Since(startTime))

	if e != nil {
		logger.From(request).Infow("Failed to upload to blobstore.", "error", e)
		notifyErr := handler.updater.NotifyUploadFailed(identifier, e)
		if notifyErr != nil {
			logger.From(request).Errorw("Failed to notifying CC about failed upload.", "error", notifyErr)
		}
		if _, noSpaceLeft := e.(*NoSpaceLeftError); noSpaceLeft {
			http.Error(responseWriter, "Request Entity Too Large", http.StatusInsufficientStorage)
			return
		}
		internalServerError(responseWriter, e)
		return
	}
	sha1, sha256 := sha1.Sum(buffer), sha256.Sum256(buffer)
	e = handler.updater.NotifyUploadSucceeded(identifier, hex.EncodeToString(sha1[:]), hex.EncodeToString(sha256[:]))
	if e != nil {
		internalServerError(responseWriter, e)
		return
	}

	writeResponseBasedOn("", nil, responseWriter, http.StatusCreated, nil, &responseBody{Guid: identifier, State: "READY", Type: "bits", CreatedAt: time.Now()}, "")
}

func handleNotificationError(e error, responseWriter http.ResponseWriter) (wasError bool) {
	switch e.(type) {
	case *StateForbiddenError:
		responseWriter.WriteHeader(http.StatusBadRequest)
		fprintDescriptionAndCodeAsJSON(responseWriter, "290008", "Cannot update an existing package.")
		return true
	case *NotFoundError:
		responseWriter.WriteHeader(http.StatusNotFound)
		fprintDescriptionAndCodeAsJSON(responseWriter, "10010", e.Error())
		return true
	case error:
		internalServerError(responseWriter, e)
		return true
	}
	return false
}

func (handler *ResourceHandler) copySourceGuid(responseWriter http.ResponseWriter, request *http.Request, identifier string) {
	sourceGuid := sourceGuidFrom(request.Body, responseWriter)
	if sourceGuid == "" {
		return // response is already handled in sourceGuidFrom
	}
	e := handler.blobstore.Copy(sourceGuid, identifier)
	// TODO use Clock instead:
	writeResponseBasedOn("", e, responseWriter, http.StatusCreated, nil, &responseBody{Guid: identifier, State: "READY", Type: "bits", CreatedAt: time.Now()}, "")
}

func sourceGuidFrom(body io.ReadCloser, responseWriter http.ResponseWriter) string {
	content, e := ioutil.ReadAll(body)
	if e != nil {
		internalServerError(responseWriter, e)
		return ""
	}
	var payload struct {
		SourceGuid string `json:"source_guid"`
	}
	e = json.Unmarshal(content, &payload)
	if e != nil {
		badRequest(responseWriter, "Body must be valid JSON when request is not multipart/form-data. %+v", e)
		return ""
	}
	return payload.SourceGuid
}

func (handler *ResourceHandler) HeadOrRedirectAsGet(responseWriter http.ResponseWriter, request *http.Request, params map[string]string) {
	redirectLocation, e := handler.blobstore.HeadOrRedirectAsGet(params["identifier"])
	writeResponseBasedOn(redirectLocation, e, responseWriter, http.StatusOK, nil, nil, "")
}

func (handler *ResourceHandler) Get(responseWriter http.ResponseWriter, request *http.Request, params map[string]string) {
	body, redirectLocation, e := handler.blobstore.GetOrRedirect(params["identifier"])
	writeResponseBasedOn(redirectLocation, e, responseWriter, http.StatusOK, body, nil, request.Header.Get("If-None-Modify"))
}

func (handler *ResourceHandler) Delete(responseWriter http.ResponseWriter, request *http.Request, params map[string]string) {
	// TODO nothing should be S3 specific here
	// this check is needed, because S3 does not return a NotFound on a Delete request:
	exists, e := handler.blobstore.Exists(params["identifier"])
	if e != nil {
		internalServerError(responseWriter, e)
		return
	}
	if !exists {
		responseWriter.WriteHeader(http.StatusNotFound)
		return
	}
	e = handler.blobstore.Delete(params["identifier"])
	writeResponseBasedOn("", e, responseWriter, http.StatusNoContent, nil, nil, "")
}

func (handler *ResourceHandler) DeleteDir(responseWriter http.ResponseWriter, request *http.Request, params map[string]string) {
	e := handler.blobstore.DeleteDir(params["identifier"])
	switch e.(type) {
	case *NotFoundError:
		responseWriter.WriteHeader(http.StatusNoContent)
		return
	}
	writeResponseBasedOn("", e, responseWriter, http.StatusNoContent, nil, nil, "")
}

var emptyReader = ioutil.NopCloser(bytes.NewReader(nil))

func writeResponseBasedOn(redirectLocation string, e error, responseWriter http.ResponseWriter, statusCode int, body io.ReadCloser, jsonBody *responseBody, ifNoneModify string) {
	switch e.(type) {
	case *NotFoundError:
		http.NotFound(responseWriter, nil)
		return
	case *NoSpaceLeftError:
		http.Error(responseWriter, "Request Entity Too Large", http.StatusInsufficientStorage)
		return
	case error:
		internalServerError(responseWriter, e)
		return
	}
	if redirectLocation != "" {
		http.Redirect(responseWriter, nil, redirectLocation, http.StatusFound)
		return
	}
	if body != nil {
		defer body.Close()
		var buffer bytes.Buffer
		sha := sha1.New()
		_, e := io.Copy(io.MultiWriter(&buffer, sha), body)
		if e != nil {
			internalServerError(responseWriter, e)
			return
		}
		eTag := hex.EncodeToString(sha.Sum(nil))
		logger.Log.Debugw("Cache check", "if-none-modify", ifNoneModify, "etag", eTag)
		responseWriter.Header().Set("ETag", eTag)
		if ifNoneModify == eTag {
			responseWriter.WriteHeader(http.StatusNotModified)
			return
		}
		responseWriter.WriteHeader(statusCode)
		io.Copy(responseWriter, &buffer)
		return
	}
	if jsonBody != nil {
		respBody, marshallingErr := json.Marshal(jsonBody)
		if marshallingErr != nil {
			internalServerError(responseWriter, marshallingErr)
			return
		}
		responseWriter.WriteHeader(statusCode)
		responseWriter.Write(respBody)
		return
	}
	responseWriter.WriteHeader(statusCode)
}

func internalServerError(responseWriter http.ResponseWriter, e error) {
	logger.Log.Errorw("Internal Server Error.", "error", fmt.Sprintf("%+v", e))
	http.Error(responseWriter, "Internal Server Error.", http.StatusInternalServerError)
}

func badRequest(responseWriter http.ResponseWriter, message string, args ...interface{}) {
	responseBody := fmt.Sprintf(message, args...)
	logger.Log.Infow("Bad request", "body", responseBody)
	responseWriter.WriteHeader(http.StatusBadRequest)
	fprintDescriptionAndCodeAsJSON(responseWriter, "290003", message, args...)
}

func fprintDescriptionAndCodeAsJSON(responseWriter http.ResponseWriter, code string, description string, a ...interface{}) {
	fmt.Fprintf(responseWriter, `{"description":"%v","code":%v}`, fmt.Sprintf(description, a...), code)
}
<|endoftext|>


<commit_before>package rabbitsmpp

import (
	"encoding/json"
	"errors"

	"github.com/streadway/amqp"
	"github.com/veoo/go-smpp/smpp/pdu"
)

type Job struct {
	PDU        pdu.Body          `json:"pdu"`
	Attributes map[string]string `json:"attributes"`
	delivery   *amqp.Delivery    `json:"-"`
}

// Since pdu.Body is an interface, we need an special method to pass a concrete type
func (j *Job) UnmarshalJSON(b []byte) error {
	s := &struct {
		PDU        *pdu.Codec        `json:"pdu"`
		Attributes map[string]string `json:"attributes"`
	}{}

	err := json.Unmarshal(b, &s)
	if err != nil {
		return err
	}
	j.PDU = s.PDU
	j.Attributes = s.Attributes
	return nil
}

func (j *Job) Ack(multiple bool) error {
	if j.delivery == nil {
		return errors.New("Delivery not set in Job")
	}
	return j.delivery.Ack(multiple)
}

func (j *Job) Nack(multiple, requeue bool) error {
	if j.delivery == nil {
		return errors.New("Delivery not set in Job")
	}
	return j.delivery.Nack(multiple, requeue)
}

func (j *Job) Reject(multiple bool) error {
	if j.delivery == nil {
		return errors.New("Delivery not set in Job")
	}
	return j.delivery.Reject(multiple)
}
<commit_msg>Modify Job to contain an array of PDUs<commit_after>package rabbitsmpp

import (
	"encoding/json"
	"errors"

	"github.com/streadway/amqp"
	"github.com/veoo/go-smpp/smpp/pdu"
)

type Job struct {
	PDUs       []pdu.Body        `json:"pdus"`
	Attributes map[string]string `json:"attributes"`
	delivery   *amqp.Delivery    `json:"-"`
}

// Since pdu.Body is an interface, we need an special method to pass a concrete type
func (j *Job) UnmarshalJSON(b []byte) error {
	s := &struct {
		PDUs       []*pdu.Codec      `json:"pdus"`
		Attributes map[string]string `json:"attributes"`
	}{}

	err := json.Unmarshal(b, &s)
	if err != nil {
		return err
	}
	j.PDUs = make([]pdu.Body, len(s.PDUs))
	for i := range j.PDUs {
		j.PDUs[i] = s.PDUs[i]
	}
	j.Attributes = s.Attributes
	return nil
}

func (j *Job) Ack(multiple bool) error {
	if j.delivery == nil {
		return errors.New("Delivery not set in Job")
	}
	return j.delivery.Ack(multiple)
}

func (j *Job) Nack(multiple, requeue bool) error {
	if j.delivery == nil {
		return errors.New("Delivery not set in Job")
	}
	return j.delivery.Nack(multiple, requeue)
}

func (j *Job) Reject(multiple bool) error {
	if j.delivery == nil {
		return errors.New("Delivery not set in Job")
	}
	return j.delivery.Reject(multiple)
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"os"
	"path"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/apimachinery/registered"
	client "k8s.io/kubernetes/pkg/client/unversioned"

	. "github.com/onsi/ginkgo"
)

//TODO : Consolidate this code with the code for emptyDir.
//This will require some smart.
var _ = Describe("hostPath", func() {
	framework := NewDefaultFramework("hostpath")
	var c *client.Client
	var namespace *api.Namespace

	BeforeEach(func() {
		c = framework.Client
		namespace = framework.Namespace

		//cleanup before running the test.
		_ = os.Remove("/tmp/test-file")
	})

	It("should give a volume the correct mode [Conformance]", func() {
		volumePath := "/test-volume"
		source := &api.HostPathVolumeSource{
			Path: "/tmp",
		}
		pod := testPodWithHostVol(volumePath, source)

		pod.Spec.Containers[0].Args = []string{
			fmt.Sprintf("--fs_type=%v", volumePath),
			fmt.Sprintf("--file_mode=%v", volumePath),
		}
		testContainerOutput("hostPath mode", c, pod, 0, []string{
			"mode of file \"/test-volume\": dtrwxrwxrwx", // we expect the sticky bit (mode flag t) to be set for the dir
		},
			namespace.Name)
	})

	It("should support r/w [Conformance]", func() {
		volumePath := "/test-volume"
		filePath := path.Join(volumePath, "test-file")
		retryDuration := 180
		source := &api.HostPathVolumeSource{
			Path: "/tmp",
		}
		pod := testPodWithHostVol(volumePath, source)

		pod.Spec.Containers[0].Args = []string{
			fmt.Sprintf("--new_file_0644=%v", filePath),
			fmt.Sprintf("--file_mode=%v", filePath),
		}

		pod.Spec.Containers[1].Args = []string{
			fmt.Sprintf("--file_content_in_loop=%v", filePath),
			fmt.Sprintf("--retry_time=%d", retryDuration),
		}
		//Read the content of the file with the second container to
		//verify volumes  being shared properly among containers within the pod.
		testContainerOutput("hostPath r/w", c, pod, 1, []string{
			"content of file \"/test-volume/test-file\": mount-tester new file",
		}, namespace.Name,
		)
	})
})

//These constants are borrowed from the other test.
//const volumeName = "test-volume"
const containerName1 = "test-container-1"
const containerName2 = "test-container-2"

func mount(source *api.HostPathVolumeSource) []api.Volume {
	return []api.Volume{
		{
			Name: volumeName,
			VolumeSource: api.VolumeSource{
				HostPath: source,
			},
		},
	}
}

//TODO: To merge this with the emptyDir tests, we can make source a lambda.
func testPodWithHostVol(path string, source *api.HostPathVolumeSource) *api.Pod {
	podName := "pod-host-path-test"

	return &api.Pod{
		TypeMeta: unversioned.TypeMeta{
			Kind:       "Pod",
			APIVersion: registered.GroupOrDie(api.GroupName).GroupVersion.String(),
		},
		ObjectMeta: api.ObjectMeta{
			Name: podName,
		},
		Spec: api.PodSpec{
			Containers: []api.Container{
				{
					Name:  containerName1,
					Image: "gcr.io/google_containers/mounttest:0.6",
					VolumeMounts: []api.VolumeMount{
						{
							Name:      volumeName,
							MountPath: path,
						},
					},
				},
				{
					Name:  containerName2,
					Image: "gcr.io/google_containers/mounttest:0.6",
					VolumeMounts: []api.VolumeMount{
						{
							Name:      volumeName,
							MountPath: path,
						},
					},
				},
			},
			RestartPolicy: api.RestartPolicyNever,
			Volumes:       mount(source),
		},
	}
}
<commit_msg>minimal hostPath fix<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"os"
	"path"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/apimachinery/registered"
	client "k8s.io/kubernetes/pkg/client/unversioned"

	. "github.com/onsi/ginkgo"
)

//TODO : Consolidate this code with the code for emptyDir.
//This will require some smart.
var _ = Describe("hostPath", func() {
	framework := NewDefaultFramework("hostpath")
	var c *client.Client
	var namespace *api.Namespace

	BeforeEach(func() {
		c = framework.Client
		namespace = framework.Namespace

		//cleanup before running the test.
		_ = os.Remove("/tmp/test-file")
	})

	It("should give a volume the correct mode [Conformance]", func() {
		volumePath := "/test-volume"
		source := &api.HostPathVolumeSource{
			Path: "/tmp",
		}
		pod := testPodWithHostVol(volumePath, source)

		pod.Spec.Containers[0].Args = []string{
			fmt.Sprintf("--fs_type=%v", volumePath),
			fmt.Sprintf("--file_mode=%v", volumePath),
		}
		testContainerOutput("hostPath mode", c, pod, 0, []string{
			"mode of file \"/test-volume\": dtrwxrwxrwx", // we expect the sticky bit (mode flag t) to be set for the dir
		},
			namespace.Name)
	})

	// This test requires mounting a folder into a container with write privileges.
	It("should support r/w", func() {
		volumePath := "/test-volume"
		filePath := path.Join(volumePath, "test-file")
		retryDuration := 180
		source := &api.HostPathVolumeSource{
			Path: "/tmp",
		}
		pod := testPodWithHostVol(volumePath, source)

		pod.Spec.Containers[0].Args = []string{
			fmt.Sprintf("--new_file_0644=%v", filePath),
			fmt.Sprintf("--file_mode=%v", filePath),
		}

		pod.Spec.Containers[1].Args = []string{
			fmt.Sprintf("--file_content_in_loop=%v", filePath),
			fmt.Sprintf("--retry_time=%d", retryDuration),
		}
		//Read the content of the file with the second container to
		//verify volumes  being shared properly among containers within the pod.
		testContainerOutput("hostPath r/w", c, pod, 1, []string{
			"content of file \"/test-volume/test-file\": mount-tester new file",
		}, namespace.Name,
		)
	})
})

//These constants are borrowed from the other test.
//const volumeName = "test-volume"
const containerName1 = "test-container-1"
const containerName2 = "test-container-2"

func mount(source *api.HostPathVolumeSource) []api.Volume {
	return []api.Volume{
		{
			Name: volumeName,
			VolumeSource: api.VolumeSource{
				HostPath: source,
			},
		},
	}
}

//TODO: To merge this with the emptyDir tests, we can make source a lambda.
func testPodWithHostVol(path string, source *api.HostPathVolumeSource) *api.Pod {
	podName := "pod-host-path-test"

	return &api.Pod{
		TypeMeta: unversioned.TypeMeta{
			Kind:       "Pod",
			APIVersion: registered.GroupOrDie(api.GroupName).GroupVersion.String(),
		},
		ObjectMeta: api.ObjectMeta{
			Name: podName,
		},
		Spec: api.PodSpec{
			Containers: []api.Container{
				{
					Name:  containerName1,
					Image: "gcr.io/google_containers/mounttest:0.6",
					VolumeMounts: []api.VolumeMount{
						{
							Name:      volumeName,
							MountPath: path,
						},
					},
				},
				{
					Name:  containerName2,
					Image: "gcr.io/google_containers/mounttest:0.6",
					VolumeMounts: []api.VolumeMount{
						{
							Name:      volumeName,
							MountPath: path,
						},
					},
				},
			},
			RestartPolicy: api.RestartPolicyNever,
			Volumes:       mount(source),
		},
	}
}
<|endoftext|>


<commit_before>// +build e2e

/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package example

import (
	"context"
	"os"
	"testing"

	"github.com/n3wscott/rigging/pkg/lifecycle"
	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"

	"knative.dev/pkg/injection"
)

var (
	test_context context.Context
)

func TestMain(m *testing.M) {
	ctx, startInformers := injection.EnableInjectionOrDie(nil, nil) //nolint
	lifecycle.InjectClients(ctx)
	test_context = ctx
	startInformers()
	os.Exit(m.Run())
}

// TestSmoke makes sure a ClusterDuckType goes ready.
func TestSmoke(t *testing.T) {
	// TOOD: add logstream back.
	SmokeTestImpl(t)
}
<commit_msg>Run github.com/client9/misspell (#98)<commit_after>// +build e2e

/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package example

import (
	"context"
	"os"
	"testing"

	"github.com/n3wscott/rigging/pkg/lifecycle"
	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"

	"knative.dev/pkg/injection"
)

var (
	test_context context.Context
)

func TestMain(m *testing.M) {
	ctx, startInformers := injection.EnableInjectionOrDie(nil, nil) //nolint
	lifecycle.InjectClients(ctx)
	test_context = ctx
	startInformers()
	os.Exit(m.Run())
}

// TestSmoke makes sure a ClusterDuckType goes ready.
func TestSmoke(t *testing.T) {
	// TODO: add logstream back.
	SmokeTestImpl(t)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"log"
	"os"
	"time"

	"github.com/rainforestapp/rainforest-cli/rainforest"
	"github.com/urfave/cli"
)

const (
	// Version of the app in SemVer
	version = "2.0.0-alpha.3"
	// This is the default spec folder for RFML tests
	defaultSpecFolder = "./spec/rainforest"
)

var (
	// Build info to be set while building using:
	// go build -ldflags "-X main.build=build"
	build string

	// Release channel to be set while building using:
	// go build -ldflags "-X main.releaseChannel=channel"
	releaseChannel string

	// Rainforest API client
	api *rainforest.Client

	// default output for printing resource tables
	tablesOut io.Writer = os.Stdout

	// Run status polling interval
	runStatusPollInterval = time.Second * 5

	// Batch size (number of rows) for tabular var upload
	tabularBatchSize = 50
	// Concurrent connections when uploading CSV rows
	tabularConcurrency = 1
	// Maximum concurrent connections with Rainforest server
	rfmlDownloadConcurrency = 4
	// Concurrent connections when uploading RFML files
	rfmlUploadConcurrency = 4
)

// cliContext is an interface providing context of running application
// i.e. command line options and flags. One of the types that provides the interface is
// cli.Context, the other is fakeCLIContext which is used for testing.
type cliContext interface {
	String(flag string) (val string)
	StringSlice(flag string) (vals []string)
	Bool(flag string) (val bool)
	Int(flag string) (val int)

	Args() (args cli.Args)
}

// Create custom writer which will use timestamps
type logWriter struct{}

func (l *logWriter) Write(p []byte) (int, error) {
	log.Printf("%s", p)
	return len(p), nil
}

// main is an entry point of the app. It sets up the new cli app, and defines the API.
func main() {
	updateFinishedChan := make(chan struct{})
	app := cli.NewApp()
	app.Usage = "Rainforest QA CLI - https://www.rainforestqa.com/"
	app.Version = version
	if releaseChannel != "" {
		app.Version = fmt.Sprintf("%v - %v channel", app.Version, releaseChannel)
	}
	if build != "" {
		app.Version = fmt.Sprintf("%v - build: %v", app.Version, build)
	}

	// Use our custom writer to print our errors with timestamps
	cli.ErrWriter = &logWriter{}

	// Before running any of the commands we init the API Client & update
	app.Before = func(c *cli.Context) error {
		go autoUpdate(c, updateFinishedChan)

		api = rainforest.NewClient(c.String("token"))

		// Set the User-Agent that will be used for api calls
		if build != "" {
			api.UserAgent = "rainforest-cli/" + version + " build: " + build
		} else {
			api.UserAgent = "rainforest-cli/" + version
		}

		return nil
	}

	// Wait for the update to finish if it's still going on
	app.After = func(c *cli.Context) error {
		<-updateFinishedChan
		return nil
	}

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "token, t",
			Usage:  "API token. You can find it at https://app.rainforestqa.com/settings/integrations",
			EnvVar: "RAINFOREST_API_TOKEN",
		},
		cli.BoolFlag{
			Name:  "skip-update",
			Usage: "Used to disable auto-updating of the cli",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:    "run",
			Aliases: []string{"r"},
			Usage:   "Run your tests on Rainforest",
			Action:  startRun,
			Description: "Runs your tests on Rainforest platform. " +
				"You need to specify list of test IDs to run or use keyword 'all'. " +
				"Alternatively you can use one of the filtering options.",
			ArgsUsage: "[test IDs]",
			Flags: []cli.Flag{
				cli.StringSliceFlag{
					Name:  "tag",
					Usage: "filter tests by `TAG`. Can be used multiple times for filtering by multiple tags.",
				},
				cli.StringFlag{
					Name:  "site, site-id",
					Usage: "filter tests by a specific site. You can see a list of your `SITE-ID`s with sites command.",
				},
				cli.StringFlag{
					Name:  "folder",
					Usage: "filter tests by a specific folder. You can see a list of your `FOLDER-ID`s with folders command.",
				},
				cli.StringSliceFlag{
					Name: "browser, browsers",
					Usage: "specify the `BROWSER` you wish to run against. This overrides test level settings." +
						"Can be used multiple times to run against multiple browsers.",
				},
				cli.StringFlag{
					Name:  "environment-id",
					Usage: "run your tests using specified `ENVIRONMENT`. Otherwise it will use your default one.",
				},
				cli.StringFlag{
					Name:  "crowd",
					Value: "default",
					Usage: "run your tests using specified `CROWD`. Available choices are: default or on_premise_crowd. " +
						"Contact your CSM for more details.",
				},
				cli.StringFlag{
					Name: "conflict",
					Usage: "use the abort option to abort any runs in the same environment or " +
						"use the abort-all option to abort all runs in progress.",
				},
				cli.BoolFlag{
					Name: "bg, background",
					Usage: "run in the background. This option makes cli return after succesfully starting a run, " +
						"without waiting for the run results.",
				},
				cli.BoolFlag{
					Name: "fail-fast, ff",
					Usage: "fail the build as soon as the first failed result comes in. " +
						"If you don't pass this it will wait until 100% of the run is done. Use with --fg.",
				},
				cli.StringFlag{
					Name: "custom-url",
					Usage: "use a custom `URL` for this run. Example use case: an ad-hoc QA environment with Fourchette. " +
						"You will need to specify a site_id too for this to work.",
				},
				cli.BoolFlag{
					Name: "git-trigger",
					Usage: "only trigger a run when the last commit (for a git repo in the current working directory) " +
						"contains @rainforest and a list of one or more tags. rainforest-cli exits with 0 otherwise.",
				},
				cli.StringFlag{
					Name:  "description",
					Usage: "add arbitrary `DESCRIPTION` to the run.",
				},
				cli.StringFlag{
					Name:  "junit-file",
					Usage: "Create a JUnit XML report `FILE` with the specified name. Must be run in foreground mode.",
				},
				cli.StringFlag{
					Name:  "import-variable-name",
					Usage: "`NAME` of the tabular variable to be created or updated.",
				},
				cli.StringFlag{
					Name:  "import-variable-csv-file",
					Usage: "`PATH` to the CSV file to be uploaded.",
				},
				cli.BoolFlag{
					Name:  "overwrite-variable",
					Usage: "If the flag is set, named variable will be updated.",
				},
				cli.BoolFlag{
					Name:  "single-use",
					Usage: "This option marks uploaded variable as single-use",
				},
				cli.StringFlag{
					Name:  "wait, reattach",
					Usage: "monitor existing run with `RUN_ID` instead of starting a new one.",
				},
			},
		},
		{
			Name:      "new",
			Usage:     "Create a new RFML test",
			ArgsUsage: "[name]",
			Description: "Create new Rainforest test in RFML format (Rainforest Markup Language). " +
				"You may also specify a custom test title or file name.",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:   "test-folder",
					Value:  defaultSpecFolder,
					Usage:  "`PATH` at which to create new test.",
					EnvVar: "RAINFOREST_TEST_FOLDER",
				},
			},
			Action: newRFMLTest,
		},
		{
			Name:      "validate",
			Usage:     "Validate your RFML tests",
			ArgsUsage: "[path to RFML file]",
			Description: "Validate your test for syntax. " +
				"If no filepath is given it validates all RFML tests and performs additional checks for RFML ID validity and more. " +
				"If API token is set it'll validate your tests against server data as well.",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:   "test-folder",
					Value:  "./spec/rainforest/",
					Usage:  "`PATH` where to look for a tests to validate.",
					EnvVar: "RAINFOREST_TEST_FOLDER",
				},
			},
			Action: validateRFML,
		},
		{
			Name:      "upload",
			Usage:     "Upload your RFML tests",
			ArgsUsage: "[path to RFML file]",
			Description: "Uploads specified test to Rainforest. " +
				"If no filepath is given it uploads all RFML tests.",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:   "test-folder",
					Value:  "./spec/rainforest/",
					Usage:  "`PATH` where to look for a tests to upload.",
					EnvVar: "RAINFOREST_TEST_FOLDER",
				},
				cli.BoolFlag{
					Name:  "synchronous-upload",
					Usage: "uploads your test in a synchronous manner i.e. not using concurrency.",
				},
			},
			Action: uploadRFML,
		},
		{
			Name:        "rm",
			Usage:       "Remove an RFML test locally and remotely",
			ArgsUsage:   "[path to RFML file]",
			Description: "Remove RFML file and remove test from Rainforest test suite.",
			Action:      deleteRFML,
		},
		{
			Name: "download",
			// Left for legacy reason, should nuke?
			Aliases:   []string{"export"},
			Usage:     "Download your remote Rainforest tests to RFML",
			ArgsUsage: "[test IDs]",
			Description: "Download your remote tests from Rainforest to RFML. " +
				"You need to specify list of test IDs to download or use keyword 'all'. " +
				"Alternatively you can use one of the filtering options.",
			Flags: []cli.Flag{
				cli.StringSliceFlag{
					Name:  "tag",
					Usage: "filter tests by `TAG`. Can be used multiple times for filtering by multiple tags.",
				},
				cli.IntFlag{
					Name:  "site, site-id",
					Usage: "filter tests by a specific site. You can see a list of your `SITE-ID`s with sites command.",
				},
				cli.IntFlag{
					Name:  "folder, folder-id",
					Usage: "filter tests by a specific folder. You can see a list of your `FOLDER-ID`s with folders command.",
				},
				cli.StringFlag{
					Name:   "test-folder",
					Value:  "./spec/rainforest/",
					Usage:  "`PATH` at which to save all the downloaded tests.",
					EnvVar: "RAINFOREST_TEST_FOLDER",
				},
				cli.BoolFlag{
					Name:  "embed-tests",
					Usage: "download your tests without extracting the steps of an embedded test.",
				},
			},
			Action: func(c *cli.Context) error {
				return downloadRFML(c, api)
			},
		},
		{
			Name:        "csv-upload",
			Usage:       "Create or update tabular var from CSV.",
			Description: "Upload a CSV file to create or update tabular variables.",
			ArgsUsage:   "[path to CSV file]",
			Flags: []cli.Flag{
				cli.StringFlag{
					// Alternative name left for legacy reason.
					Name:  "name, import-variable-name",
					Usage: "`NAME` of the tabular variable to be created or updated.",
				},
				cli.BoolFlag{
					Name:  "overwrite-variable, overwrite",
					Usage: "If the flag is set, named variable will be updated.",
				},
				cli.BoolFlag{
					Name:  "single-use",
					Usage: "This option marks uploaded variable as single-use",
				},
				// Left here for legacy reason, but imho we should move that to args
				cli.StringFlag{
					Name:  "csv-file, import-variable-csv-file",
					Usage: "DEPRECATED: `PATH` to the CSV file to be uploaded. Since v2 please provide the path as an argument.",
				},
			},
			Action: func(c *cli.Context) error {
				return csvUpload(c, api)
			},
		},
		{
			Name:  "report",
			Usage: "Create a report from your run results",
			Description: "Creates a report from your specified run." +
				"You can specify output file using options, otherwise report will be generated to STDOUT",
			ArgsUsage: "[run ID]",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "junit-file",
					Usage: "`PATH` of file to which write a JUnit report for the specified run.",
				},
				cli.StringFlag{
					Name:  "run-id",
					Usage: "DEPRECATED: ID of a run for which to generate results. Since v2 please provide the run ID as an argument.",
				},
			},
			Action: createReport,
		},
		{
			Name:  "sites",
			Usage: "Lists available sites",
			Action: func(c *cli.Context) error {
				return printSites(c, api)
			},
		},
		{
			Name:  "folders",
			Usage: "Lists available folders",
			Action: func(c *cli.Context) error {
				return printFolders(c, api)
			},
		},
		{
			Name:  "browsers",
			Usage: "Lists available browsers",
			Action: func(c *cli.Context) error {
				return printBrowsers(c, api)
			},
		},
		{
			Name:      "update",
			Usage:     "Updates application to the latest version on specified release channel (stable/beta)",
			ArgsUsage: "[CHANNEL]",
			Action:    updateCmd,
		},
	}

	app.Run(shuffleFlags(os.Args))
}

// shuffleFlags moves global flags to the beginning of args array (where they are supposed to be),
// so they are picked up by the cli package, even though they are supplied as a command argument.
// might not be needed if upstream makes this change as well
func shuffleFlags(originalArgs []string) []string {
	globalOptions := []string{}
	rest := []string{}

	// We need to skip the filename as its arg[0] that's why iteration starts at 1
	// then filter out global flags and put them into separate array than the rest of arg
	for i := 1; i < len(originalArgs); i++ {
		option := originalArgs[i]
		if (option == "--token" || option == "-t") && originalArgs[i+1][:1] != "-" {
			globalOptions = append(globalOptions, originalArgs[i:i+2]...)
			i++
		} else if option == "--skip-update" {
			globalOptions = append(globalOptions, option)
		} else {
			rest = append(rest, option)
		}
	}

	shuffledFlags := []string{originalArgs[0]}
	shuffledFlags = append(shuffledFlags, globalOptions...)
	shuffledFlags = append(shuffledFlags, rest...)

	return shuffledFlags
}
<commit_msg>add something that looks better than index out of range error<commit_after>package main

import (
	"fmt"
	"io"
	"log"
	"os"
	"time"

	"github.com/rainforestapp/rainforest-cli/rainforest"
	"github.com/urfave/cli"
)

const (
	// Version of the app in SemVer
	version = "2.0.0-alpha.3"
	// This is the default spec folder for RFML tests
	defaultSpecFolder = "./spec/rainforest"
)

var (
	// Build info to be set while building using:
	// go build -ldflags "-X main.build=build"
	build string

	// Release channel to be set while building using:
	// go build -ldflags "-X main.releaseChannel=channel"
	releaseChannel string

	// Rainforest API client
	api *rainforest.Client

	// default output for printing resource tables
	tablesOut io.Writer = os.Stdout

	// Run status polling interval
	runStatusPollInterval = time.Second * 5

	// Batch size (number of rows) for tabular var upload
	tabularBatchSize = 50
	// Concurrent connections when uploading CSV rows
	tabularConcurrency = 1
	// Maximum concurrent connections with Rainforest server
	rfmlDownloadConcurrency = 4
	// Concurrent connections when uploading RFML files
	rfmlUploadConcurrency = 4
)

// cliContext is an interface providing context of running application
// i.e. command line options and flags. One of the types that provides the interface is
// cli.Context, the other is fakeCLIContext which is used for testing.
type cliContext interface {
	String(flag string) (val string)
	StringSlice(flag string) (vals []string)
	Bool(flag string) (val bool)
	Int(flag string) (val int)

	Args() (args cli.Args)
}

// Create custom writer which will use timestamps
type logWriter struct{}

func (l *logWriter) Write(p []byte) (int, error) {
	log.Printf("%s", p)
	return len(p), nil
}

// main is an entry point of the app. It sets up the new cli app, and defines the API.
func main() {
	updateFinishedChan := make(chan struct{})
	app := cli.NewApp()
	app.Usage = "Rainforest QA CLI - https://www.rainforestqa.com/"
	app.Version = version
	if releaseChannel != "" {
		app.Version = fmt.Sprintf("%v - %v channel", app.Version, releaseChannel)
	}
	if build != "" {
		app.Version = fmt.Sprintf("%v - build: %v", app.Version, build)
	}

	// Use our custom writer to print our errors with timestamps
	cli.ErrWriter = &logWriter{}

	// Before running any of the commands we init the API Client & update
	app.Before = func(c *cli.Context) error {
		go autoUpdate(c, updateFinishedChan)

		api = rainforest.NewClient(c.String("token"))

		// Set the User-Agent that will be used for api calls
		if build != "" {
			api.UserAgent = "rainforest-cli/" + version + " build: " + build
		} else {
			api.UserAgent = "rainforest-cli/" + version
		}

		return nil
	}

	// Wait for the update to finish if it's still going on
	app.After = func(c *cli.Context) error {
		<-updateFinishedChan
		return nil
	}

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "token, t",
			Usage:  "API token. You can find it at https://app.rainforestqa.com/settings/integrations",
			EnvVar: "RAINFOREST_API_TOKEN",
		},
		cli.BoolFlag{
			Name:  "skip-update",
			Usage: "Used to disable auto-updating of the cli",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:    "run",
			Aliases: []string{"r"},
			Usage:   "Run your tests on Rainforest",
			Action:  startRun,
			Description: "Runs your tests on Rainforest platform. " +
				"You need to specify list of test IDs to run or use keyword 'all'. " +
				"Alternatively you can use one of the filtering options.",
			ArgsUsage: "[test IDs]",
			Flags: []cli.Flag{
				cli.StringSliceFlag{
					Name:  "tag",
					Usage: "filter tests by `TAG`. Can be used multiple times for filtering by multiple tags.",
				},
				cli.StringFlag{
					Name:  "site, site-id",
					Usage: "filter tests by a specific site. You can see a list of your `SITE-ID`s with sites command.",
				},
				cli.StringFlag{
					Name:  "folder",
					Usage: "filter tests by a specific folder. You can see a list of your `FOLDER-ID`s with folders command.",
				},
				cli.StringSliceFlag{
					Name: "browser, browsers",
					Usage: "specify the `BROWSER` you wish to run against. This overrides test level settings." +
						"Can be used multiple times to run against multiple browsers.",
				},
				cli.StringFlag{
					Name:  "environment-id",
					Usage: "run your tests using specified `ENVIRONMENT`. Otherwise it will use your default one.",
				},
				cli.StringFlag{
					Name:  "crowd",
					Value: "default",
					Usage: "run your tests using specified `CROWD`. Available choices are: default or on_premise_crowd. " +
						"Contact your CSM for more details.",
				},
				cli.StringFlag{
					Name: "conflict",
					Usage: "use the abort option to abort any runs in the same environment or " +
						"use the abort-all option to abort all runs in progress.",
				},
				cli.BoolFlag{
					Name: "bg, background",
					Usage: "run in the background. This option makes cli return after succesfully starting a run, " +
						"without waiting for the run results.",
				},
				cli.BoolFlag{
					Name: "fail-fast, ff",
					Usage: "fail the build as soon as the first failed result comes in. " +
						"If you don't pass this it will wait until 100% of the run is done. Use with --fg.",
				},
				cli.StringFlag{
					Name: "custom-url",
					Usage: "use a custom `URL` for this run. Example use case: an ad-hoc QA environment with Fourchette. " +
						"You will need to specify a site_id too for this to work.",
				},
				cli.BoolFlag{
					Name: "git-trigger",
					Usage: "only trigger a run when the last commit (for a git repo in the current working directory) " +
						"contains @rainforest and a list of one or more tags. rainforest-cli exits with 0 otherwise.",
				},
				cli.StringFlag{
					Name:  "description",
					Usage: "add arbitrary `DESCRIPTION` to the run.",
				},
				cli.StringFlag{
					Name:  "junit-file",
					Usage: "Create a JUnit XML report `FILE` with the specified name. Must be run in foreground mode.",
				},
				cli.StringFlag{
					Name:  "import-variable-name",
					Usage: "`NAME` of the tabular variable to be created or updated.",
				},
				cli.StringFlag{
					Name:  "import-variable-csv-file",
					Usage: "`PATH` to the CSV file to be uploaded.",
				},
				cli.BoolFlag{
					Name:  "overwrite-variable",
					Usage: "If the flag is set, named variable will be updated.",
				},
				cli.BoolFlag{
					Name:  "single-use",
					Usage: "This option marks uploaded variable as single-use",
				},
				cli.StringFlag{
					Name:  "wait, reattach",
					Usage: "monitor existing run with `RUN_ID` instead of starting a new one.",
				},
			},
		},
		{
			Name:      "new",
			Usage:     "Create a new RFML test",
			ArgsUsage: "[name]",
			Description: "Create new Rainforest test in RFML format (Rainforest Markup Language). " +
				"You may also specify a custom test title or file name.",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:   "test-folder",
					Value:  defaultSpecFolder,
					Usage:  "`PATH` at which to create new test.",
					EnvVar: "RAINFOREST_TEST_FOLDER",
				},
			},
			Action: newRFMLTest,
		},
		{
			Name:      "validate",
			Usage:     "Validate your RFML tests",
			ArgsUsage: "[path to RFML file]",
			Description: "Validate your test for syntax. " +
				"If no filepath is given it validates all RFML tests and performs additional checks for RFML ID validity and more. " +
				"If API token is set it'll validate your tests against server data as well.",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:   "test-folder",
					Value:  "./spec/rainforest/",
					Usage:  "`PATH` where to look for a tests to validate.",
					EnvVar: "RAINFOREST_TEST_FOLDER",
				},
			},
			Action: validateRFML,
		},
		{
			Name:      "upload",
			Usage:     "Upload your RFML tests",
			ArgsUsage: "[path to RFML file]",
			Description: "Uploads specified test to Rainforest. " +
				"If no filepath is given it uploads all RFML tests.",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:   "test-folder",
					Value:  "./spec/rainforest/",
					Usage:  "`PATH` where to look for a tests to upload.",
					EnvVar: "RAINFOREST_TEST_FOLDER",
				},
				cli.BoolFlag{
					Name:  "synchronous-upload",
					Usage: "uploads your test in a synchronous manner i.e. not using concurrency.",
				},
			},
			Action: uploadRFML,
		},
		{
			Name:        "rm",
			Usage:       "Remove an RFML test locally and remotely",
			ArgsUsage:   "[path to RFML file]",
			Description: "Remove RFML file and remove test from Rainforest test suite.",
			Action:      deleteRFML,
		},
		{
			Name: "download",
			// Left for legacy reason, should nuke?
			Aliases:   []string{"export"},
			Usage:     "Download your remote Rainforest tests to RFML",
			ArgsUsage: "[test IDs]",
			Description: "Download your remote tests from Rainforest to RFML. " +
				"You need to specify list of test IDs to download or use keyword 'all'. " +
				"Alternatively you can use one of the filtering options.",
			Flags: []cli.Flag{
				cli.StringSliceFlag{
					Name:  "tag",
					Usage: "filter tests by `TAG`. Can be used multiple times for filtering by multiple tags.",
				},
				cli.IntFlag{
					Name:  "site, site-id",
					Usage: "filter tests by a specific site. You can see a list of your `SITE-ID`s with sites command.",
				},
				cli.IntFlag{
					Name:  "folder, folder-id",
					Usage: "filter tests by a specific folder. You can see a list of your `FOLDER-ID`s with folders command.",
				},
				cli.StringFlag{
					Name:   "test-folder",
					Value:  "./spec/rainforest/",
					Usage:  "`PATH` at which to save all the downloaded tests.",
					EnvVar: "RAINFOREST_TEST_FOLDER",
				},
				cli.BoolFlag{
					Name:  "embed-tests",
					Usage: "download your tests without extracting the steps of an embedded test.",
				},
			},
			Action: func(c *cli.Context) error {
				return downloadRFML(c, api)
			},
		},
		{
			Name:        "csv-upload",
			Usage:       "Create or update tabular var from CSV.",
			Description: "Upload a CSV file to create or update tabular variables.",
			ArgsUsage:   "[path to CSV file]",
			Flags: []cli.Flag{
				cli.StringFlag{
					// Alternative name left for legacy reason.
					Name:  "name, import-variable-name",
					Usage: "`NAME` of the tabular variable to be created or updated.",
				},
				cli.BoolFlag{
					Name:  "overwrite-variable, overwrite",
					Usage: "If the flag is set, named variable will be updated.",
				},
				cli.BoolFlag{
					Name:  "single-use",
					Usage: "This option marks uploaded variable as single-use",
				},
				// Left here for legacy reason, but imho we should move that to args
				cli.StringFlag{
					Name:  "csv-file, import-variable-csv-file",
					Usage: "DEPRECATED: `PATH` to the CSV file to be uploaded. Since v2 please provide the path as an argument.",
				},
			},
			Action: func(c *cli.Context) error {
				return csvUpload(c, api)
			},
		},
		{
			Name:  "report",
			Usage: "Create a report from your run results",
			Description: "Creates a report from your specified run." +
				"You can specify output file using options, otherwise report will be generated to STDOUT",
			ArgsUsage: "[run ID]",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "junit-file",
					Usage: "`PATH` of file to which write a JUnit report for the specified run.",
				},
				cli.StringFlag{
					Name:  "run-id",
					Usage: "DEPRECATED: ID of a run for which to generate results. Since v2 please provide the run ID as an argument.",
				},
			},
			Action: createReport,
		},
		{
			Name:  "sites",
			Usage: "Lists available sites",
			Action: func(c *cli.Context) error {
				return printSites(c, api)
			},
		},
		{
			Name:  "folders",
			Usage: "Lists available folders",
			Action: func(c *cli.Context) error {
				return printFolders(c, api)
			},
		},
		{
			Name:  "browsers",
			Usage: "Lists available browsers",
			Action: func(c *cli.Context) error {
				return printBrowsers(c, api)
			},
		},
		{
			Name:      "update",
			Usage:     "Updates application to the latest version on specified release channel (stable/beta)",
			ArgsUsage: "[CHANNEL]",
			Action:    updateCmd,
		},
	}

	app.Run(shuffleFlags(os.Args))
}

// shuffleFlags moves global flags to the beginning of args array (where they are supposed to be),
// so they are picked up by the cli package, even though they are supplied as a command argument.
// might not be needed if upstream makes this change as well
func shuffleFlags(originalArgs []string) []string {
	globalOptions := []string{}
	rest := []string{}

	// We need to skip the filename as its arg[0] that's why iteration starts at 1
	// then filter out global flags and put them into separate array than the rest of arg
	for i := 1; i < len(originalArgs); i++ {
		option := originalArgs[i]
		if option == "--token" || option == "-t" {
			if i+1 < len(originalArgs) && originalArgs[i+1][:1] != "-" {
				globalOptions = append(globalOptions, originalArgs[i:i+2]...)
				i++
			} else {
				log.Fatalln("No token specified with --token flag")
			}
		} else if option == "--skip-update" {
			globalOptions = append(globalOptions, option)
		} else {
			rest = append(rest, option)
		}
	}

	shuffledFlags := []string{originalArgs[0]}
	shuffledFlags = append(shuffledFlags, globalOptions...)
	shuffledFlags = append(shuffledFlags, rest...)

	return shuffledFlags
}
<|endoftext|>