<commit_before>package http

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"path"

	"github.com/influxdata/platform"
	"github.com/influxdata/platform/kit/errors"
	"github.com/julienschmidt/httprouter"
)

// OrgHandler represents an HTTP API handler for orgs.
type OrgHandler struct {
	*httprouter.Router

	OrganizationService platform.OrganizationService
}

// NewOrgHandler returns a new instance of OrgHandler.
func NewOrgHandler() *OrgHandler {
	h := &OrgHandler{
		Router: httprouter.New(),
	}

	h.HandlerFunc("POST", "/v1/orgs", h.handlePostOrg)
	h.HandlerFunc("GET", "/v1/orgs", h.handleGetOrgs)
	h.HandlerFunc("GET", "/v1/orgs/:id", h.handleGetOrg)
	h.HandlerFunc("PATCH", "/v1/orgs/:id", h.handlePatchOrg)
	h.HandlerFunc("DELETE", "/v1/orgs/:id", h.handleDeleteOrg)
	return h
}

// handlePostOrg is the HTTP handler for the POST /v1/orgs route.
func (h *OrgHandler) handlePostOrg(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()

	req, err := decodePostOrgRequest(ctx, r)
	if err != nil {
		errors.EncodeHTTP(ctx, err, w)
		return
	}

	if err := h.OrganizationService.CreateOrganization(ctx, req.Org); err != nil {
		errors.EncodeHTTP(ctx, err, w)
		return
	}

	if err := encodeResponse(ctx, w, http.StatusCreated, req.Org); err != nil {
		errors.EncodeHTTP(ctx, err, w)
		return
	}
}

type postOrgRequest struct {
	Org *platform.Organization
}

func decodePostOrgRequest(ctx context.Context, r *http.Request) (*postOrgRequest, error) {
	o := &platform.Organization{}
	if err := json.NewDecoder(r.Body).Decode(o); err != nil {
		return nil, err
	}

	return &postOrgRequest{
		Org: o,
	}, nil
}

// handleGetOrg is the HTTP handler for the GET /v1/orgs/:id route.
func (h *OrgHandler) handleGetOrg(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()

	req, err := decodeGetOrgRequest(ctx, r)
	if err != nil {
		errors.EncodeHTTP(ctx, err, w)
		return
	}

	b, err := h.OrganizationService.FindOrganizationByID(ctx, req.OrgID)
	if err != nil {
		errors.EncodeHTTP(ctx, err, w)
		return
	}

	if err := encodeResponse(ctx, w, http.StatusOK, b); err != nil {
		errors.EncodeHTTP(ctx, err, w)
		return
	}
}

type getOrgRequest struct {
	OrgID platform.ID
}

func decodeGetOrgRequest(ctx context.Context, r *http.Request) (*getOrgRequest, error) {
	params := httprouter.ParamsFromContext(ctx)
	id := params.ByName("id")
	if id == "" {
		return nil, errors.InvalidDataf("url missing id")
	}

	var i platform.ID
	if err := i.DecodeFromString(id); err != nil {
		return nil, err
	}

	req := &getOrgRequest{
		OrgID: i,
	}

	return req, nil
}

// handleGetOrgs is the HTTP handler for the GET /v1/orgs route.
func (h *OrgHandler) handleGetOrgs(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()

	req, err := decodeGetOrgsRequest(ctx, r)
	if err != nil {
		errors.EncodeHTTP(ctx, err, w)
		return
	}

	orgs, _, err := h.OrganizationService.FindOrganizations(ctx, req.filter)
	if err != nil {
		errors.EncodeHTTP(ctx, err, w)
		return
	}

	if err := encodeResponse(ctx, w, http.StatusOK, orgs); err != nil {
		errors.EncodeHTTP(ctx, err, w)
		return
	}
}

type getOrgsRequest struct {
	filter platform.OrganizationFilter
}

func decodeGetOrgsRequest(ctx context.Context, r *http.Request) (*getOrgsRequest, error) {
	qp := r.URL.Query()
	req := &getOrgsRequest{}

	if id := qp.Get("id"); id != "" {
		req.filter.ID = &platform.ID{}
		if err := req.filter.ID.DecodeFromString(id); err != nil {
			return nil, err
		}
	}

	if name := qp.Get("org"); name != "" {
		req.filter.Name = &name
	}

	return req, nil
}

// handleDeleteOrganization is the HTTP handler for the DELETE /v1/organizations/:id route.
func (h *OrgHandler) handleDeleteOrg(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()

	req, err := decodeDeleteOrganizationRequest(ctx, r)
	if err != nil {
		errors.EncodeHTTP(ctx, err, w)
		return
	}

	if err := h.OrganizationService.DeleteOrganization(ctx, req.OrganizationID); err != nil {
		errors.EncodeHTTP(ctx, err, w)
		return
	}

	w.WriteHeader(http.StatusAccepted)
}

type deleteOrganizationRequest struct {
	OrganizationID platform.ID
}

func decodeDeleteOrganizationRequest(ctx context.Context, r *http.Request) (*deleteOrganizationRequest, error) {
	params := httprouter.ParamsFromContext(ctx)
	id := params.ByName("id")
	if id == "" {
		return nil, errors.InvalidDataf("url missing id")
	}

	var i platform.ID
	if err := i.DecodeFromString(id); err != nil {
		return nil, err
	}
	req := &deleteOrganizationRequest{
		OrganizationID: i,
	}

	return req, nil
}

// handlePatchOrg is the HTTP handler for the PATH /v1/orgs route.
func (h *OrgHandler) handlePatchOrg(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()

	req, err := decodePatchOrgRequest(ctx, r)
	if err != nil {
		errors.EncodeHTTP(ctx, err, w)
		return
	}

	o, err := h.OrganizationService.UpdateOrganization(ctx, req.OrgID, req.Update)
	if err != nil {
		errors.EncodeHTTP(ctx, err, w)
		return
	}

	if err := encodeResponse(ctx, w, http.StatusOK, o); err != nil {
		errors.EncodeHTTP(ctx, err, w)
		return
	}
}

type patchOrgRequest struct {
	Update platform.OrganizationUpdate
	OrgID  platform.ID
}

func decodePatchOrgRequest(ctx context.Context, r *http.Request) (*patchOrgRequest, error) {
	params := httprouter.ParamsFromContext(ctx)
	id := params.ByName("id")
	if id == "" {
		return nil, errors.InvalidDataf("url missing id")
	}

	var i platform.ID
	if err := i.DecodeFromString(id); err != nil {
		return nil, err
	}

	var upd platform.OrganizationUpdate
	if err := json.NewDecoder(r.Body).Decode(&upd); err != nil {
		return nil, err
	}

	return &patchOrgRequest{
		Update: upd,
		OrgID:  i,
	}, nil
}

const (
	organizationPath = "/v1/orgs"
)

// OrganizationService connects to Influx via HTTP using tokens to manage organizations.
type OrganizationService struct {
	Addr               string
	Token              string
	InsecureSkipVerify bool
}

func (s *OrganizationService) FindOrganizationByID(ctx context.Context, id platform.ID) (*platform.Organization, error) {
	filter := platform.OrganizationFilter{ID: &id}
	return s.FindOrganization(ctx, filter)
}

func (s *OrganizationService) FindOrganization(ctx context.Context, filter platform.OrganizationFilter) (*platform.Organization, error) {
	os, n, err := s.FindOrganizations(ctx, filter)
	if err != nil {
		return nil, err
	}

	if n < 1 {
		return nil, fmt.Errorf("expected at least one organization")
	}

	return os[0], nil
}

func (s *OrganizationService) FindOrganizations(ctx context.Context, filter platform.OrganizationFilter, opt ...platform.FindOptions) ([]*platform.Organization, int, error) {
	url, err := newURL(s.Addr, organizationPath)
	if err != nil {
		return nil, 0, err
	}
	qp := url.Query()

	if filter.Name != nil {
		qp.Add("name", *filter.Name)
	}
	if filter.ID != nil {
		qp.Add("id", filter.ID.String())
	}
	url.RawQuery = qp.Encode()

	req, err := http.NewRequest("GET", url.String(), nil)
	if err != nil {
		return nil, 0, err
	}

	req.Header.Set("Authorization", s.Token)
	hc := newClient(url.Scheme, s.InsecureSkipVerify)

	resp, err := hc.Do(req)
	if err != nil {
		return nil, 0, err
	}

	if resp.StatusCode != http.StatusOK {
		return nil, 0, fmt.Errorf(resp.Header.Get("X-Influx-Error"))
	}

	var os []*platform.Organization
	if err := json.NewDecoder(resp.Body).Decode(&os); err != nil {
		return nil, 0, err
	}

	return os, len(os), nil

}

// CreateOrganization creates an organization.
func (s *OrganizationService) CreateOrganization(ctx context.Context, o *platform.Organization) error {
	url, err := newURL(s.Addr, organizationPath)
	if err != nil {
		return err
	}

	octets, err := json.Marshal(o)
	if err != nil {
		return err
	}

	req, err := http.NewRequest("POST", url.String(), bytes.NewReader(octets))
	if err != nil {
		return err
	}

	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("Authorization", s.Token)

	hc := newClient(url.Scheme, s.InsecureSkipVerify)

	resp, err := hc.Do(req)
	if err != nil {
		return err
	}

	// TODO: this should really check the error from the headers
	if resp.StatusCode != http.StatusCreated {
		return fmt.Errorf(resp.Header.Get("X-Influx-Error"))
	}

	if err := json.NewDecoder(resp.Body).Decode(o); err != nil {
		return err
	}

	return nil
}

func (s *OrganizationService) UpdateOrganization(ctx context.Context, id platform.ID, upd platform.OrganizationUpdate) (*platform.Organization, error) {
	u, err := newURL(s.Addr, organizationIDPath(id))
	if err != nil {
		return nil, err
	}

	octets, err := json.Marshal(upd)
	if err != nil {
		return nil, err
	}

	req, err := http.NewRequest("PATCH", u.String(), bytes.NewReader(octets))
	if err != nil {
		return nil, err
	}

	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("Authorization", s.Token)

	hc := newClient(u.Scheme, s.InsecureSkipVerify)

	resp, err := hc.Do(req)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf(resp.Header.Get("X-Influx-Error"))
	}

	var o platform.Organization
	if err := json.NewDecoder(resp.Body).Decode(&o); err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	return &o, nil
}

func (s *OrganizationService) DeleteOrganization(ctx context.Context, id platform.ID) error {
	u, err := newURL(s.Addr, organizationIDPath(id))
	if err != nil {
		return err
	}

	req, err := http.NewRequest("DELETE", u.String(), nil)
	if err != nil {
		return err
	}
	req.Header.Set("Authorization", s.Token)

	hc := newClient(u.Scheme, s.InsecureSkipVerify)
	resp, err := hc.Do(req)
	if err != nil {
		return err
	}

	switch resp.StatusCode {
	case http.StatusNoContent, http.StatusAccepted:
		return nil
	}

	return fmt.Errorf(resp.Header.Get("X-Influx-Error"))
}

func organizationIDPath(id platform.ID) string {
	return path.Join(organizationPath, id.String())
}
<commit_msg>fix(http): use name query parameter instead of org in org routes<commit_after>package http

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"path"

	"github.com/influxdata/platform"
	"github.com/influxdata/platform/kit/errors"
	"github.com/julienschmidt/httprouter"
)

// OrgHandler represents an HTTP API handler for orgs.
type OrgHandler struct {
	*httprouter.Router

	OrganizationService platform.OrganizationService
}

// NewOrgHandler returns a new instance of OrgHandler.
func NewOrgHandler() *OrgHandler {
	h := &OrgHandler{
		Router: httprouter.New(),
	}

	h.HandlerFunc("POST", "/v1/orgs", h.handlePostOrg)
	h.HandlerFunc("GET", "/v1/orgs", h.handleGetOrgs)
	h.HandlerFunc("GET", "/v1/orgs/:id", h.handleGetOrg)
	h.HandlerFunc("PATCH", "/v1/orgs/:id", h.handlePatchOrg)
	h.HandlerFunc("DELETE", "/v1/orgs/:id", h.handleDeleteOrg)
	return h
}

// handlePostOrg is the HTTP handler for the POST /v1/orgs route.
func (h *OrgHandler) handlePostOrg(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()

	req, err := decodePostOrgRequest(ctx, r)
	if err != nil {
		errors.EncodeHTTP(ctx, err, w)
		return
	}

	if err := h.OrganizationService.CreateOrganization(ctx, req.Org); err != nil {
		errors.EncodeHTTP(ctx, err, w)
		return
	}

	if err := encodeResponse(ctx, w, http.StatusCreated, req.Org); err != nil {
		errors.EncodeHTTP(ctx, err, w)
		return
	}
}

type postOrgRequest struct {
	Org *platform.Organization
}

func decodePostOrgRequest(ctx context.Context, r *http.Request) (*postOrgRequest, error) {
	o := &platform.Organization{}
	if err := json.NewDecoder(r.Body).Decode(o); err != nil {
		return nil, err
	}

	return &postOrgRequest{
		Org: o,
	}, nil
}

// handleGetOrg is the HTTP handler for the GET /v1/orgs/:id route.
func (h *OrgHandler) handleGetOrg(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()

	req, err := decodeGetOrgRequest(ctx, r)
	if err != nil {
		errors.EncodeHTTP(ctx, err, w)
		return
	}

	b, err := h.OrganizationService.FindOrganizationByID(ctx, req.OrgID)
	if err != nil {
		errors.EncodeHTTP(ctx, err, w)
		return
	}

	if err := encodeResponse(ctx, w, http.StatusOK, b); err != nil {
		errors.EncodeHTTP(ctx, err, w)
		return
	}
}

type getOrgRequest struct {
	OrgID platform.ID
}

func decodeGetOrgRequest(ctx context.Context, r *http.Request) (*getOrgRequest, error) {
	params := httprouter.ParamsFromContext(ctx)
	id := params.ByName("id")
	if id == "" {
		return nil, errors.InvalidDataf("url missing id")
	}

	var i platform.ID
	if err := i.DecodeFromString(id); err != nil {
		return nil, err
	}

	req := &getOrgRequest{
		OrgID: i,
	}

	return req, nil
}

// handleGetOrgs is the HTTP handler for the GET /v1/orgs route.
func (h *OrgHandler) handleGetOrgs(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()

	req, err := decodeGetOrgsRequest(ctx, r)
	if err != nil {
		errors.EncodeHTTP(ctx, err, w)
		return
	}

	orgs, _, err := h.OrganizationService.FindOrganizations(ctx, req.filter)
	if err != nil {
		errors.EncodeHTTP(ctx, err, w)
		return
	}

	if err := encodeResponse(ctx, w, http.StatusOK, orgs); err != nil {
		errors.EncodeHTTP(ctx, err, w)
		return
	}
}

type getOrgsRequest struct {
	filter platform.OrganizationFilter
}

func decodeGetOrgsRequest(ctx context.Context, r *http.Request) (*getOrgsRequest, error) {
	qp := r.URL.Query()
	req := &getOrgsRequest{}

	if id := qp.Get("id"); id != "" {
		req.filter.ID = &platform.ID{}
		if err := req.filter.ID.DecodeFromString(id); err != nil {
			return nil, err
		}
	}

	if name := qp.Get("name"); name != "" {
		req.filter.Name = &name
	}

	return req, nil
}

// handleDeleteOrganization is the HTTP handler for the DELETE /v1/organizations/:id route.
func (h *OrgHandler) handleDeleteOrg(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()

	req, err := decodeDeleteOrganizationRequest(ctx, r)
	if err != nil {
		errors.EncodeHTTP(ctx, err, w)
		return
	}

	if err := h.OrganizationService.DeleteOrganization(ctx, req.OrganizationID); err != nil {
		errors.EncodeHTTP(ctx, err, w)
		return
	}

	w.WriteHeader(http.StatusAccepted)
}

type deleteOrganizationRequest struct {
	OrganizationID platform.ID
}

func decodeDeleteOrganizationRequest(ctx context.Context, r *http.Request) (*deleteOrganizationRequest, error) {
	params := httprouter.ParamsFromContext(ctx)
	id := params.ByName("id")
	if id == "" {
		return nil, errors.InvalidDataf("url missing id")
	}

	var i platform.ID
	if err := i.DecodeFromString(id); err != nil {
		return nil, err
	}
	req := &deleteOrganizationRequest{
		OrganizationID: i,
	}

	return req, nil
}

// handlePatchOrg is the HTTP handler for the PATH /v1/orgs route.
func (h *OrgHandler) handlePatchOrg(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()

	req, err := decodePatchOrgRequest(ctx, r)
	if err != nil {
		errors.EncodeHTTP(ctx, err, w)
		return
	}

	o, err := h.OrganizationService.UpdateOrganization(ctx, req.OrgID, req.Update)
	if err != nil {
		errors.EncodeHTTP(ctx, err, w)
		return
	}

	if err := encodeResponse(ctx, w, http.StatusOK, o); err != nil {
		errors.EncodeHTTP(ctx, err, w)
		return
	}
}

type patchOrgRequest struct {
	Update platform.OrganizationUpdate
	OrgID  platform.ID
}

func decodePatchOrgRequest(ctx context.Context, r *http.Request) (*patchOrgRequest, error) {
	params := httprouter.ParamsFromContext(ctx)
	id := params.ByName("id")
	if id == "" {
		return nil, errors.InvalidDataf("url missing id")
	}

	var i platform.ID
	if err := i.DecodeFromString(id); err != nil {
		return nil, err
	}

	var upd platform.OrganizationUpdate
	if err := json.NewDecoder(r.Body).Decode(&upd); err != nil {
		return nil, err
	}

	return &patchOrgRequest{
		Update: upd,
		OrgID:  i,
	}, nil
}

const (
	organizationPath = "/v1/orgs"
)

// OrganizationService connects to Influx via HTTP using tokens to manage organizations.
type OrganizationService struct {
	Addr               string
	Token              string
	InsecureSkipVerify bool
}

func (s *OrganizationService) FindOrganizationByID(ctx context.Context, id platform.ID) (*platform.Organization, error) {
	filter := platform.OrganizationFilter{ID: &id}
	return s.FindOrganization(ctx, filter)
}

func (s *OrganizationService) FindOrganization(ctx context.Context, filter platform.OrganizationFilter) (*platform.Organization, error) {
	os, n, err := s.FindOrganizations(ctx, filter)
	if err != nil {
		return nil, err
	}

	if n < 1 {
		return nil, fmt.Errorf("expected at least one organization")
	}

	return os[0], nil
}

func (s *OrganizationService) FindOrganizations(ctx context.Context, filter platform.OrganizationFilter, opt ...platform.FindOptions) ([]*platform.Organization, int, error) {
	url, err := newURL(s.Addr, organizationPath)
	if err != nil {
		return nil, 0, err
	}
	qp := url.Query()

	if filter.Name != nil {
		qp.Add("name", *filter.Name)
	}
	if filter.ID != nil {
		qp.Add("id", filter.ID.String())
	}
	url.RawQuery = qp.Encode()

	req, err := http.NewRequest("GET", url.String(), nil)
	if err != nil {
		return nil, 0, err
	}

	req.Header.Set("Authorization", s.Token)
	hc := newClient(url.Scheme, s.InsecureSkipVerify)

	resp, err := hc.Do(req)
	if err != nil {
		return nil, 0, err
	}

	if resp.StatusCode != http.StatusOK {
		return nil, 0, fmt.Errorf(resp.Header.Get("X-Influx-Error"))
	}

	var os []*platform.Organization
	if err := json.NewDecoder(resp.Body).Decode(&os); err != nil {
		return nil, 0, err
	}

	return os, len(os), nil

}

// CreateOrganization creates an organization.
func (s *OrganizationService) CreateOrganization(ctx context.Context, o *platform.Organization) error {
	url, err := newURL(s.Addr, organizationPath)
	if err != nil {
		return err
	}

	octets, err := json.Marshal(o)
	if err != nil {
		return err
	}

	req, err := http.NewRequest("POST", url.String(), bytes.NewReader(octets))
	if err != nil {
		return err
	}

	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("Authorization", s.Token)

	hc := newClient(url.Scheme, s.InsecureSkipVerify)

	resp, err := hc.Do(req)
	if err != nil {
		return err
	}

	// TODO: this should really check the error from the headers
	if resp.StatusCode != http.StatusCreated {
		return fmt.Errorf(resp.Header.Get("X-Influx-Error"))
	}

	if err := json.NewDecoder(resp.Body).Decode(o); err != nil {
		return err
	}

	return nil
}

func (s *OrganizationService) UpdateOrganization(ctx context.Context, id platform.ID, upd platform.OrganizationUpdate) (*platform.Organization, error) {
	u, err := newURL(s.Addr, organizationIDPath(id))
	if err != nil {
		return nil, err
	}

	octets, err := json.Marshal(upd)
	if err != nil {
		return nil, err
	}

	req, err := http.NewRequest("PATCH", u.String(), bytes.NewReader(octets))
	if err != nil {
		return nil, err
	}

	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("Authorization", s.Token)

	hc := newClient(u.Scheme, s.InsecureSkipVerify)

	resp, err := hc.Do(req)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf(resp.Header.Get("X-Influx-Error"))
	}

	var o platform.Organization
	if err := json.NewDecoder(resp.Body).Decode(&o); err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	return &o, nil
}

func (s *OrganizationService) DeleteOrganization(ctx context.Context, id platform.ID) error {
	u, err := newURL(s.Addr, organizationIDPath(id))
	if err != nil {
		return err
	}

	req, err := http.NewRequest("DELETE", u.String(), nil)
	if err != nil {
		return err
	}
	req.Header.Set("Authorization", s.Token)

	hc := newClient(u.Scheme, s.InsecureSkipVerify)
	resp, err := hc.Do(req)
	if err != nil {
		return err
	}

	switch resp.StatusCode {
	case http.StatusNoContent, http.StatusAccepted:
		return nil
	}

	return fmt.Errorf(resp.Header.Get("X-Influx-Error"))
}

func organizationIDPath(id platform.ID) string {
	return path.Join(organizationPath, id.String())
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/rjw57/aonui"
)

const maximumSimultaneousDownloads = 5

// Global semaphore used to limit the number of simultaneous downloads
var fetchSem = make(chan int, maximumSimultaneousDownloads)

type TemporaryFileSource struct {
	BaseDir string
	Prefix  string

	files []*os.File
}

func (tfs *TemporaryFileSource) Create() (*os.File, error) {
	f, err := ioutil.TempFile(tfs.BaseDir, tfs.Prefix)
	if err != nil {
		return nil, err
	}

	tfs.files = append(tfs.files, f)
	return f, nil
}

func (tfs *TemporaryFileSource) Remove(f *os.File) error {
	// Find index of f in files
	for fIdx := 0; fIdx < len(tfs.files); fIdx++ {
		if tfs.files[fIdx] != f {
			continue
		}

		// We found f, remove it from our list
		tfs.files = append(tfs.files[:fIdx], tfs.files[fIdx+1:]...)

		// Remove it from disk
		if err := os.Remove(f.Name()); err != nil {
			return err
		}
	}

	// If we get here, f was not in files
	return errors.New("Temporary file was not managed by me")
}

func (tfs *TemporaryFileSource) RemoveAll() error {
	var lastErr error

	for _, f := range tfs.files {
		if err := os.Remove(f.Name()); err != nil {
			lastErr = err
		}
	}

	return lastErr
}

// Command-line flags
var (
	baseDir string
	highRes bool
	maxRuns int
)

func main() {
	// Parse command line
	flag.StringVar(&baseDir, "basedir", ".", "directory to download data to")
	flag.BoolVar(&highRes, "highres", false, "download 0.25deg data as opposed to 0.5deg")
	flag.IntVar(&maxRuns, "maxruns", 3, "maximum number of runs to examine before giving up")
	flag.Parse()

	// Which source to use?
	src := aonui.GFSHalfDegreeDataset
	if highRes {
		src = aonui.GFSQuarterDegreeDataset
	}

	// Fetch all of the runs
	runs, err := src.FetchRuns()
	if err != nil {
		log.Fatal(err)
	}

	// Sort by *descending* date
	sort.Sort(sort.Reverse(ByDate(runs)))

	succeeded := false
	for _, run := range runs[:maxRuns] {
		destFn := filepath.Join(baseDir, run.Identifier+".grib2")

		if err := syncRun(run, destFn); err != nil {
			log.Print("error syncing run: ", err)

			// ensure we remove destFn if we created it
			if os.IsExist(err) {
				log.Print("Removing ", destFn)
				os.Remove(destFn)
			}
		} else {
			// success!
			log.Print("run downloaded successfully")
			succeeded = true
			break
		}
	}

	if !succeeded {
		log.Fatal("no runs were downloaded")
	}
}

func syncRun(run *aonui.Run, destFn string) error {
	log.Print("Fetching data for run at ", run.When)

	// Get datasets for this run
	datasets, err := run.FetchDatasets()
	if err != nil {
		return err
	}
	log.Print("Run has ", len(datasets), " dataset(s)")

	if len(datasets) < run.Source.MinDatasets {
		log.Print("Run has too few, expecting at least ", run.Source.MinDatasets)
		return errors.New("Too few datasets in source")
	}

	// File source for temporary files
	tfs := TemporaryFileSource{BaseDir: baseDir, Prefix: "dataset-"}
	defer tfs.RemoveAll()

	// Make sure to remove temporary files on keyboard interrupt
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		for s := range c {
			log.Printf("captured %v, deleting temporary files", s)
			tfs.RemoveAll()
			os.Exit(1)
		}
	}()

	// Open the output file
	log.Print("Fetching run to ", destFn)
	output, err := os.Create(destFn)
	if err != nil {
		log.Print("Error creating output: ", err)
		return err
	}

	// Ensure the file is closed on function exit
	defer output.Close()

	// Concatenate temporary files as they are finished
	fetchStart := time.Now()
	for f := range fetchDatasetsData(&tfs, datasets) {
		if input, err := os.Open(f.Name()); err != nil {
			log.Print("Error copying temporary file: ", err)
		} else {
			io.Copy(output, input)
			input.Close()
		}
		tfs.Remove(f)
	}

	fetchDuration := time.Since(fetchStart)
	fi, err := output.Stat()
	if err != nil {
		log.Print("Error: ", err)
		return err
	}
	log.Print(fmt.Sprintf("Overall download speed: %v/sec",
		ByteCount(float64(fi.Size())/fetchDuration.Seconds())))

	return nil
}

func fetchDatasetsData(tfs *TemporaryFileSource, datasets []*aonui.Dataset) chan *os.File {
	// Which records are we interested in?
	paramsOfInterest := []string{"HGT", "UGRD", "VGRD"}

	var wg sync.WaitGroup
	tmpFilesChan := make(chan *os.File)

	trySleepDuration, err := time.ParseDuration("10s")
	if err != nil {
		log.Fatal(err)
	}

	for _, ds := range datasets {
		// If we have a max forecast hour, and this dataset is later, skip
		if ds.Run.Source.MaxForecastHour > 0 && ds.ForecastHour > ds.Run.Source.MaxForecastHour {
			continue
		}

		wg.Add(1)

		go func(dataset *aonui.Dataset) {
			defer wg.Done()

			fetchSem <- 1
			defer func() { <-fetchSem }()

			// Perform download. Attempt download repeatedly
			maximumTries := dataset.Run.Source.FetchStrategy.MaximumRetries
			var tmpFile *os.File
			for tries := 0; tries < maximumTries; tries++ {
				// Create a temporary file for output
				tmpFile, err = tfs.Create()
				if err != nil {
					log.Print("Error creating temporary file: ", err)
				}

				log.Print("Fetching ", dataset.Identifier,
					" (try ", tries+1, " of ", maximumTries, ")")
				err := fetchDataset(tmpFile, dataset, paramsOfInterest)
				if err == nil {
					break
				} else {
					log.Print("Error fetching dataset: ", err)
				}

				// Remove this temporary file
				tmpFile.Close()
				tfs.Remove(tmpFile)
				tmpFile = nil

				// Sleep until the next try
				time.Sleep(trySleepDuration)
			}

			if tmpFile == nil {
				log.Print("error: failed to download ", dataset.Identifier)
			} else {
				tmpFile.Close()
				tmpFilesChan <- tmpFile
			}
		}(ds)
	}

	// Launch a goroutine to wait for all datasets to be downloaded and
	// then close the channel.
	go func() {
		wg.Wait()
		close(tmpFilesChan)
	}()

	return tmpFilesChan
}

func fetchDataset(output io.Writer, dataset *aonui.Dataset, paramsOfInterest []string) error {
	// Fetch inventory for this dataset
	inventory, err := dataset.FetchInventory()
	if err != nil {
		return err
	}

	// Calculate which items to save
	var (
		totalToFetch int64 = 0
		fetchItems   []*aonui.InventoryItem
	)
	for _, item := range inventory {
		saveItem := false
		for _, poi := range paramsOfInterest {
			for _, p := range item.Parameters {
				saveItem = saveItem || poi == p
			}
		}

		// HACK: we also are only interested in wind velocities at a
		// particular pressure. (i.e. ones whose "LayerName" field is of
		// the form "XXX mb".)
		saveItem = saveItem && strings.HasSuffix(item.LayerName, " mb")

		if saveItem {
			fetchItems = append(fetchItems, item)
			totalToFetch += item.Extent
		}
	}

	if len(fetchItems) == 0 {
		log.Print("No items to fetch")
		return nil
	}

	log.Print(fmt.Sprintf("Fetching %d records from %v (%v)",
		len(fetchItems), dataset.Identifier, ByteCount(totalToFetch)))
	if _, err := dataset.FetchAndWriteRecords(output, fetchItems); err != nil {
		return err
	}

	return nil
}
<commit_msg>aonuisync: don't overwrite existing files<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/rjw57/aonui"
)

const maximumSimultaneousDownloads = 5

// Global semaphore used to limit the number of simultaneous downloads
var fetchSem = make(chan int, maximumSimultaneousDownloads)

type TemporaryFileSource struct {
	BaseDir string
	Prefix  string

	files []*os.File
}

func (tfs *TemporaryFileSource) Create() (*os.File, error) {
	f, err := ioutil.TempFile(tfs.BaseDir, tfs.Prefix)
	if err != nil {
		return nil, err
	}

	tfs.files = append(tfs.files, f)
	return f, nil
}

func (tfs *TemporaryFileSource) Remove(f *os.File) error {
	// Find index of f in files
	for fIdx := 0; fIdx < len(tfs.files); fIdx++ {
		if tfs.files[fIdx] != f {
			continue
		}

		// We found f, remove it from our list
		tfs.files = append(tfs.files[:fIdx], tfs.files[fIdx+1:]...)

		// Remove it from disk
		if err := os.Remove(f.Name()); err != nil {
			return err
		}
	}

	// If we get here, f was not in files
	return errors.New("Temporary file was not managed by me")
}

func (tfs *TemporaryFileSource) RemoveAll() error {
	var lastErr error

	for _, f := range tfs.files {
		if err := os.Remove(f.Name()); err != nil {
			lastErr = err
		}
	}

	return lastErr
}

// Command-line flags
var (
	baseDir string
	highRes bool
	maxRuns int
)

func main() {
	// Parse command line
	flag.StringVar(&baseDir, "basedir", ".", "directory to download data to")
	flag.BoolVar(&highRes, "highres", false, "download 0.25deg data as opposed to 0.5deg")
	flag.IntVar(&maxRuns, "maxruns", 3, "maximum number of runs to examine before giving up")
	flag.Parse()

	// Which source to use?
	src := aonui.GFSHalfDegreeDataset
	if highRes {
		src = aonui.GFSQuarterDegreeDataset
	}

	// Fetch all of the runs
	runs, err := src.FetchRuns()
	if err != nil {
		log.Fatal(err)
	}

	// Sort by *descending* date
	sort.Sort(sort.Reverse(ByDate(runs)))

	succeeded := false
	for _, run := range runs[:maxRuns] {
		destFn := filepath.Join(baseDir, run.Identifier+".grib2")

		if _, err := os.Stat(destFn); err != nil {
			log.Print("not overwriting ", destFn)
			continue
		}

		if err := syncRun(run, destFn); err != nil {
			log.Print("error syncing run: ", err)

			// ensure we remove destFn if we created it
			if os.IsExist(err) {
				log.Print("Removing ", destFn)
				os.Remove(destFn)
			}
		} else {
			// success!
			log.Print("run downloaded successfully")
			succeeded = true
			break
		}
	}

	if !succeeded {
		log.Fatal("no runs were downloaded")
	}
}

func syncRun(run *aonui.Run, destFn string) error {
	log.Print("Fetching data for run at ", run.When)

	// Get datasets for this run
	datasets, err := run.FetchDatasets()
	if err != nil {
		return err
	}
	log.Print("Run has ", len(datasets), " dataset(s)")

	if len(datasets) < run.Source.MinDatasets {
		log.Print("Run has too few, expecting at least ", run.Source.MinDatasets)
		return errors.New("Too few datasets in source")
	}

	// File source for temporary files
	tfs := TemporaryFileSource{BaseDir: baseDir, Prefix: "dataset-"}
	defer tfs.RemoveAll()

	// Make sure to remove temporary files on keyboard interrupt
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		for s := range c {
			log.Printf("captured %v, deleting temporary files", s)
			tfs.RemoveAll()
			os.Exit(1)
		}
	}()

	// Open the output file
	log.Print("Fetching run to ", destFn)
	output, err := os.Create(destFn)
	if err != nil {
		log.Print("Error creating output: ", err)
		return err
	}

	// Ensure the file is closed on function exit
	defer output.Close()

	// Concatenate temporary files as they are finished
	fetchStart := time.Now()
	for f := range fetchDatasetsData(&tfs, datasets) {
		if input, err := os.Open(f.Name()); err != nil {
			log.Print("Error copying temporary file: ", err)
		} else {
			io.Copy(output, input)
			input.Close()
		}
		tfs.Remove(f)
	}

	fetchDuration := time.Since(fetchStart)
	fi, err := output.Stat()
	if err != nil {
		log.Print("Error: ", err)
		return err
	}
	log.Print(fmt.Sprintf("Overall download speed: %v/sec",
		ByteCount(float64(fi.Size())/fetchDuration.Seconds())))

	return nil
}

func fetchDatasetsData(tfs *TemporaryFileSource, datasets []*aonui.Dataset) chan *os.File {
	// Which records are we interested in?
	paramsOfInterest := []string{"HGT", "UGRD", "VGRD"}

	var wg sync.WaitGroup
	tmpFilesChan := make(chan *os.File)

	trySleepDuration, err := time.ParseDuration("10s")
	if err != nil {
		log.Fatal(err)
	}

	for _, ds := range datasets {
		// If we have a max forecast hour, and this dataset is later, skip
		if ds.Run.Source.MaxForecastHour > 0 && ds.ForecastHour > ds.Run.Source.MaxForecastHour {
			continue
		}

		wg.Add(1)

		go func(dataset *aonui.Dataset) {
			defer wg.Done()

			fetchSem <- 1
			defer func() { <-fetchSem }()

			// Perform download. Attempt download repeatedly
			maximumTries := dataset.Run.Source.FetchStrategy.MaximumRetries
			var tmpFile *os.File
			for tries := 0; tries < maximumTries; tries++ {
				// Create a temporary file for output
				tmpFile, err = tfs.Create()
				if err != nil {
					log.Print("Error creating temporary file: ", err)
				}

				log.Print("Fetching ", dataset.Identifier,
					" (try ", tries+1, " of ", maximumTries, ")")
				err := fetchDataset(tmpFile, dataset, paramsOfInterest)
				if err == nil {
					break
				} else {
					log.Print("Error fetching dataset: ", err)
				}

				// Remove this temporary file
				tmpFile.Close()
				tfs.Remove(tmpFile)
				tmpFile = nil

				// Sleep until the next try
				time.Sleep(trySleepDuration)
			}

			if tmpFile == nil {
				log.Print("error: failed to download ", dataset.Identifier)
			} else {
				tmpFile.Close()
				tmpFilesChan <- tmpFile
			}
		}(ds)
	}

	// Launch a goroutine to wait for all datasets to be downloaded and
	// then close the channel.
	go func() {
		wg.Wait()
		close(tmpFilesChan)
	}()

	return tmpFilesChan
}

func fetchDataset(output io.Writer, dataset *aonui.Dataset, paramsOfInterest []string) error {
	// Fetch inventory for this dataset
	inventory, err := dataset.FetchInventory()
	if err != nil {
		return err
	}

	// Calculate which items to save
	var (
		totalToFetch int64 = 0
		fetchItems   []*aonui.InventoryItem
	)
	for _, item := range inventory {
		saveItem := false
		for _, poi := range paramsOfInterest {
			for _, p := range item.Parameters {
				saveItem = saveItem || poi == p
			}
		}

		// HACK: we also are only interested in wind velocities at a
		// particular pressure. (i.e. ones whose "LayerName" field is of
		// the form "XXX mb".)
		saveItem = saveItem && strings.HasSuffix(item.LayerName, " mb")

		if saveItem {
			fetchItems = append(fetchItems, item)
			totalToFetch += item.Extent
		}
	}

	if len(fetchItems) == 0 {
		log.Print("No items to fetch")
		return nil
	}

	log.Print(fmt.Sprintf("Fetching %d records from %v (%v)",
		len(fetchItems), dataset.Identifier, ByteCount(totalToFetch)))
	if _, err := dataset.FetchAndWriteRecords(output, fetchItems); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2019 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"log"
	"net"
	"net/http"
	"net/rpc"
	"os/exec"
	"strconv"

	"github.com/google/gerritfmt"
)

func main() {
	port := flag.Int("port", 4001, "port to serve on")
	java := flag.String("java_jar", "", "where to find the google-java-format jar")
	buildifier := flag.String("buildifier", "", "where to find the buildifier")
	gofmt := flag.String("gofmt", "", "where to find gofmt")

	*java, _ = exec.LookPath("google-java-format-all-deps.jar")
	*buildifier, _ = exec.LookPath("buildifier")
	*gofmt, _ = exec.LookPath("gofmt")
	flag.Parse()

	s := gerritfmt.NewServer()
	var err error
	s.JavaJar = *java
	s.Buildifier = *buildifier
	s.Gofmt = *gofmt
	if err != nil {
		log.Fatal(err)
	}
	rpc.Register(s)
	rpc.HandleHTTP()

	addr := ":" + strconv.Itoa(*port)
	log.Printf("fmtserver serving on %s", addr)

	l, err := net.Listen("tcp", addr)
	if err != nil {
		log.Fatal("listen error:", err)
	}
	http.Serve(l, nil)

	log.Fatal(http.ListenAndServe(addr, nil))
}
<commit_msg>cmd/fmtserver: delete obsolete file<commit_after><|endoftext|>


<commit_before>package main

import (
	"compress/gzip"
	"errors"
	"fmt"
	"io"
	"net/http"
	"strings"
	"sync"
	"time"

	"github.com/heroku/authenticater"
	metrics "github.com/rcrowley/go-metrics"
	log "github.com/sirupsen/logrus"
)

type payload struct {
	SourceAddr string
	RequestID  string
	Body       []byte
	WaitCh     chan struct{}
}

func NewPayload(sa string, ri string, b []byte) payload {
	return payload{
		SourceAddr: sa,
		RequestID:  ri,
		Body:       b,
		WaitCh:     make(chan struct{}, 1),
	}
}

type FixerFunc func(*http.Request, io.Reader, string, string, string) (bool, int64, []byte, error)

type httpServer struct {
	Config                IssConfig
	FixerFunc             FixerFunc
	shutdownCh            shutdownCh
	deliverer             deliverer
	isShuttingDown        bool
	auth                  authenticater.Authenticater
	posts                 metrics.Timer   // tracks metrics about posts
	healthChecks          metrics.Timer   // tracks metrics about health checks
	pErrors               metrics.Counter // tracks the count of post errors
	pSuccesses            metrics.Counter // tracks the number of post successes
	pAuthErrors           metrics.Counter // tracks the count of auth errors
	pAuthSuccesses        metrics.Counter // tracks the number of auth successes
	pMetadataLogsReceived metrics.Counter // tracks the number of logs that have metadata that have been received
	pLogsReceived         metrics.Counter // tracks the number of logs that have been received
	pMetadataLogsSent     metrics.Counter // tracks the number of logs that have metadata that have been received
	pLogsSent             metrics.Counter // tracks the number of logs that have been received
	pAuthUsers            map[string]metrics.Counter
	sync.WaitGroup
}

func newHTTPServer(config IssConfig, auth authenticater.Authenticater, fixerFunc FixerFunc, deliverer deliverer) *httpServer {
	return &httpServer{
		auth:                  auth,
		Config:                config,
		FixerFunc:             fixerFunc,
		deliverer:             deliverer,
		shutdownCh:            make(shutdownCh),
		posts:                 metrics.GetOrRegisterTimer("log-iss.http.logs", config.MetricsRegistry),
		healthChecks:          metrics.GetOrRegisterTimer("log-iss.http.healthchecks", config.MetricsRegistry),
		pErrors:               metrics.GetOrRegisterCounter("log-iss.http.logs.errors", config.MetricsRegistry),
		pSuccesses:            metrics.GetOrRegisterCounter("log-iss.http.logs.successes", config.MetricsRegistry),
		pAuthErrors:           metrics.GetOrRegisterCounter("log-iss.auth.errors", config.MetricsRegistry),
		pAuthSuccesses:        metrics.GetOrRegisterCounter("log-iss.auth.successes", config.MetricsRegistry),
		pMetadataLogsReceived: metrics.GetOrRegisterCounter("log-iss.metadata_logs.received", config.MetricsRegistry),
		pLogsReceived:         metrics.GetOrRegisterCounter("log-iss.logs.received", config.MetricsRegistry),
		pMetadataLogsSent:     metrics.GetOrRegisterCounter("log-iss.metadata_logs.sent", config.MetricsRegistry),
		pLogsSent:             metrics.GetOrRegisterCounter("log-iss.logs.sent", config.MetricsRegistry),
		pAuthUsers:            make(map[string]metrics.Counter),
		isShuttingDown:        false,
	}
}

func (s *httpServer) handleHTTPError(w http.ResponseWriter, errMsg string, errCode int, fields ...log.Fields) {
	ff := log.Fields{"post.code": errCode}
	for _, f := range fields {
		for k, v := range f {
			ff[k] = v
		}
	}

	s.pErrors.Inc(1)
	log.WithFields(ff).Error(errMsg)
	http.Error(w, errMsg, errCode)
}

func extractRemoteAddr(r *http.Request) string {
	remoteAddr := r.Header.Get("X-Forwarded-For")
	if remoteAddr == "" {
		remoteAddrParts := strings.Split(r.RemoteAddr, ":")
		remoteAddr = strings.Join(remoteAddrParts[:len(remoteAddrParts)-1], ":")
	}
	return remoteAddr
}

func (s *httpServer) Run() error {
	go s.awaitShutdown()

	//FXME: check outlet depth?
	http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
		defer s.healthChecks.UpdateSince(time.Now())
		if s.isShuttingDown {
			http.Error(w, "Shutting down", 503)
			return
		}

	})

	http.HandleFunc("/logs", func(w http.ResponseWriter, r *http.Request) {
		defer s.posts.UpdateSince(time.Now())

		if s.Config.EnforceSsl && r.Header.Get("X-Forwarded-Proto") != "https" {
			s.handleHTTPError(w, "Only SSL requests accepted", 400)
			return
		}

		if s.isShuttingDown {
			s.handleHTTPError(w, "Shutting down", 503)
			return
		}

		if r.Method != "POST" {
			s.handleHTTPError(w, "Only POST is accepted", 400)
			return
		}

		if r.Header.Get("Content-Type") != "application/logplex-1" {
			s.handleHTTPError(w, "Only Content-Type application/logplex-1 is accepted", 400)
			return
		}

		if !s.auth.Authenticate(r) {
			s.pAuthErrors.Inc(1)
			s.handleHTTPError(w, "Unable to authenticate request", 401)
			return
		} else {
			s.pAuthSuccesses.Inc(1)
		}

		remoteAddr := extractRemoteAddr(r)
		requestID := r.Header.Get("X-Request-Id")
		logplexDrainToken := r.Header.Get("Logplex-Drain-Token")

		body := r.Body
		var err error

		if r.Header.Get("Content-Encoding") == "gzip" {
			body, err = gzip.NewReader(r.Body)
			if err != nil {
				s.handleHTTPError(w, "Could not decode gzip request", 500)
				return
			}
			defer body.Close()
		}

		// This should only be reached if authentication information is valid.
		if authUser, _, ok := r.BasicAuth(); ok {
			var um metrics.Counter
			um, ok = s.pAuthUsers[authUser]
			if !ok {
				if s.Config.Debug {
					fmt.Printf("DEBUG: create: log-iss.auth.user.%s\n", authUser)
				}
				um = metrics.GetOrRegisterCounter(fmt.Sprintf("log-iss.auth.user.%s", authUser), s.Config.MetricsRegistry)
				s.pAuthUsers[authUser] = um
			}

			if s.Config.Debug {
				fmt.Printf("DEBUG: log-iss.auth.user.%s++\n", authUser)
			}
			um.Inc(1)
		}

		if err, status := s.process(r, body, remoteAddr, requestID, logplexDrainToken, s.Config.MetadataId); err != nil {
			s.handleHTTPError(
				w, err.Error(), status,
				log.Fields{"remote_addr": remoteAddr, "requestId": requestID, "logdrain_token": logplexDrainToken},
			)
			return
		}

		s.pSuccesses.Inc(1)
	})

	return http.ListenAndServe(":"+s.Config.HttpPort, nil)
}

func (s *httpServer) awaitShutdown() {
	<-s.shutdownCh
	s.isShuttingDown = true
	log.WithFields(log.Fields{"ns": "http", "at": "shutdown"}).Info()
}

func (s *httpServer) process(req *http.Request, r io.Reader, remoteAddr string, requestID string, logplexDrainToken string, metadataId string) (error, int) {
	s.Add(1)
	defer s.Done()

	hasMetadata, numLogs, fixedBody, err := s.FixerFunc(req, r, remoteAddr, logplexDrainToken, metadataId)
	if err != nil {
		return errors.New("Problem fixing body: " + err.Error()), http.StatusBadRequest
	}

	s.pLogsReceived.Inc(numLogs)
	if hasMetadata {
		s.pMetadataLogsReceived.Inc(numLogs)
	}

	payload := NewPayload(remoteAddr, requestID, fixedBody)
	if err := s.deliverer.Deliver(payload); err != nil {
		return errors.New("Problem delivering body: " + err.Error()), http.StatusGatewayTimeout
	}

	s.pLogsSent.Inc(numLogs)
	if hasMetadata {
		s.pMetadataLogsSent.Inc(numLogs)
	}

	return nil, 200
}
<commit_msg>Adding in-stream token replacement<commit_after>package main

import (
	"compress/gzip"
	"errors"
	"fmt"
	"io"
	"net/http"
	"strings"
	"sync"
	"time"

	"github.com/heroku/authenticater"
	metrics "github.com/rcrowley/go-metrics"
	log "github.com/sirupsen/logrus"
)

type payload struct {
	SourceAddr string
	RequestID  string
	Body       []byte
	WaitCh     chan struct{}
}

func NewPayload(sa string, ri string, b []byte) payload {
	return payload{
		SourceAddr: sa,
		RequestID:  ri,
		Body:       b,
		WaitCh:     make(chan struct{}, 1),
	}
}

type FixerFunc func(*http.Request, io.Reader, string, string, string) (bool, int64, []byte, error)

type httpServer struct {
	Config                IssConfig
	FixerFunc             FixerFunc
	shutdownCh            shutdownCh
	deliverer             deliverer
	isShuttingDown        bool
	auth                  authenticater.Authenticater
	posts                 metrics.Timer   // tracks metrics about posts
	healthChecks          metrics.Timer   // tracks metrics about health checks
	pErrors               metrics.Counter // tracks the count of post errors
	pSuccesses            metrics.Counter // tracks the number of post successes
	pAuthErrors           metrics.Counter // tracks the count of auth errors
	pAuthSuccesses        metrics.Counter // tracks the number of auth successes
	pMetadataLogsReceived metrics.Counter // tracks the number of logs that have metadata that have been received
	pLogsReceived         metrics.Counter // tracks the number of logs that have been received
	pMetadataLogsSent     metrics.Counter // tracks the number of logs that have metadata that have been received
	pLogsSent             metrics.Counter // tracks the number of logs that have been received
	pAuthUsers            map[string]metrics.Counter
	sync.WaitGroup
}

func newHTTPServer(config IssConfig, auth authenticater.Authenticater, fixerFunc FixerFunc, deliverer deliverer) *httpServer {
	return &httpServer{
		auth:                  auth,
		Config:                config,
		FixerFunc:             fixerFunc,
		deliverer:             deliverer,
		shutdownCh:            make(shutdownCh),
		posts:                 metrics.GetOrRegisterTimer("log-iss.http.logs", config.MetricsRegistry),
		healthChecks:          metrics.GetOrRegisterTimer("log-iss.http.healthchecks", config.MetricsRegistry),
		pErrors:               metrics.GetOrRegisterCounter("log-iss.http.logs.errors", config.MetricsRegistry),
		pSuccesses:            metrics.GetOrRegisterCounter("log-iss.http.logs.successes", config.MetricsRegistry),
		pAuthErrors:           metrics.GetOrRegisterCounter("log-iss.auth.errors", config.MetricsRegistry),
		pAuthSuccesses:        metrics.GetOrRegisterCounter("log-iss.auth.successes", config.MetricsRegistry),
		pMetadataLogsReceived: metrics.GetOrRegisterCounter("log-iss.metadata_logs.received", config.MetricsRegistry),
		pLogsReceived:         metrics.GetOrRegisterCounter("log-iss.logs.received", config.MetricsRegistry),
		pMetadataLogsSent:     metrics.GetOrRegisterCounter("log-iss.metadata_logs.sent", config.MetricsRegistry),
		pLogsSent:             metrics.GetOrRegisterCounter("log-iss.logs.sent", config.MetricsRegistry),
		pAuthUsers:            make(map[string]metrics.Counter),
		isShuttingDown:        false,
	}
}

func (s *httpServer) handleHTTPError(w http.ResponseWriter, errMsg string, errCode int, fields ...log.Fields) {
	ff := log.Fields{"post.code": errCode}
	for _, f := range fields {
		for k, v := range f {
			ff[k] = v
		}
	}

	s.pErrors.Inc(1)
	log.WithFields(ff).Error(errMsg)
	http.Error(w, errMsg, errCode)
}

func extractRemoteAddr(r *http.Request) string {
	remoteAddr := r.Header.Get("X-Forwarded-For")
	if remoteAddr == "" {
		remoteAddrParts := strings.Split(r.RemoteAddr, ":")
		remoteAddr = strings.Join(remoteAddrParts[:len(remoteAddrParts)-1], ":")
	}
	return remoteAddr
}

func (s *httpServer) Run() error {
	go s.awaitShutdown()

	//FXME: check outlet depth?
	http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
		defer s.healthChecks.UpdateSince(time.Now())
		if s.isShuttingDown {
			http.Error(w, "Shutting down", 503)
			return
		}

	})

	http.HandleFunc("/logs", func(w http.ResponseWriter, r *http.Request) {
		defer s.posts.UpdateSince(time.Now())

		if s.Config.EnforceSsl && r.Header.Get("X-Forwarded-Proto") != "https" {
			s.handleHTTPError(w, "Only SSL requests accepted", 400)
			return
		}

		if s.isShuttingDown {
			s.handleHTTPError(w, "Shutting down", 503)
			return
		}

		if r.Method != "POST" {
			s.handleHTTPError(w, "Only POST is accepted", 400)
			return
		}

		if r.Header.Get("Content-Type") != "application/logplex-1" {
			s.handleHTTPError(w, "Only Content-Type application/logplex-1 is accepted", 400)
			return
		}

		if !s.auth.Authenticate(r) {
			s.pAuthErrors.Inc(1)
			s.handleHTTPError(w, "Unable to authenticate request", 401)
			return
		} else {
			s.pAuthSuccesses.Inc(1)
		}

		remoteAddr := extractRemoteAddr(r)
		requestID := r.Header.Get("X-Request-Id")
		logplexDrainToken := r.Header.Get("Logplex-Drain-Token")

		body := r.Body
		var err error

		if r.Header.Get("Content-Encoding") == "gzip" {
			body, err = gzip.NewReader(r.Body)
			if err != nil {
				s.handleHTTPError(w, "Could not decode gzip request", 500)
				return
			}
			defer body.Close()
		}

		// This should only be reached if authentication information is valid.
		if authUser, _, ok := r.BasicAuth(); ok {
			var um metrics.Counter
			um, ok = s.pAuthUsers[authUser]
			if !ok {
				if s.Config.Debug {
					fmt.Printf("DEBUG: create: log-iss.auth.user.%s\n", authUser)
				}
				um = metrics.GetOrRegisterCounter(fmt.Sprintf("log-iss.auth.user.%s", authUser), s.Config.MetricsRegistry)
				s.pAuthUsers[authUser] = um
			}

			if s.Config.Debug {
				fmt.Printf("DEBUG: log-iss.auth.user.%s++\n", authUser)
			}
			um.Inc(1)
		}

		if err, status := s.process(r, body, remoteAddr, requestID, logplexDrainToken, s.Config.MetadataId); err != nil {
			s.handleHTTPError(
				w, err.Error(), status,
				log.Fields{"remote_addr": remoteAddr, "requestId": requestID, "logdrain_token": logplexDrainToken},
			)
			return
		}

		s.pSuccesses.Inc(1)
	})

	return http.ListenAndServe(":"+s.Config.HttpPort, nil)
}

func (s *httpServer) awaitShutdown() {
	<-s.shutdownCh
	s.isShuttingDown = true
	log.WithFields(log.Fields{"ns": "http", "at": "shutdown"}).Info()
}

func (s *httpServer) process(req *http.Request, r io.Reader, remoteAddr string, requestID string, logplexDrainToken string, metadataId string) (error, int) {
	s.Add(1)
	defer s.Done()

	hasMetadata, numLogs, fixedBody, err := s.FixerFunc(req, r, remoteAddr, logplexDrainToken, metadataId)
	if err != nil {
		return errors.New("Problem fixing body: " + err.Error()), http.StatusBadRequest
	}

	s.pLogsReceived.Inc(numLogs)
	if hasMetadata {
		s.pMetadataLogsReceived.Inc(numLogs)
	}

	// Scrub tokens from fixedBody
	for user, token := range s.Config.TokenMap() {
		t := []byte(token)
		if bytes.Contains(fixedBody, t) {
			fixedBody = bytes.Replace(fixedBody, t, []byte(user), -1)
		}
	}

	payload := NewPayload(remoteAddr, requestID, fixedBody)
	if err := s.deliverer.Deliver(payload); err != nil {
		return errors.New("Problem delivering body: " + err.Error()), http.StatusGatewayTimeout
	}

	s.pLogsSent.Inc(numLogs)
	if hasMetadata {
		s.pMetadataLogsSent.Inc(numLogs)
	}

	return nil, 200
}
<|endoftext|>


<commit_before>package crosscoap

import (
	"net"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"

	"github.com/dustin/go-coap"
)

func TestProxyWithConfirmableRequest(t *testing.T) {
	const backendResponse = "This is the response text"
	const backendStatus = 404
	backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Method != "POST" {
			t.Errorf("backend got method %q, want %q", r.Method, "POST")
		}
		if r.URL.String() != "/base/dir/some/path" {
			t.Errorf("backend got URL %q, want %q", r.URL, "/base/dir/some/path")
		}
		if len(r.TransferEncoding) > 0 {
			t.Errorf("backend got unexpected Transfer-Encoding: %v", r.TransferEncoding)
		}
		if r.UserAgent() != "crosscoap/1.0" {
			t.Errorf("backend got unexpected User-Agent: %v", r.UserAgent())
		}
		//if r.Header.Get("X-Forwarded-For") == "" {
		//	t.Errorf("didn't get X-Forwarded-For header")
		//}
		w.Header().Set("Content-Type", "application/xml")
		w.WriteHeader(backendStatus)
		w.Write([]byte(backendResponse))
	}))
	defer backend.Close()

	udpListener, crosscoapAddr := createLocalUDPListener(t)
	defer udpListener.Close()
	proxy := Proxy{Listener: udpListener, BackendURL: backend.URL + "/base/dir"}
	go proxy.Serve()

	req := coap.Message{
		Type:      coap.Confirmable,
		Code:      coap.POST,
		MessageID: 12345,
		Payload:   []byte(`{"key":"Content of CoAP packet payload"}`),
	}
	req.SetPathString("/some/path")
	req.SetOption(coap.ContentFormat, coap.AppJSON)

	c, err := coap.Dial("udp", crosscoapAddr)
	if err != nil {
		t.Fatalf("Error dialing: %v", err)
	}
	rv, err := c.Send(req)
	if err != nil {
		t.Fatalf("Error sending request: %v", err)
	}
	if rv == nil {
		t.Fatalf("Didn't receive CoAP response")
	}

	if string(rv.Payload) != backendResponse {
		t.Errorf("got body %q; expected %q", string(rv.Payload), backendResponse)
	}
	if rv.Code != coap.NotFound {
		t.Errorf("got CoAP code %v; expected %v", rv.Code, coap.NotFound)
	}
	if rv.Option(coap.ContentFormat) != coap.AppXML {
		t.Errorf("got content format %v; expected %v", rv.Option(coap.ContentFormat), coap.AppXML)
	}
}

func TestProxyWithExternalServer(t *testing.T) {
	udpListener, crosscoapAddr := createLocalUDPListener(t)
	defer udpListener.Close()
	proxy := Proxy{Listener: udpListener, BackendURL: "https://api.github.com/"}
	go proxy.Serve()

	req := coap.Message{
		Type:      coap.Confirmable,
		Code:      coap.GET,
		MessageID: 9876,
	}
	req.SetPathString("/users/octocat")

	c, err := coap.Dial("udp", crosscoapAddr)
	if err != nil {
		t.Fatalf("Error dialing: %v", err)
	}
	rv, err := c.Send(req)
	if err != nil {
		t.Fatalf("Error sending request: %v", err)
	}
	if rv == nil {
		t.Fatalf("Didn't receive CoAP response")
	}

	if !strings.Contains(string(rv.Payload), `"name":"The Octocat"`) {
		t.Errorf("got body %q which doesn't include the required string", string(rv.Payload))
	}
	if rv.Code != coap.Content {
		t.Errorf("got CoAP code %v; expected %v", rv.Code, coap.Content)
	}
	if rv.Option(coap.ContentFormat) != coap.AppJSON {
		t.Errorf("got content format %v; expected %v", rv.Option(coap.ContentFormat), coap.AppJSON)
	}
}

func createLocalUDPListener(t *testing.T) (*net.UDPConn, string) {
	udpAddr, err := net.ResolveUDPAddr("udp", "127.0.0.1:0")
	if err != nil {
		t.Fatal("Can't resolve UDP addr")
	}
	udpListener, err := net.ListenUDP("udp", udpAddr)
	if err != nil {
		t.Fatal("Can't listen on UDP")
	}
	listenerAddr := udpListener.LocalAddr().String()
	return udpListener, listenerAddr
}
<commit_msg>test: Use Amazon S3 as the backend HTTPS server for tests<commit_after>package crosscoap

import (
	"net"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"

	"github.com/dustin/go-coap"
)

func TestProxyWithConfirmableRequest(t *testing.T) {
	const backendResponse = "This is the response text"
	const backendStatus = 404
	backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Method != "POST" {
			t.Errorf("backend got method %q, want %q", r.Method, "POST")
		}
		if r.URL.String() != "/base/dir/some/path" {
			t.Errorf("backend got URL %q, want %q", r.URL, "/base/dir/some/path")
		}
		if len(r.TransferEncoding) > 0 {
			t.Errorf("backend got unexpected Transfer-Encoding: %v", r.TransferEncoding)
		}
		if r.UserAgent() != "crosscoap/1.0" {
			t.Errorf("backend got unexpected User-Agent: %v", r.UserAgent())
		}
		//if r.Header.Get("X-Forwarded-For") == "" {
		//	t.Errorf("didn't get X-Forwarded-For header")
		//}
		w.Header().Set("Content-Type", "application/xml")
		w.WriteHeader(backendStatus)
		w.Write([]byte(backendResponse))
	}))
	defer backend.Close()

	udpListener, crosscoapAddr := createLocalUDPListener(t)
	defer udpListener.Close()
	proxy := Proxy{Listener: udpListener, BackendURL: backend.URL + "/base/dir"}
	go proxy.Serve()

	req := coap.Message{
		Type:      coap.Confirmable,
		Code:      coap.POST,
		MessageID: 12345,
		Payload:   []byte(`{"key":"Content of CoAP packet payload"}`),
	}
	req.SetPathString("/some/path")
	req.SetOption(coap.ContentFormat, coap.AppJSON)

	c, err := coap.Dial("udp", crosscoapAddr)
	if err != nil {
		t.Fatalf("Error dialing: %v", err)
	}
	rv, err := c.Send(req)
	if err != nil {
		t.Fatalf("Error sending request: %v", err)
	}
	if rv == nil {
		t.Fatalf("Didn't receive CoAP response")
	}

	if string(rv.Payload) != backendResponse {
		t.Errorf("got body %q; expected %q", string(rv.Payload), backendResponse)
	}
	if rv.Code != coap.NotFound {
		t.Errorf("got CoAP code %v; expected %v", rv.Code, coap.NotFound)
	}
	if rv.Option(coap.ContentFormat) != coap.AppXML {
		t.Errorf("got content format %v; expected %v", rv.Option(coap.ContentFormat), coap.AppXML)
	}
}

func TestProxyWithExternalServer(t *testing.T) {
	udpListener, crosscoapAddr := createLocalUDPListener(t)
	defer udpListener.Close()
	proxy := Proxy{Listener: udpListener, BackendURL: "https://s3.amazonaws.com/"}
	go proxy.Serve()

	req := coap.Message{
		Type:      coap.Confirmable,
		Code:      coap.GET,
		MessageID: 9876,
	}
	req.SetPathString("/non-existing-test-s3-bucket/coap-test/file")

	c, err := coap.Dial("udp", crosscoapAddr)
	if err != nil {
		t.Fatalf("Error dialing: %v", err)
	}
	rv, err := c.Send(req)
	if err != nil {
		t.Fatalf("Error sending request: %v", err)
	}
	if rv == nil {
		t.Fatalf("Didn't receive CoAP response")
	}

	if !strings.Contains(string(rv.Payload), "NoSuchBucket") {
		t.Errorf("got body %q which doesn't include the required string", string(rv.Payload))
	}
	if rv.Code != coap.NotFound {
		t.Errorf("got CoAP code %v; expected %v", rv.Code, coap.NotFound)
	}
	if rv.Option(coap.ContentFormat) != coap.AppXML {
		t.Errorf("got content format %v; expected %v", rv.Option(coap.ContentFormat), coap.AppXML)
	}
}

func createLocalUDPListener(t *testing.T) (*net.UDPConn, string) {
	udpAddr, err := net.ResolveUDPAddr("udp", "127.0.0.1:0")
	if err != nil {
		t.Fatal("Can't resolve UDP addr")
	}
	udpListener, err := net.ListenUDP("udp", udpAddr)
	if err != nil {
		t.Fatal("Can't listen on UDP")
	}
	listenerAddr := udpListener.LocalAddr().String()
	return udpListener, listenerAddr
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	"os"
	"os/signal"
	"path/filepath"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
	process "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess"
	homedir "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/mitchellh/go-homedir"
	fsnotify "github.com/jbenet/go-ipfs/Godeps/_workspace/src/gopkg.in/fsnotify.v1"
	commands "github.com/jbenet/go-ipfs/commands"
	core "github.com/jbenet/go-ipfs/core"
	corehttp "github.com/jbenet/go-ipfs/core/corehttp"
	coreunix "github.com/jbenet/go-ipfs/core/coreunix"
	config "github.com/jbenet/go-ipfs/repo/config"
	fsrepo "github.com/jbenet/go-ipfs/repo/fsrepo"
)

var http = flag.Bool("http", false, "expose IPFS HTTP API")
var repoPath = flag.String("repo", os.Getenv("IPFS_PATH"), "IPFS_PATH to use")
var watchPath = flag.String("path", ".", "the path to watch")

func main() {
	flag.Parse()

	// precedence
	// 1. --repo flag
	// 2. IPFS_PATH environment variable
	// 3. default repo path
	ipfsPath := config.DefaultPathRoot
	if *repoPath != "" {
		ipfsPath = *repoPath
	}

	if err := run(ipfsPath, *watchPath); err != nil {
		log.Fatal(err)
	}
}

func run(ipfsPath, watchPath string) error {

	proc := process.WithParent(process.Background())
	log.Printf("running IPFSWatch on %s using repo at %s...", watchPath, ipfsPath)

	ipfsPath, err := homedir.Expand(ipfsPath)
	if err != nil {
		return err
	}
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return err
	}
	defer watcher.Close()

	if err := addTree(watcher, watchPath); err != nil {
		return err
	}

	r := fsrepo.At(ipfsPath)
	if err := r.Open(); err != nil {
		// TODO handle case: daemon running
		// TODO handle case: repo doesn't exist or isn't initialized
		return err
	}
	node, err := core.NewIPFSNode(context.Background(), core.Online(r))
	if err != nil {
		return err
	}
	defer node.Close()

	if *http {
		maddr, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/5001")
		if err != nil {
			return err
		}
		var opts = []corehttp.ServeOption{
			corehttp.GatewayOption,
			corehttp.WebUIOption,
			corehttp.CommandsOption(cmdCtx(node, ipfsPath)),
		}
		proc.Go(func(p process.Process) {
			if err := corehttp.ListenAndServe(node, maddr, opts...); err != nil {
				return
			}
		})
	}

	interrupts := make(chan os.Signal)
	signal.Notify(interrupts, os.Interrupt, os.Kill)

	for {
		select {
		case <-interrupts:
			return nil
		case e := <-watcher.Events:
			log.Printf("received event: %s", e)
			isDir, err := IsDirectory(e.Name)
			if err != nil {
				continue
			}
			switch e.Op {
			case fsnotify.Remove:
				if isDir {
					if err := watcher.Remove(e.Name); err != nil {
						return err
					}
				}
			default:
				// all events except for Remove result in an IPFS.Add, but only
				// directory creation triggers a new watch
				switch e.Op {
				case fsnotify.Create:
					if isDir {
						addTree(watcher, e.Name)
					}
				}
				func() {
					file, err := os.Open(e.Name)
					if err != nil {
						log.Println(err)
					}
					defer file.Close()
					k, err := coreunix.Add(node, file)
					if err != nil {
						log.Println(err)
					}
					log.Printf("added %s... key: %s", e.Name, k)
				}()
			}
		case err := <-watcher.Errors:
			log.Println(err)
		}
	}
	return nil
}

func addTree(w *fsnotify.Watcher, root string) error {
	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
		isDir, err := IsDirectory(path)
		if err != nil {
			log.Println(err)
			return nil
		}
		switch {
		case isDir && IsHidden(path):
			log.Println(path)
			return filepath.SkipDir
		case isDir:
			log.Println(path)
			if err := w.Add(path); err != nil {
				return err
			}
		default:
			return nil
		}
		return nil
	})
	if err != nil {
		return err
	}
	return nil
}

func IsDirectory(path string) (bool, error) {
	fileInfo, err := os.Stat(path)
	return fileInfo.IsDir(), err
}

func IsHidden(path string) bool {
	path = filepath.Base(path)
	if path == "." || path == "" {
		return false
	}
	if rune(path[0]) == rune('.') {
		return true
	}
	return false
}

func cmdCtx(node *core.IpfsNode, repoPath string) commands.Context {
	return commands.Context{
		// TODO deprecate this shit
		Context:    context.Background(),
		Online:     true,
		ConfigRoot: repoPath,
		LoadConfig: func(path string) (*config.Config, error) {
			return node.Repo.Config(), nil
		},
		ConstructNode: func() (*core.IpfsNode, error) {
			return node, nil
		},
	}
}
<commit_msg>add async<commit_after>package main

import (
	"flag"
	"log"
	"os"
	"os/signal"
	"path/filepath"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	ma "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
	process "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess"
	homedir "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/mitchellh/go-homedir"
	fsnotify "github.com/jbenet/go-ipfs/Godeps/_workspace/src/gopkg.in/fsnotify.v1"
	commands "github.com/jbenet/go-ipfs/commands"
	core "github.com/jbenet/go-ipfs/core"
	corehttp "github.com/jbenet/go-ipfs/core/corehttp"
	coreunix "github.com/jbenet/go-ipfs/core/coreunix"
	config "github.com/jbenet/go-ipfs/repo/config"
	fsrepo "github.com/jbenet/go-ipfs/repo/fsrepo"
)

var http = flag.Bool("http", false, "expose IPFS HTTP API")
var repoPath = flag.String("repo", os.Getenv("IPFS_PATH"), "IPFS_PATH to use")
var watchPath = flag.String("path", ".", "the path to watch")

func main() {
	flag.Parse()

	// precedence
	// 1. --repo flag
	// 2. IPFS_PATH environment variable
	// 3. default repo path
	ipfsPath := config.DefaultPathRoot
	if *repoPath != "" {
		ipfsPath = *repoPath
	}

	if err := run(ipfsPath, *watchPath); err != nil {
		log.Fatal(err)
	}
}

func run(ipfsPath, watchPath string) error {

	proc := process.WithParent(process.Background())
	log.Printf("running IPFSWatch on %s using repo at %s...", watchPath, ipfsPath)

	ipfsPath, err := homedir.Expand(ipfsPath)
	if err != nil {
		return err
	}
	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		return err
	}
	defer watcher.Close()

	if err := addTree(watcher, watchPath); err != nil {
		return err
	}

	r := fsrepo.At(ipfsPath)
	if err := r.Open(); err != nil {
		// TODO handle case: daemon running
		// TODO handle case: repo doesn't exist or isn't initialized
		return err
	}
	node, err := core.NewIPFSNode(context.Background(), core.Online(r))
	if err != nil {
		return err
	}
	defer node.Close()

	if *http {
		maddr, err := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/5001")
		if err != nil {
			return err
		}
		var opts = []corehttp.ServeOption{
			corehttp.GatewayOption,
			corehttp.WebUIOption,
			corehttp.CommandsOption(cmdCtx(node, ipfsPath)),
		}
		proc.Go(func(p process.Process) {
			if err := corehttp.ListenAndServe(node, maddr, opts...); err != nil {
				return
			}
		})
	}

	interrupts := make(chan os.Signal)
	signal.Notify(interrupts, os.Interrupt, os.Kill)

	for {
		select {
		case <-interrupts:
			return nil
		case e := <-watcher.Events:
			log.Printf("received event: %s", e)
			isDir, err := IsDirectory(e.Name)
			if err != nil {
				continue
			}
			switch e.Op {
			case fsnotify.Remove:
				if isDir {
					if err := watcher.Remove(e.Name); err != nil {
						return err
					}
				}
			default:
				// all events except for Remove result in an IPFS.Add, but only
				// directory creation triggers a new watch
				switch e.Op {
				case fsnotify.Create:
					if isDir {
						addTree(watcher, e.Name)
					}
				}
				proc.Go(func(p process.Process) {
					file, err := os.Open(e.Name)
					if err != nil {
						log.Println(err)
					}
					defer file.Close()
					k, err := coreunix.Add(node, file)
					if err != nil {
						log.Println(err)
					}
					log.Printf("added %s... key: %s", e.Name, k)
				})
			}
		case err := <-watcher.Errors:
			log.Println(err)
		}
	}
	return nil
}

func addTree(w *fsnotify.Watcher, root string) error {
	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
		isDir, err := IsDirectory(path)
		if err != nil {
			log.Println(err)
			return nil
		}
		switch {
		case isDir && IsHidden(path):
			log.Println(path)
			return filepath.SkipDir
		case isDir:
			log.Println(path)
			if err := w.Add(path); err != nil {
				return err
			}
		default:
			return nil
		}
		return nil
	})
	if err != nil {
		return err
	}
	return nil
}

func IsDirectory(path string) (bool, error) {
	fileInfo, err := os.Stat(path)
	return fileInfo.IsDir(), err
}

func IsHidden(path string) bool {
	path = filepath.Base(path)
	if path == "." || path == "" {
		return false
	}
	if rune(path[0]) == rune('.') {
		return true
	}
	return false
}

func cmdCtx(node *core.IpfsNode, repoPath string) commands.Context {
	return commands.Context{
		// TODO deprecate this shit
		Context:    context.Background(),
		Online:     true,
		ConfigRoot: repoPath,
		LoadConfig: func(path string) (*config.Config, error) {
			return node.Repo.Config(), nil
		},
		ConstructNode: func() (*core.IpfsNode, error) {
			return node, nil
		},
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Keyserver is a wrapper for a key implementation that presents it as an HTTP
// interface.
package main

import (
	"flag"
	"net"
	"net/http"

	"upspin.io/cloud/https"
	cloudLog "upspin.io/cloud/log"
	"upspin.io/config"
	"upspin.io/errors"
	"upspin.io/factotum"
	"upspin.io/flags"
	"upspin.io/key/inprocess"
	"upspin.io/key/server"
	"upspin.io/log"
	"upspin.io/metric"
	"upspin.io/rpc/keyserver"
	"upspin.io/upspin"

	// Load required transports
	_ "upspin.io/key/transports"
)

const (
	// serverName is the upspin username for this server.
	serverName = "keyserver"

	// metricSampleSize is the size of the sample from which pick one metric
	// to save.
	metricSampleSize = 100

	// metricMaxQPS is the maximum number of metric batches to save per
	// second.
	metricMaxQPS = 5
)

var (
	testUser    = flag.String("test_user", "", "initialize a test `user` (localhost, inprocess only)")
	testSecrets = flag.String("test_secrets", "", "initialize test user with the secrets in this `directory`")
	// The format of the email config file must be lines: api key, incoming email provider user name and password.
	mailConfigFile = flag.String("mail_config", "", "config file name for incoming email signups")
)

func main() {
	flags.Parse("addr", "config", "https", "kind", "letscache", "log", "project", "serverconfig", "tls")

	if flags.Project != "" {
		cloudLog.Connect(flags.Project, serverName)
		// Disable logging locally so we don't pay the price of local
		// unbuffered writes on a busy server.
		log.SetOutput(nil)
		svr, err := metric.NewGCPSaver(flags.Project, metricSampleSize, metricMaxQPS, "serverName", serverName)
		if err != nil {
			log.Fatalf("Can't start a metric saver for GCP project %q: %s", flags.Project, err)
		} else {
			metric.RegisterSaver(svr)
		}
	}

	cfg, err := config.FromFile(flags.Config)
	if err != nil {
		log.Fatal(err)
	}

	// Create a new key implementation.
	var key upspin.KeyServer
	switch flags.ServerKind {
	case "inprocess":
		key = inprocess.New()
	case "server":
		key, err = server.New(flags.ServerConfig...)
	default:
		err = errors.Errorf("bad -kind %q", flags.ServerKind)

	}
	if err != nil {
		log.Fatalf("Setting up KeyServer: %v", err)
	}

	// Special hack for bootstrapping the inprocess key server.
	setupTestUser(key)

	httpStore := keyserver.New(cfg, key, upspin.NetAddr(flags.NetAddr))
	http.Handle("/api/Key/", httpStore)

	if logger, ok := key.(server.Logger); ok {
		http.Handle("/log", logHandler{logger: logger})
	}
	if *mailConfigFile != "" {
		f := cfg.Factotum()
		if f == nil {
			log.Fatal("supplied config must include keys when -mail_config set")
		}
		h, err := newSignupHandler(f, key, *mailConfigFile)
		if err != nil {
			log.Fatal(err)
		}
		http.Handle("/signup", h)
	}

	https.ListenAndServeFromFlags(nil, "keyserver")
}

// isLocal returns true if the name only resolves to loopback addresses.
func isLocal(addr string) bool {
	host, _, err := net.SplitHostPort(addr)
	if err != nil {
		return false
	}
	ips, err := net.LookupIP(host)
	if err != nil {
		return false
	}
	for _, ip := range ips {
		if !ip.IsLoopback() {
			return false
		}
	}
	return true
}

func setupTestUser(key upspin.KeyServer) {
	if *testUser == "" {
		return
	}
	if *testSecrets == "" {
		log.Fatalf("cannot set up a test user without specifying -test_secrets")
	}

	// Sanity checks to make sure we're not doing this in production.
	if key.Endpoint().Transport != upspin.InProcess {
		log.Fatalf("cannot use testuser for endpoint %q", key.Endpoint())
	}
	if !isLocal(flags.HTTPSAddr) {
		log.Fatal("cannot use -testuser flag except on localhost:port")
	}

	f, err := factotum.NewFromDir(*testSecrets)
	if err != nil {
		log.Fatalf("unable to initialize factotum for %q: %v", *testUser, err)
	}
	userStruct := &upspin.User{
		Name:      upspin.UserName(*testUser),
		PublicKey: f.PublicKey(),
	}
	err = key.Put(userStruct)
	if err != nil {
		log.Fatalf("Put %q failed: %v", *testUser, err)
	}
}
<commit_msg>keyserver: warn that signup will be deactivated if not mailconfig arg is defined<commit_after>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Keyserver is a wrapper for a key implementation that presents it as an HTTP
// interface.
package main

import (
	"flag"
	"net"
	"net/http"

	"upspin.io/cloud/https"
	cloudLog "upspin.io/cloud/log"
	"upspin.io/config"
	"upspin.io/errors"
	"upspin.io/factotum"
	"upspin.io/flags"
	"upspin.io/key/inprocess"
	"upspin.io/key/server"
	"upspin.io/log"
	"upspin.io/metric"
	"upspin.io/rpc/keyserver"
	"upspin.io/upspin"

	// Load required transports
	_ "upspin.io/key/transports"
)

const (
	// serverName is the upspin username for this server.
	serverName = "keyserver"

	// metricSampleSize is the size of the sample from which pick one metric
	// to save.
	metricSampleSize = 100

	// metricMaxQPS is the maximum number of metric batches to save per
	// second.
	metricMaxQPS = 5
)

var (
	testUser    = flag.String("test_user", "", "initialize a test `user` (localhost, inprocess only)")
	testSecrets = flag.String("test_secrets", "", "initialize test user with the secrets in this `directory`")
	// The format of the email config file must be lines: api key, incoming email provider user name and password.
	mailConfigFile = flag.String("mail_config", "", "config file name for incoming email signups")
)

func main() {
	flags.Parse("addr", "config", "https", "kind", "letscache", "log", "project", "serverconfig", "tls")

	if flags.Project != "" {
		cloudLog.Connect(flags.Project, serverName)
		// Disable logging locally so we don't pay the price of local
		// unbuffered writes on a busy server.
		log.SetOutput(nil)
		svr, err := metric.NewGCPSaver(flags.Project, metricSampleSize, metricMaxQPS, "serverName", serverName)
		if err != nil {
			log.Fatalf("Can't start a metric saver for GCP project %q: %s", flags.Project, err)
		} else {
			metric.RegisterSaver(svr)
		}
	}

	cfg, err := config.FromFile(flags.Config)
	if err != nil {
		log.Fatal(err)
	}

	// Create a new key implementation.
	var key upspin.KeyServer
	switch flags.ServerKind {
	case "inprocess":
		key = inprocess.New()
	case "server":
		key, err = server.New(flags.ServerConfig...)
	default:
		err = errors.Errorf("bad -kind %q", flags.ServerKind)

	}
	if err != nil {
		log.Fatalf("Setting up KeyServer: %v", err)
	}

	// Special hack for bootstrapping the inprocess key server.
	setupTestUser(key)

	httpStore := keyserver.New(cfg, key, upspin.NetAddr(flags.NetAddr))
	http.Handle("/api/Key/", httpStore)

	if logger, ok := key.(server.Logger); ok {
		http.Handle("/log", logHandler{logger: logger})
	}
	if *mailConfigFile != "" {
		f := cfg.Factotum()
		if f == nil {
			log.Fatal("supplied config must include keys when -mail_config set")
		}
		h, err := newSignupHandler(f, key, *mailConfigFile)
		if err != nil {
			log.Fatal(err)
		}
		http.Handle("/signup", h)
	} else {
		log.Println("-mail_config not set, /signup deactivated")
	}

	https.ListenAndServeFromFlags(nil, "keyserver")
}

// isLocal returns true if the name only resolves to loopback addresses.
func isLocal(addr string) bool {
	host, _, err := net.SplitHostPort(addr)
	if err != nil {
		return false
	}
	ips, err := net.LookupIP(host)
	if err != nil {
		return false
	}
	for _, ip := range ips {
		if !ip.IsLoopback() {
			return false
		}
	}
	return true
}

func setupTestUser(key upspin.KeyServer) {
	if *testUser == "" {
		return
	}
	if *testSecrets == "" {
		log.Fatalf("cannot set up a test user without specifying -test_secrets")
	}

	// Sanity checks to make sure we're not doing this in production.
	if key.Endpoint().Transport != upspin.InProcess {
		log.Fatalf("cannot use testuser for endpoint %q", key.Endpoint())
	}
	if !isLocal(flags.HTTPSAddr) {
		log.Fatal("cannot use -testuser flag except on localhost:port")
	}

	f, err := factotum.NewFromDir(*testSecrets)
	if err != nil {
		log.Fatalf("unable to initialize factotum for %q: %v", *testUser, err)
	}
	userStruct := &upspin.User{
		Name:      upspin.UserName(*testUser),
		PublicKey: f.PublicKey(),
	}
	err = key.Put(userStruct)
	if err != nil {
		log.Fatalf("Put %q failed: %v", *testUser, err)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	"github.com/spf13/cobra"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/kops/cmd/kops/util"
	api "k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/apis/kops/validation"
	"k8s.io/kops/pkg/kopscodecs"
	"k8s.io/kops/upup/pkg/fi/cloudup"
	"k8s.io/kubernetes/pkg/kubectl/cmd/templates"
	"k8s.io/kubernetes/pkg/kubectl/cmd/util/editor"
	"k8s.io/kubernetes/pkg/kubectl/util/i18n"
)

type CreateInstanceGroupOptions struct {
	Role    string
	Subnets []string
	// DryRun mode output an ig manifest of Output type.
	DryRun bool
	// Output type during a DryRun
	Output string
	// Edit will launch an editor when creating an instance group
	Edit bool
}

var (
	create_ig_long = templates.LongDesc(i18n.T(`
		Create an InstanceGroup configuration.

	    An InstanceGroup is a group of similar virtual machines.
		On AWS, an InstanceGroup maps to an AutoScalingGroup.

		The Role of an InstanceGroup defines whether machines will act as a Kubernetes master or node.`))

	create_ig_example = templates.Examples(i18n.T(`

		# Create an instancegroup for the k8s-cluster.example.com cluster.
		kops create ig --name=k8s-cluster.example.com node-example \
		  --role node --subnet my-subnet-name

		# Create a YAML manifest for an instancegroup for the k8s-cluster.example.com cluster.
		kops create ig --name=k8s-cluster.example.com node-example \
		  --role node --subnet my-subnet-name --dry-run -oyaml
		`))

	create_ig_short = i18n.T(`Create an instancegroup.`)
)

// NewCmdCreateInstanceGroup create a new cobra command object for creating a instancegroup.
func NewCmdCreateInstanceGroup(f *util.Factory, out io.Writer) *cobra.Command {
	options := &CreateInstanceGroupOptions{
		Role: string(api.InstanceGroupRoleNode),
		Edit: true,
	}

	cmd := &cobra.Command{
		Use:     "instancegroup",
		Aliases: []string{"instancegroups", "ig"},
		Short:   create_ig_short,
		Long:    create_ig_long,
		Example: create_ig_example,
		Run: func(cmd *cobra.Command, args []string) {
			err := RunCreateInstanceGroup(f, cmd, args, os.Stdout, options)
			if err != nil {
				exitWithError(err)
			}
		},
	}

	// TODO: Create Enum helper - or is there one in k8s already?
	var allRoles []string
	for _, r := range api.AllInstanceGroupRoles {
		allRoles = append(allRoles, string(r))
	}

	cmd.Flags().StringVar(&options.Role, "role", options.Role, "Type of instance group to create ("+strings.Join(allRoles, ",")+")")
  cmd.Flags().StringSliceVar(&options.Subnets, "subnet", options.Subnets, "Subnet in which to create instance group. One of Availability Zone like eu-west-1a or a comma-separated list of multiple Availability Zones.")
	// DryRun mode that will print YAML or JSON
	cmd.Flags().BoolVar(&options.DryRun, "dry-run", options.DryRun, "If true, only print the object that would be sent, without sending it. This flag can be used to create a cluster YAML or JSON manifest.")
	cmd.Flags().StringVarP(&options.Output, "output", "o", options.Output, "Ouput format. One of json|yaml")
	cmd.Flags().BoolVar(&options.Edit, "edit", options.Edit, "If true, an editor will be opened to edit default values.")

	return cmd
}

func RunCreateInstanceGroup(f *util.Factory, cmd *cobra.Command, args []string, out io.Writer, options *CreateInstanceGroupOptions) error {
	if len(args) == 0 {
		return fmt.Errorf("Specify name of instance group to create")
	}
	if len(args) != 1 {
		return fmt.Errorf("Can only create one instance group at a time!")
	}
	groupName := args[0]

	cluster, err := rootCommand.Cluster()
	if err != nil {
		return err
	}

	clientset, err := rootCommand.Clientset()
	if err != nil {
		return err
	}

	channel, err := cloudup.ChannelForCluster(cluster)
	if err != nil {
		return err
	}

	existing, err := clientset.InstanceGroupsFor(cluster).Get(groupName, metav1.GetOptions{})
	if err != nil {
		return err
	}

	if existing != nil {
		return fmt.Errorf("instance group %q already exists", groupName)
	}

	// Populate some defaults
	ig := &api.InstanceGroup{}
	ig.ObjectMeta.Name = groupName

	role, ok := api.ParseInstanceGroupRole(options.Role, true)
	if !ok {
		return fmt.Errorf("unknown role %q", options.Role)
	}
	ig.Spec.Role = role

	ig.Spec.Subnets = options.Subnets

	ig, err = cloudup.PopulateInstanceGroupSpec(cluster, ig, channel)
	if err != nil {
		return err
	}

	ig.AddInstanceGroupNodeLabel()

	if options.DryRun {

		if options.Output == "" {
			return fmt.Errorf("must set output flag; yaml or json")
		}

		// Cluster name is not populated, and we need it
		ig.ObjectMeta.Labels = make(map[string]string)
		ig.ObjectMeta.Labels[api.LabelClusterName] = cluster.ObjectMeta.Name

		switch options.Output {
		case OutputYaml:
			if err := fullOutputYAML(out, ig); err != nil {
				return fmt.Errorf("error writing cluster yaml to stdout: %v", err)
			}
			return nil
		case OutputJSON:
			if err := fullOutputJSON(out, ig); err != nil {
				return fmt.Errorf("error writing cluster json to stdout: %v", err)
			}
			return nil
		default:
			return fmt.Errorf("unsupported output type %q", options.Output)
		}
	}

	if options.Edit {
		var (
			edit = editor.NewDefaultEditor(editorEnvs)
		)

		raw, err := kopscodecs.ToVersionedYaml(ig)
		if err != nil {
			return err
		}
		ext := "yaml"

		// launch the editor
		edited, file, err := edit.LaunchTempFile(fmt.Sprintf("%s-edit-", filepath.Base(os.Args[0])), ext, bytes.NewReader(raw))
		defer func() {
			if file != "" {
				os.Remove(file)
			}
		}()
		if err != nil {
			return fmt.Errorf("error launching editor: %v", err)
		}

		obj, _, err := kopscodecs.ParseVersionedYaml(edited)
		if err != nil {
			return fmt.Errorf("error parsing yaml: %v", err)
		}
		group, ok := obj.(*api.InstanceGroup)
		if !ok {
			return fmt.Errorf("unexpected object type: %T", obj)
		}

		err = validation.ValidateInstanceGroup(group)
		if err != nil {
			return err
		}

		ig = group
	}

	_, err = clientset.InstanceGroupsFor(cluster).Create(ig)
	if err != nil {
		return fmt.Errorf("error storing InstanceGroup: %v", err)
	}

	return nil
}
<commit_msg>Fix formatting<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	"github.com/spf13/cobra"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/kops/cmd/kops/util"
	api "k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/pkg/apis/kops/validation"
	"k8s.io/kops/pkg/kopscodecs"
	"k8s.io/kops/upup/pkg/fi/cloudup"
	"k8s.io/kubernetes/pkg/kubectl/cmd/templates"
	"k8s.io/kubernetes/pkg/kubectl/cmd/util/editor"
	"k8s.io/kubernetes/pkg/kubectl/util/i18n"
)

type CreateInstanceGroupOptions struct {
	Role    string
	Subnets []string
	// DryRun mode output an ig manifest of Output type.
	DryRun bool
	// Output type during a DryRun
	Output string
	// Edit will launch an editor when creating an instance group
	Edit bool
}

var (
	create_ig_long = templates.LongDesc(i18n.T(`
		Create an InstanceGroup configuration.

	    An InstanceGroup is a group of similar virtual machines.
		On AWS, an InstanceGroup maps to an AutoScalingGroup.

		The Role of an InstanceGroup defines whether machines will act as a Kubernetes master or node.`))

	create_ig_example = templates.Examples(i18n.T(`

		# Create an instancegroup for the k8s-cluster.example.com cluster.
		kops create ig --name=k8s-cluster.example.com node-example \
		  --role node --subnet my-subnet-name

		# Create a YAML manifest for an instancegroup for the k8s-cluster.example.com cluster.
		kops create ig --name=k8s-cluster.example.com node-example \
		  --role node --subnet my-subnet-name --dry-run -oyaml
		`))

	create_ig_short = i18n.T(`Create an instancegroup.`)
)

// NewCmdCreateInstanceGroup create a new cobra command object for creating a instancegroup.
func NewCmdCreateInstanceGroup(f *util.Factory, out io.Writer) *cobra.Command {
	options := &CreateInstanceGroupOptions{
		Role: string(api.InstanceGroupRoleNode),
		Edit: true,
	}

	cmd := &cobra.Command{
		Use:     "instancegroup",
		Aliases: []string{"instancegroups", "ig"},
		Short:   create_ig_short,
		Long:    create_ig_long,
		Example: create_ig_example,
		Run: func(cmd *cobra.Command, args []string) {
			err := RunCreateInstanceGroup(f, cmd, args, os.Stdout, options)
			if err != nil {
				exitWithError(err)
			}
		},
	}

	// TODO: Create Enum helper - or is there one in k8s already?
	var allRoles []string
	for _, r := range api.AllInstanceGroupRoles {
		allRoles = append(allRoles, string(r))
	}

	cmd.Flags().StringVar(&options.Role, "role", options.Role, "Type of instance group to create ("+strings.Join(allRoles, ",")+")")
	cmd.Flags().StringSliceVar(&options.Subnets, "subnet", options.Subnets, "Subnet in which to create instance group. One of Availability Zone like eu-west-1a or a comma-separated list of multiple Availability Zones.")
	// DryRun mode that will print YAML or JSON
	cmd.Flags().BoolVar(&options.DryRun, "dry-run", options.DryRun, "If true, only print the object that would be sent, without sending it. This flag can be used to create a cluster YAML or JSON manifest.")
	cmd.Flags().StringVarP(&options.Output, "output", "o", options.Output, "Ouput format. One of json|yaml")
	cmd.Flags().BoolVar(&options.Edit, "edit", options.Edit, "If true, an editor will be opened to edit default values.")

	return cmd
}

func RunCreateInstanceGroup(f *util.Factory, cmd *cobra.Command, args []string, out io.Writer, options *CreateInstanceGroupOptions) error {
	if len(args) == 0 {
		return fmt.Errorf("Specify name of instance group to create")
	}
	if len(args) != 1 {
		return fmt.Errorf("Can only create one instance group at a time!")
	}
	groupName := args[0]

	cluster, err := rootCommand.Cluster()
	if err != nil {
		return err
	}

	clientset, err := rootCommand.Clientset()
	if err != nil {
		return err
	}

	channel, err := cloudup.ChannelForCluster(cluster)
	if err != nil {
		return err
	}

	existing, err := clientset.InstanceGroupsFor(cluster).Get(groupName, metav1.GetOptions{})
	if err != nil {
		return err
	}

	if existing != nil {
		return fmt.Errorf("instance group %q already exists", groupName)
	}

	// Populate some defaults
	ig := &api.InstanceGroup{}
	ig.ObjectMeta.Name = groupName

	role, ok := api.ParseInstanceGroupRole(options.Role, true)
	if !ok {
		return fmt.Errorf("unknown role %q", options.Role)
	}
	ig.Spec.Role = role

	ig.Spec.Subnets = options.Subnets

	ig, err = cloudup.PopulateInstanceGroupSpec(cluster, ig, channel)
	if err != nil {
		return err
	}

	ig.AddInstanceGroupNodeLabel()

	if options.DryRun {

		if options.Output == "" {
			return fmt.Errorf("must set output flag; yaml or json")
		}

		// Cluster name is not populated, and we need it
		ig.ObjectMeta.Labels = make(map[string]string)
		ig.ObjectMeta.Labels[api.LabelClusterName] = cluster.ObjectMeta.Name

		switch options.Output {
		case OutputYaml:
			if err := fullOutputYAML(out, ig); err != nil {
				return fmt.Errorf("error writing cluster yaml to stdout: %v", err)
			}
			return nil
		case OutputJSON:
			if err := fullOutputJSON(out, ig); err != nil {
				return fmt.Errorf("error writing cluster json to stdout: %v", err)
			}
			return nil
		default:
			return fmt.Errorf("unsupported output type %q", options.Output)
		}
	}

	if options.Edit {
		var (
			edit = editor.NewDefaultEditor(editorEnvs)
		)

		raw, err := kopscodecs.ToVersionedYaml(ig)
		if err != nil {
			return err
		}
		ext := "yaml"

		// launch the editor
		edited, file, err := edit.LaunchTempFile(fmt.Sprintf("%s-edit-", filepath.Base(os.Args[0])), ext, bytes.NewReader(raw))
		defer func() {
			if file != "" {
				os.Remove(file)
			}
		}()
		if err != nil {
			return fmt.Errorf("error launching editor: %v", err)
		}

		obj, _, err := kopscodecs.ParseVersionedYaml(edited)
		if err != nil {
			return fmt.Errorf("error parsing yaml: %v", err)
		}
		group, ok := obj.(*api.InstanceGroup)
		if !ok {
			return fmt.Errorf("unexpected object type: %T", obj)
		}

		err = validation.ValidateInstanceGroup(group)
		if err != nil {
			return err
		}

		ig = group
	}

	_, err = clientset.InstanceGroupsFor(cluster).Create(ig)
	if err != nil {
		return fmt.Errorf("error storing InstanceGroup: %v", err)
	}

	return nil
}
<|endoftext|>


<commit_before>// ledgerfmt pretty-prints ledger files.
package main

import (
	"bytes"
	"flag"
	"io"
	"io/ioutil"
	"log"
	"os"

	"github.com/xconstruct/ledger/parse"
	"github.com/xconstruct/ledger/print"
)

var writeOutput = flag.Bool("w", false, "Write back to input file")

func main() {
	flag.Parse()

	var source io.Reader
	source = os.Stdin
	var filename = "stdin"

	inFile := flag.Arg(0)
	if inFile != "" {
		sourceFile, err := os.Open(inFile)
		if err != nil {
			log.Fatalln("Couldn't open source file:", err)
		}

		source = sourceFile
		filename = inFile
		defer sourceFile.Close()
	}

	cnt, err := ioutil.ReadAll(source)
	if err != nil {
		log.Fatalln("Error reading input:", err)
	}

	t := parse.New(filename, string(cnt))
	err = t.Parse()
	if err != nil {
		log.Fatalln("Parsing error:", err)
	}

	printer := print.New(t)

	buf := &bytes.Buffer{}
	err = printer.Print(buf)
	if err != nil {
		log.Fatalln("rendering ledger file:", err)
	}

	var dest io.Writer
	dest = os.Stdout
	if inFile != "" && *writeOutput {
		destFile, err := os.Create(inFile)
		if err != nil {
			log.Fatalln("Couldn't write to file:", inFile)
		}
		dest = destFile
		defer destFile.Close()
	}

	_, err = dest.Write(buf.Bytes())
	if err != nil {
		log.Fatalln("Error writing to file:", err)

	}
}
<commit_msg>ledgerfmt: add -sort flag to sort xacts by date<commit_after>// ledgerfmt pretty-prints ledger files.
package main

import (
	"bytes"
	"flag"
	"io"
	"io/ioutil"
	"log"
	"os"
	"sort"

	"github.com/xconstruct/ledger/parse"
	"github.com/xconstruct/ledger/print"
)

var writeOutput = flag.Bool("w", false, "Write back to input file")
var sortXacts = flag.Bool("sort", false, "Sort transactions by date")

func main() {
	flag.Parse()

	var source io.Reader
	source = os.Stdin
	var filename = "stdin"

	inFile := flag.Arg(0)
	if inFile != "" {
		sourceFile, err := os.Open(inFile)
		if err != nil {
			log.Fatalln("Couldn't open source file:", err)
		}

		source = sourceFile
		filename = inFile
		defer sourceFile.Close()
	}

	cnt, err := ioutil.ReadAll(source)
	if err != nil {
		log.Fatalln("Error reading input:", err)
	}

	t := parse.New(filename, string(cnt))
	err = t.Parse()
	if err != nil {
		log.Fatalln("Parsing error:", err)
	}

	if *sortXacts {
		sortByDate(t)
	}

	printer := print.New(t)

	buf := &bytes.Buffer{}
	err = printer.Print(buf)
	if err != nil {
		log.Fatalln("rendering ledger file:", err)
	}

	var dest io.Writer
	dest = os.Stdout
	if inFile != "" && *writeOutput {
		destFile, err := os.Create(inFile)
		if err != nil {
			log.Fatalln("Couldn't write to file:", inFile)
		}
		dest = destFile
		defer destFile.Close()
	}

	_, err = dest.Write(buf.Bytes())
	if err != nil {
		log.Fatalln("Error writing to file:", err)

	}
}

// Represents a single xact plus whatever non-xact nodes come before it (eg,
// comments).
type xactWithPrefix struct {
	prefix []parse.Node
	xact   *parse.XactNode
}

// Sorts the nodes in t in-place by xact date, with non-xact notes staying in
// front of whatever xact they come before.
func sortByDate(t *parse.Tree) {
	// Lump in all non-xact nodes with the xact after them.
	var xacts []*xactWithPrefix
	var prefix []parse.Node
	for _, n := range t.Root.Nodes {
		if xact, ok := n.(*parse.XactNode); ok {
			xacts = append(xacts, &xactWithPrefix{prefix: prefix, xact: xact})
			prefix = nil
		} else {
			prefix = append(prefix, n)
		}
	}
	// Note that this leaves some trailing nodes in "prefix". We'll deal with them
	// later.

	sort.SliceStable(xacts, func(i, j int) bool {
		return xacts[i].xact.Date.Before(xacts[j].xact.Date)
	})

	t.Root.Nodes = nil
	for _, x := range xacts {
		t.Root.Nodes = append(t.Root.Nodes, x.prefix...)
		t.Root.Nodes = append(t.Root.Nodes, x.xact)
	}
	// Deal with any non-xacts at the end of the file.
	t.Root.Nodes = append(t.Root.Nodes, prefix...)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/jfyuen/recycleme"
	"log"
	"net/http"
	"os"
	"path"
)

var jsonFlag bool
var serverFlag bool
var dirFlag string

func init() {
	flag.Usage = func() {
		name := path.Base(os.Args[0])
		fmt.Fprintf(os.Stderr, "Usage: %s -d DIR [options] EAN:\n", name)
		flag.PrintDefaults()
	}
	flag.BoolVar(&jsonFlag, "json", false, "Print json export")
	flag.BoolVar(&serverFlag, "server", false, "Run in server mode, serving json (EAN as input is useless)")
	flag.StringVar(&dirFlag, "d", "", "Directory where to load product and packaging data")
}

func main() {
	flag.Parse()
	if (len(flag.Args()) != 1 && !serverFlag) || (serverFlag && len(flag.Args()) != 0) || dirFlag == "" {
		flag.Usage()
		os.Exit(1)
	}

	logger := log.New(os.Stdout, "", log.Ldate|log.Ltime)
	recycleme.LoadJSONFiles(dirFlag, logger)

	if serverFlag {
		http.HandleFunc("/bin/", recycleme.BinHandler)
		http.HandleFunc("/bins/", recycleme.BinsHandler)
		http.HandleFunc("/materials/", recycleme.MaterialsHandler)
		http.HandleFunc("/throwaway/", recycleme.ThrowAwayHandler)
		http.HandleFunc("/", recycleme.HomeHandler)
		fs := http.FileServer(http.Dir("data/static"))

		http.Handle("/static/", http.StripPrefix("/static/", fs))
		err := http.ListenAndServe(":8080", nil)
		if err != nil {
			logger.Fatalln(err)
		}
		logger.Println("Running in server mode")
	} else {
		product, err := recycleme.Scrap(flag.Arg(0))
		if err != nil {
			logger.Fatalln(err)
		}
		pkg := recycleme.NewProductPackage(product)
		if jsonFlag {
			jsonBytes, err := pkg.ThrowAwayJSON()
			if err != nil {
				logger.Fatalln(err)
			}
			logger.Println(string(jsonBytes))
		} else {
			logger.Println(pkg.ThrowAway())
		}
	}
}
<commit_msg>Allow specifying port on command line<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/jfyuen/recycleme"
	"log"
	"net/http"
	"os"
	"path"
)

var jsonFlag bool
var serverFlag bool
var dirFlag string
var serverPort string

func init() {
	flag.Usage = func() {
		name := path.Base(os.Args[0])
		fmt.Fprintf(os.Stderr, "Usage: %s -d DIR [options] EAN:\n", name)
		flag.PrintDefaults()
	}
	flag.BoolVar(&jsonFlag, "json", false, "Print json export")
	flag.BoolVar(&serverFlag, "server", false, "Run in server mode, serving json (EAN as input is useless)")
	flag.StringVar(&dirFlag, "d", "", "Directory where to load product and packaging data")
	flag.StringVar(&serverPort, "p", "8080", "Port to listen to")
}

func main() {
	flag.Parse()
	if (len(flag.Args()) != 1 && !serverFlag) || (serverFlag && len(flag.Args()) != 0) || dirFlag == "" {
		flag.Usage()
		os.Exit(1)
	}

	logger := log.New(os.Stdout, "", log.Ldate|log.Ltime)
	recycleme.LoadJSONFiles(dirFlag, logger)

	if serverFlag {
		http.HandleFunc("/bin/", recycleme.BinHandler)
		http.HandleFunc("/bins/", recycleme.BinsHandler)
		http.HandleFunc("/materials/", recycleme.MaterialsHandler)
		http.HandleFunc("/throwaway/", recycleme.ThrowAwayHandler)
		http.HandleFunc("/", recycleme.HomeHandler)
		fs := http.FileServer(http.Dir("data/static"))

		http.Handle("/static/", http.StripPrefix("/static/", fs))
		err := http.ListenAndServe(":"+serverPort, nil)
		if err != nil {
			logger.Fatalln(err)
		}
		logger.Println("Running in server mode")
	} else {
		product, err := recycleme.Scrap(flag.Arg(0))
		if err != nil {
			logger.Fatalln(err)
		}
		pkg := recycleme.NewProductPackage(product)
		if jsonFlag {
			jsonBytes, err := pkg.ThrowAwayJSON()
			if err != nil {
				logger.Fatalln(err)
			}
			logger.Println(string(jsonBytes))
		} else {
			logger.Println(pkg.ThrowAway())
		}
	}
}
<|endoftext|>