<commit_before>// Package local is a persistent local storage backend for Shade.
//
// It stores files and chunks locally to disk.  You may define full filepaths
// to store the files and chunks in the config, or via flag.  If you define
// neither, the flags will choose sensible defaults for your operating system.
package local

import (
	"encoding/hex"
	"errors"
	"flag"
	"io/ioutil"
	"log"
	"os"
	"path"
	"sync"

	"github.com/asjoyner/shade"
	"github.com/asjoyner/shade/drive"
)

var (
	chunkCacheDir = flag.String(
		"local.chunkCacheDir",
		path.Join(shade.ConfigDir(), "local"),
		"Default path for local Drive chunk storage.",
	)
	fileCacheDir = flag.String(
		"local.fileCacheDir",
		path.Join(shade.ConfigDir(), "local"),
		"Default path for local Drive file storage.",
	)
)

func init() {
	drive.RegisterProvider("local", NewClient)
}

// NewClient returns a fully initlized local client.
func NewClient(c drive.Config) (drive.Client, error) {
	if c.ChunkParentID == "" {
		c.ChunkParentID = *chunkCacheDir
	}
	if c.FileParentID == "" {
		c.FileParentID = *fileCacheDir
	}
	for _, dir := range []string{
		c.ChunkParentID,
		c.FileParentID,
	} {
		if fh, err := os.Open(dir); err != nil {
			if err := os.Mkdir(dir, 0700); err != nil {
				return nil, err
			}
		} else {
			fh.Close()
		}
	}

	return &Drive{config: c}, nil
}

// Drive implements the drive.Client interface by storing Files and Chunks
// to the local filesystem.  It treats the ChunkParentID and FileParentID as
// filepaths to the directory to store data in.  If FileParentId and
// ChunkParentID are not provided, it uses chunkCacheDir and fileCacheDir
// flags, which have sensible defaults for your operating system.
type Drive struct {
	sync.RWMutex // serializes accesses to the directories on local disk
	config       drive.Config
}

// ListFiles retrieves all of the File objects known to the client.  The return
// values are the sha256sum of the file object.  The keys may be passed to
// GetChunk() to retrieve the corresponding shade.File.
func (s *Drive) ListFiles() ([][]byte, error) {
	var resp [][]byte
	s.Lock()
	defer s.Unlock()
	nodes, err := ioutil.ReadDir(s.config.FileParentID)
	if err != nil {
		return nil, err
	}
	for _, n := range nodes {
		if !n.IsDir() {
			h, err := hex.DecodeString(n.Name())
			if err != nil {
				log.Printf("file with non-hex string value name: %s", n.Name())
				continue
			}
			resp = append(resp, h)
		}
	}
	return resp, nil
}

// PutFile writes the metadata describing a new file.
// f should be marshalled JSON, and may be encrypted.
func (s *Drive) PutFile(sha256sum, data []byte) error {
	s.Lock()
	defer s.Unlock()
	filename := path.Join(s.config.FileParentID, hex.EncodeToString(sha256sum))
	if fh, err := os.Open(filename); err == nil {
		fh.Close()
		return nil
	}
	if err := ioutil.WriteFile(filename, data, 0400); err != nil {
		return err
	}
	return nil
}

// GetChunk retrieves a chunk with a given SHA-256 sum
func (s *Drive) GetChunk(sha256sum []byte) ([]byte, error) {
	s.RLock()
	defer s.RUnlock()
	paths := []string{s.config.FileParentID, s.config.ChunkParentID}
	for _, p := range paths {
		filename := path.Join(p, hex.EncodeToString(sha256sum))
		if f, err := ioutil.ReadFile(filename); err == nil {
			return f, nil
		}
	}
	return nil, errors.New("chunk not found")
}

// PutChunk writes a chunk to local disk
func (s *Drive) PutChunk(sha256sum []byte, data []byte) error {
	s.Lock()
	defer s.Unlock()
	filename := path.Join(s.config.ChunkParentID, hex.EncodeToString(sha256sum))
	if fh, err := os.Open(filename); err == nil {
		fh.Close()
		return nil
	}
	if err := ioutil.WriteFile(filename, data, 0400); err != nil {
		return err
	}
	return nil
}

// GetConfig returns the config used to initialize this client.
func (s *Drive) GetConfig() drive.Config {
	return s.config
}

// Local returns whether the storage is local to this machine.
func (s *Drive) Local() bool { return true }

// Persistent returns whether the storage is persistent across task restarts.
func (s *Drive) Persistent() bool { return true }
<commit_msg>Drop the flags in drive/local for chunk and flag<commit_after>// Package local is a persistent local storage backend for Shade.
//
// It stores files and chunks locally to disk.  You may define full filepaths
// to store the files and chunks in the config, or via flag.  If you define
// neither, the flags will choose sensible defaults for your operating system.
package local

import (
	"encoding/hex"
	"errors"
	"io/ioutil"
	"log"
	"os"
	"path"
	"sync"

	"github.com/asjoyner/shade/drive"
)

func init() {
	drive.RegisterProvider("local", NewClient)
}

// NewClient returns a fully initlized local client.
func NewClient(c drive.Config) (drive.Client, error) {
	if c.ChunkParentID == "" {
		return nil, errors.New("specify the path to store local chunks as ChunkParentID")
	}
	if c.FileParentID == "" {
		return nil, errors.New("specify the path to store local files as FileParentID")
	}
	for _, dir := range []string{
		c.ChunkParentID,
		c.FileParentID,
	} {
		if fh, err := os.Open(dir); err != nil {
			if err := os.Mkdir(dir, 0700); err != nil {
				return nil, err
			}
		} else {
			fh.Close()
		}
	}

	return &Drive{config: c}, nil
}

// Drive implements the drive.Client interface by storing Files and Chunks
// to the local filesystem.  It treats the ChunkParentID and FileParentID as
// filepaths to the directory to store data in.
type Drive struct {
	sync.RWMutex // serializes accesses to the directories on local disk
	config       drive.Config
}

// ListFiles retrieves all of the File objects known to the client.  The return
// values are the sha256sum of the file object.  The keys may be passed to
// GetChunk() to retrieve the corresponding shade.File.
func (s *Drive) ListFiles() ([][]byte, error) {
	var resp [][]byte
	s.Lock()
	defer s.Unlock()
	nodes, err := ioutil.ReadDir(s.config.FileParentID)
	if err != nil {
		return nil, err
	}
	for _, n := range nodes {
		if !n.IsDir() {
			h, err := hex.DecodeString(n.Name())
			if err != nil {
				log.Printf("file with non-hex string value name: %s", n.Name())
				continue
			}
			resp = append(resp, h)
		}
	}
	return resp, nil
}

// PutFile writes the metadata describing a new file.
// f should be marshalled JSON, and may be encrypted.
func (s *Drive) PutFile(sha256sum, data []byte) error {
	s.Lock()
	defer s.Unlock()
	filename := path.Join(s.config.FileParentID, hex.EncodeToString(sha256sum))
	if fh, err := os.Open(filename); err == nil {
		fh.Close()
		return nil
	}
	if err := ioutil.WriteFile(filename, data, 0400); err != nil {
		return err
	}
	return nil
}

// GetChunk retrieves a chunk with a given SHA-256 sum
func (s *Drive) GetChunk(sha256sum []byte) ([]byte, error) {
	s.RLock()
	defer s.RUnlock()
	paths := []string{s.config.FileParentID, s.config.ChunkParentID}
	for _, p := range paths {
		filename := path.Join(p, hex.EncodeToString(sha256sum))
		if f, err := ioutil.ReadFile(filename); err == nil {
			return f, nil
		}
	}
	return nil, errors.New("chunk not found")
}

// PutChunk writes a chunk to local disk
func (s *Drive) PutChunk(sha256sum []byte, data []byte) error {
	s.Lock()
	defer s.Unlock()
	filename := path.Join(s.config.ChunkParentID, hex.EncodeToString(sha256sum))
	if fh, err := os.Open(filename); err == nil {
		fh.Close()
		return nil
	}
	if err := ioutil.WriteFile(filename, data, 0400); err != nil {
		return err
	}
	return nil
}

// GetConfig returns the config used to initialize this client.
func (s *Drive) GetConfig() drive.Config {
	return s.config
}

// Local returns whether the storage is local to this machine.
func (s *Drive) Local() bool { return true }

// Persistent returns whether the storage is persistent across task restarts.
func (s *Drive) Persistent() bool { return true }
<|endoftext|>


<commit_before>package files

// io.go - Responsible for performing input and output operations.

import (
	"../config"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

const maxReadSize = 5e+7 // 50MB

// File - The main object used for storing and processing a single file.
type File struct {
	source   *os.File
	tempout  *os.File
	readsize int64
	outpath  string
}

// NewFile - Creates and returns File
func NewFile(sourcePath string) File {
	// Open the source file
	source, err := os.Open(sourcePath)
	if err != nil {
		if os.IsPermission(err) || os.IsNotExist(err) {
			log.Warningf("Could not open %s (err: %s)", sourcePath, err)
		} else {
			log.Fatalf("Failed to open %s (err: %s)", sourcePath, err)
		}
	}

	// Open the temporary output file.
	tempout, err := ioutil.TempFile(os.TempDir(), "gcp")
	if err != nil {
		source.Close()
		log.Fatal("Failed to open temporoary output file (err: %s)", err)
	}

	file := File{source: source, tempout: tempout}

	// Figure out how large of a slice we're supposed to
	// take when reading data.
	sourceStat, err := file.source.Stat()
	sourceSize := sourceStat.Size()

	if sourceSize < maxReadSize {
		file.readsize = sourceSize
	} else {
		file.readsize = maxReadSize
	}

	// Figure out what the final file path should be.
	outpath := filepath.Join(config.Destination, file.source.Name())

	if file.ShouldCompress() {
		outpath += ".lzma"
	}

	if file.ShouldEncrypt() {
		outpath += ".aes"
	}

	file.outpath = outpath

	return file
}

// ShouldCompress - Returns True if the file should be compressed.  Some kinds
// of files we shouldn't compress because it's either time consuming or we
// wouldn't gamin much by enabling compression.
func (file *File) ShouldCompress() bool {
	if !config.Compress {
		return false
	}

	name := strings.ToLower(file.source.Name())
	if strings.HasSuffix(name, ".iso") {
		return false
	}

	return true
}

// ShouldEncrypt - Returns True if the file should be compressed.  Some kinds
// of files we shouldn't compress because it's either time consuming or we
// wouldn't gamin much by enabling compression.
func (file *File) ShouldEncrypt() bool {
	if !config.Encrypt {
		return false
	}

	name := strings.ToLower(file.source.Name())
	if strings.HasSuffix(name, ".iso") {
		return false
	}

	return true
}

// DestinationExists - Returns True if the output path exists.
func (file *File) DestinationExists() bool {
	_, err := os.Stat(file.outpath)
	if err == nil {
		return true
	} else if os.IsNotExist(err) {
		return false
	} else if os.IsPermission(err) {
		log.Fatalf("Failed to stat %s due to permission error", file.outpath)
	}
	log.Fatalf("Unhandled DestinationExists() for %s", file.outpath)
	return false
}

// Rename - Renames the temporary file
func (file *File) Rename() {
	log.Infof("%s -> %s", file.source.Name(), file.outpath)

	// Check if the parent directory exists, if not we'll
	// need to create it.
	err := os.MkdirAll(filepath.Dir(file.outpath), 0700)
	if err != nil {
		log.Fatal(
			"Failed to create parent directory for %s (err: %s)",
			file.outpath, err)
	}

	// If we're not performing compression or encryption then it's
	// a direct copy rather than a rename.
	if !file.ShouldCompress() && !file.ShouldEncrypt() {

	} else {
		err := os.Rename(file.tempout.Name(), file.outpath)
		if err != nil {
			log.Fatalf(
				"Failed to rename %s -> %s (err: %s)",
				file.tempout.Name(), file.outpath, err)
		}
	}

	_, err = file.tempout.Stat()
	if err == nil {
		err := os.Remove(file.tempout.Name())
		if err != nil {
			log.Warningf("Failed to remove %s", file.tempout.Name())
		}
	}
}

// Close - Closes the underlying files objects
func (file *File) Close() {
	errors := 0
	err := file.tempout.Sync()

	if err != nil {
		errors++
		log.Warningf("Failed to flush %s (err: %s)", file.tempout.Name(), err)
	}

	err = file.tempout.Close()
	if err != nil {
		errors++
		log.Warningf("Failed to close %s (err: %s)", file.tempout.Name(), err)
	}

	err = file.source.Close()
	if err != nil {
		errors++
		log.Warningf("Failed to close %s (err: %s)", file.source.Name(), err)
	}

	if errors > 0 {
		log.Fatal("One or more errors while calling Close()")
	}
}

// Done - Called when we've finished processing the requested file.
func (file *File) Done() {
	defer filesProcessing.Done()
	file.Close()
	file.Rename()
}
<commit_msg>unexporting names, better error checking, cleaner setup and code<commit_after>package files

// io.go - Responsible for performing input and output operations.

import (
	"../config"
	"bytes"
	"code.google.com/p/lzma"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

const maxReadSize = 5e+7 // 50MB

// File - The main object used for storing and processing a single file.
type File struct {
	sourcepath     string
	source         *os.File
	tempout        *os.File
	readsize       int64
	outpath        string
	shouldCompress bool
	shouldEncrypt  bool
}

func outpath(file *File) string {
	// Setup the output path
	outpath := filepath.Join(config.Destination, file.source.Name())
	if file.shouldCompress {
		outpath += ".lzma"
	}
	if file.shouldEncrypt {
		outpath += ".aes"
	}
	return outpath
}

func readsize(file *File) int64 {
	stat, err := file.source.Stat()
	if err != nil {
		log.Fatal("Failed to start %s (err: %s)", file.source.Name(), err)
	}
	size := stat.Size()
	if size > maxReadSize {
		size = maxReadSize
	}
	return size
}

func shouldCompress(file *File) bool {
	if !config.Compress {
		return false
	}
	name := strings.ToLower(file.source.Name())
	if strings.HasSuffix(name, ".iso") {
		return false
	}
	return true
}

func shouldEncrypt(file *File) bool {
	if !config.Encrypt {
		return false
	}
	name := strings.ToLower(file.source.Name())
	if strings.HasSuffix(name, ".iso") {
		return false
	}
	return true
}

func compress(file *File, data []byte, bytesRead int64) ([]byte, error) {
	var compressed bytes.Buffer
	lzmaWriter := lzma.NewWriterSizeLevel(
		&compressed, bytesRead, lzma.BestCompression)
	_, err := lzmaWriter.Write(data)
	lzmaWriter.Close()

	if err != nil {
		log.Warningf(
			"Compression failed for %s (err: %s)",
			file.source.Name(), err)
		return nil, err
	}

	return compressed.Bytes(), nil
}

// TODO
func encrypt(file *File, data []byte) ([]byte, error) {
	return data, nil
}

// Open - Opens the input and output files where applicable, also sets up the
// output path.
func (file *File) open() error {
	// Open the source file
	source, err := os.Open(file.sourcepath)

	if err != nil {
		return err
	}

	// Open the temporary output file.
	tempout, err := ioutil.TempFile(os.TempDir(), "gcp")

	if err != nil {
		source.Close()
		return err
	}

	// Establish the attributes we'll need for working
	// with the file.
	//  NOTE: Order matters here.
	file.source = source
	file.tempout = tempout
	file.readsize = readsize(file)
	file.shouldCompress = shouldCompress(file)
	file.shouldEncrypt = shouldEncrypt(file)
	file.outpath = outpath(file)

	return nil
}

// Performs the main IO operations responsible for
// processing the file.  The results end up in the
// temporary output path.
func (file *File) process() error {
	log.Debugf("%s -> %s", file.source.Name(), file.tempout.Name())
	defer file.source.Close()

	// Files which are neither compressed or encrypted will
	// just be coped over to their temporary output.
	if !file.shouldCompress && !file.shouldEncrypt {
		io.Copy(file.tempout, file.source)
		return nil
	}

	// Iterate over the whole file and compress and/or encrypt
	for {
		data := make([]byte, file.readsize)
		bytesRead, err := file.source.Read(data)
		bytesRead64 := int64(bytesRead)

		if err == io.EOF {
			break
		} else if err != nil {
			log.Warningf(
				"Failed to read %s (err: %s)", file.source.Name(), err)
			return err
		}

		// It's possible we didn't read as many bytes
		// from the file as we allocated for `data`.  If this
		// is the case, resize data so it matches the number
		// of bytes read.  Otherwise we end up with empty bytes
		// in the file we're writing to disk.
		if file.readsize > bytesRead64 {
			data = append([]byte(nil), data[:bytesRead]...)
		}

		if file.shouldCompress {
			data, err = compress(file, data, bytesRead64)
			if err != nil {
				return err
			}
		}

		if file.shouldEncrypt {
			data, err = encrypt(file, data)
			if err != nil {
				return err
			}
		}

		file.tempout.Write(data)
	}

	return nil
}

// Responsible for saving the file to the final location.
func (file *File) save() error {

	log.Infof("%s -> %s", file.source.Name(), file.outpath)

	err := file.tempout.Sync()
	if err != nil {
		log.Warning("Failed to sync temp output")
		return err
	}

	err = file.tempout.Close()
	if err != nil {
		log.Warning("Failed to close temp output")
		return err
	}

	directory := filepath.Dir(file.outpath)
	err = os.MkdirAll(directory, 0700)
	if err != nil {
		log.Warningf("Failed to create %s", directory)
		return err
	}

	err = os.Rename(file.tempout.Name(), file.outpath)
	if err != nil {
		log.Warning("Failed to rename file")
		return err
	}

	return nil

}

// Performs some final cleanup in the event of an error.  This is mainly
// aimed at closing the file handles and removing the temp. output file.  We
// ignore errors in this block of code because we expect processfiles() to
// call log.Fatal* soon after this function.
func (file *File) clean() {
	defer filesProcessing.Done()

	if file.source != nil {
		file.source.Close()
	}

	if file.tempout != nil {
		file.tempout.Close()
		os.Remove(file.tempout.Name())
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"github.com/gorilla/mux"
	"html/template"
	"io/ioutil"
	"net/http"
	"os"
)

func redirectLogin(w http.ResponseWriter, r *http.Request) {
	http.Redirect(w, r, "/login", 301)
}

type staticHtmlPlugs struct {
	CdnPrefix string
}

func initStaticRouter(router *mux.Router) error {
	for _, path := range []string{"js", "css", "images"} {
		router.PathPrefix("/" + path + "/").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			f, err := os.Stat("." + r.URL.Path)
			if err != nil || f.IsDir() {
				http.NotFound(w, r)
			}

			http.ServeFile(w, r, "."+r.URL.Path)
		})
	}

	pages := []string{
		"login",
		"signup",
		"dashboard",
		"account",
	}

	html := make(map[string]string)
	for _, page := range pages {
		contents, err := ioutil.ReadFile(page + ".html")
		if err != nil {
			logger.Errorf("cannot read file %s.html: %v", page, err)
			return err
		}

		t, err := template.New(page).Parse(string(contents))
		if err != nil {
			logger.Errorf("cannot parse %s.html template: %v", page, err)
			return err
		}

		var buf bytes.Buffer
		t.Execute(&buf, &staticHtmlPlugs{CdnPrefix: os.Getenv("CDN_PREFIX")})

		html[page] = buf.String()
	}

	for _, page := range pages {
		router.HandleFunc("/"+page, func(w http.ResponseWriter, r *http.Request) {
			fmt.Fprintf(w, html[page])
		})
	}

	router.HandleFunc("/", redirectLogin).Methods("GET")

	return nil
}
<commit_msg>router_static.go: use different template delims<commit_after>package main

import (
	"bytes"
	"fmt"
	"github.com/gorilla/mux"
	"html/template"
	"io/ioutil"
	"net/http"
	"os"
)

func redirectLogin(w http.ResponseWriter, r *http.Request) {
	http.Redirect(w, r, "/login", 301)
}

type staticHtmlPlugs struct {
	CdnPrefix string
}

func initStaticRouter(router *mux.Router) error {
	for _, path := range []string{"js", "css", "images"} {
		router.PathPrefix("/" + path + "/").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			f, err := os.Stat("." + r.URL.Path)
			if err != nil || f.IsDir() {
				http.NotFound(w, r)
			}

			http.ServeFile(w, r, "."+r.URL.Path)
		})
	}

	pages := []string{
		"login",
		"signup",
		"dashboard",
		"account",
	}

	html := make(map[string]string)
	for _, page := range pages {
		contents, err := ioutil.ReadFile(page + ".html")
		if err != nil {
			logger.Errorf("cannot read file %s.html: %v", page, err)
			return err
		}

		t, err := template.New(page).Delims("<<<", ">>>").Parse(string(contents))
		if err != nil {
			logger.Errorf("cannot parse %s.html template: %v", page, err)
			return err
		}

		var buf bytes.Buffer
		t.Execute(&buf, &staticHtmlPlugs{CdnPrefix: os.Getenv("CDN_PREFIX")})

		html[page] = buf.String()
	}

	for _, page := range pages {
		router.HandleFunc("/"+page, func(w http.ResponseWriter, r *http.Request) {
			fmt.Fprintf(w, html[page])
		})
	}

	router.HandleFunc("/", redirectLogin).Methods("GET")

	return nil
}
<|endoftext|>


<commit_before>package node

import (
	"github.com/appcelerator/amp/pkg/docker"
	"github.com/docker/docker/api/types"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
)

// Server is used to implement log.LogServer
type Server struct {
	Docker *docker.Docker
}

// GetNodes implements Node.GetNodes
func (s *Server) GetNodes(ctx context.Context, in *GetNodesRequest) (*GetNodesReply, error) {
	list, err := s.Docker.NodeList(ctx, types.NodeListOptions{})
	if err != nil {
		return nil, grpc.Errorf(codes.Internal, "%v", err)
	}
	nodeList := &GetNodesReply{}
	for _, item := range list {
		node := &NodeEntry{
			Id:           item.ID,
			Name:         item.Spec.Name,
			Hostname:     item.Description.Hostname,
			Role:         string(item.Spec.Role),
			Architecture: item.Description.Platform.Architecture,
			Os:           item.Description.Platform.OS,
			Engine:       item.Description.Engine.EngineVersion,
			Addr:         item.Status.Addr,
			Status:       string(item.Status.State),
			Availability: string(item.Spec.Availability),
			Labels:       item.Description.Engine.Labels,
		}
		if item.ManagerStatus != nil {
			node.Leader = item.ManagerStatus.Leader
			node.Reachability = string(item.ManagerStatus.Reachability)
		}
		nodeList.Entries = append(nodeList.Entries, node)
	}
	return nodeList, nil
}
<commit_msg>fix node labels retrieval (#1386)<commit_after>package node

import (
	"github.com/appcelerator/amp/pkg/docker"
	"github.com/docker/docker/api/types"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
)

// Server is used to implement log.LogServer
type Server struct {
	Docker *docker.Docker
}

// GetNodes implements Node.GetNodes
func (s *Server) GetNodes(ctx context.Context, in *GetNodesRequest) (*GetNodesReply, error) {
	list, err := s.Docker.NodeList(ctx, types.NodeListOptions{})
	if err != nil {
		return nil, grpc.Errorf(codes.Internal, "%v", err)
	}
	nodeList := &GetNodesReply{}
	for _, item := range list {
		node := &NodeEntry{
			Id:           item.ID,
			Name:         item.Spec.Name,
			Hostname:     item.Description.Hostname,
			Role:         string(item.Spec.Role),
			Architecture: item.Description.Platform.Architecture,
			Os:           item.Description.Platform.OS,
			Engine:       item.Description.Engine.EngineVersion,
			Addr:         item.Status.Addr,
			Status:       string(item.Status.State),
			Availability: string(item.Spec.Availability),
			Labels:       item.Spec.Annotations.Labels,
		}
		if item.ManagerStatus != nil {
			node.Leader = item.ManagerStatus.Leader
			node.Reachability = string(item.ManagerStatus.Reachability)
		}
		nodeList.Entries = append(nodeList.Entries, node)
	}
	return nodeList, nil
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package apiclient

import (
	"archive/zip"
	"bytes"
	"errors"
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"

	"github.com/srinandan/apigeecli/clilog"
)

//entityPayloadList stores list of entities
var entityPayloadList [][]byte //types.EntityPayloadList

//ReadArchive confirms f the file format is zip and reads the contents are a byte[]
func ReadArchive(filename string) ([]byte, error) {
	if !strings.HasSuffix(filename, ".zip") {
		clilog.Error.Println("proxy bundle must be a zip file")
		return nil, errors.New("source must be a zipfile")
	}

	file, err := os.Open(filename)
	if err != nil {
		clilog.Error.Println("cannot open/read archive: ", err)
		return nil, err
	}

	defer file.Close()

	fi, err := file.Stat()
	if err != nil {
		clilog.Error.Println("error accessing file: ", err)
		return nil, err
	}

	_, err = zip.NewReader(file, fi.Size())

	if err != nil {
		clilog.Error.Println("invalid archive format: ", err)
		return nil, err
	}

	archiveFile, err := ioutil.ReadFile(filename)
	if err != nil {
		clilog.Error.Println("Error reading archive: %s", err)
		return nil, err
	}

	return archiveFile, nil
}

//ReadBundle confirms if the file format is a zip file
func ReadBundle(filename string) error {
	if !strings.HasSuffix(filename, ".zip") {
		clilog.Error.Println("proxy bundle must be a zip file")
		return errors.New("source must be a zipfile")
	}

	file, err := os.Open(filename)
	if err != nil {
		clilog.Error.Println("cannot open/read API Proxy Bundle: ", err)
		return err
	}

	defer file.Close()

	fi, err := file.Stat()
	if err != nil {
		clilog.Error.Println("error accessing file: ", err)
		return err
	}

	_, err = zip.NewReader(file, fi.Size())

	if err != nil {
		clilog.Error.Println("invalid API Proxy Bundle: ", err)
		return err
	}

	return nil
}

//WriteByteArrayToFile accepts []bytes and writes to a file
func WriteByteArrayToFile(exportFile string, fileAppend bool, payload []byte) error {
	var fileFlags = os.O_CREATE | os.O_WRONLY

	if fileAppend {
		fileFlags |= os.O_APPEND
	} else {
		fileFlags |= os.O_TRUNC
	}

	f, err := os.OpenFile(exportFile, fileFlags, 0644)
	if err != nil {
		return err
	}

	defer f.Close()

	_, err = f.Write(payload)
	if err != nil {
		clilog.Error.Println("error writing to file: ", err)
		return err
	}
	return nil
}

//WriteArrayByteArrayToFile accepts [][]bytes and writes to a file
func WriteArrayByteArrayToFile(exportFile string, fileAppend bool, payload [][]byte) error {
	var fileFlags = os.O_CREATE | os.O_WRONLY

	if fileAppend {
		fileFlags |= os.O_APPEND
	}

	f, err := os.OpenFile(exportFile, fileFlags, 0644)
	if err != nil {
		return err
	}

	defer f.Close()

	//begin json array
	_, err = f.Write([]byte("["))
	if err != nil {
		clilog.Error.Println("error writing to file ", err)
		return err
	}

	payloadFromArray := bytes.Join(payload, []byte(","))
	//add json array terminate
	payloadFromArray = append(payloadFromArray, byte(']'))

	_, err = f.Write(payloadFromArray)

	if err != nil {
		clilog.Error.Println("error writing to file: ", err)
		return err
	}

	return nil
}

//GetAsyncEntity stores results for each entity in a list
func GetAsyncEntity(entityURL string, wg *sync.WaitGroup, mu *sync.Mutex) {
	//this is a two step process - 1) get entity details 2) store in byte[][]
	defer wg.Done()

	var respBody []byte

	//don't print to sysout
	respBody, err := HttpClient(false, entityURL)

	if err != nil {
		clilog.Error.Fatalf("error with entity: %s", entityURL)
		clilog.Error.Println(err)

		return
	}

	mu.Lock()
	entityPayloadList = append(entityPayloadList, respBody)
	mu.Unlock()
	clilog.Info.Printf("Completed entity: %s", entityURL)
}

func GetEntityPayloadList() [][]byte {
	return entityPayloadList
}

func ClearEntityPayloadList() {
	entityPayloadList = entityPayloadList[:0]
}

//FetchAsyncBundle can download a shared flow or a proxy bundle
func FetchAsyncBundle(entityType string, folder string, name string, revision string, wg *sync.WaitGroup) {
	//this method is meant to be called asynchronously
	defer wg.Done()

	_ = FetchBundle(entityType, folder, name, revision)
}

//FetchBundle can download a shared flow or proxy bundle
func FetchBundle(entityType string, folder string, name string, revision string) error {
	u, _ := url.Parse(BaseURL)
	q := u.Query()
	q.Set("format", "bundle")
	u.RawQuery = q.Encode()
	u.Path = path.Join(u.Path, GetApigeeOrg(), entityType, name, "revisions", revision)

	err := DownloadResource(u.String(), name, ".zip")
	if err != nil {
		clilog.Error.Fatalf("error with entity: %s", name)
		clilog.Error.Println(err)
		return err
	}

	if len(folder) > 0 {
		_ = os.Rename(name+".zip", path.Join(folder, name+".zip"))
	}

	return nil
}

//ImportBundleAsync imports a sharedflow or api proxy bundle meantot be called asynchronously
func ImportBundleAsync(entityType string, name string, bundlePath string, wg *sync.WaitGroup) {
	defer wg.Done()

	_ = ImportBundle(entityType, name, bundlePath)
}

//ImportBundle imports a sharedflow or api proxy bundle
func ImportBundle(entityType string, name string, bundlePath string) error {
	err := ReadBundle(bundlePath)
	if err != nil {
		return err
	}

	//when importing from a folder, proxy name = file name
	if name == "" {
		_, fileName := filepath.Split(bundlePath)
		names := strings.Split(fileName, ".")
		name = names[0]
	}

	u, _ := url.Parse(BaseURL)
	u.Path = path.Join(u.Path, GetApigeeOrg(), entityType)

	q := u.Query()
	q.Set("name", name)
	q.Set("action", "import")
	u.RawQuery = q.Encode()

	err = ReadBundle(bundlePath)
	if err != nil {
		clilog.Error.Println(err)
		return err
	}

	_, err = PostHttpOctet(true, false, u.String(), bundlePath)
	if err != nil {
		clilog.Error.Println(err)
		return err
	}

	clilog.Info.Printf("Completed entity: %s", u.String())
	return nil
}
<commit_msg>add revision number to zip<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package apiclient

import (
	"archive/zip"
	"bytes"
	"errors"
	"io/ioutil"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"

	"github.com/srinandan/apigeecli/clilog"
)

//entityPayloadList stores list of entities
var entityPayloadList [][]byte //types.EntityPayloadList

//ReadArchive confirms f the file format is zip and reads the contents are a byte[]
func ReadArchive(filename string) ([]byte, error) {
	if !strings.HasSuffix(filename, ".zip") {
		clilog.Error.Println("proxy bundle must be a zip file")
		return nil, errors.New("source must be a zipfile")
	}

	file, err := os.Open(filename)
	if err != nil {
		clilog.Error.Println("cannot open/read archive: ", err)
		return nil, err
	}

	defer file.Close()

	fi, err := file.Stat()
	if err != nil {
		clilog.Error.Println("error accessing file: ", err)
		return nil, err
	}

	_, err = zip.NewReader(file, fi.Size())

	if err != nil {
		clilog.Error.Println("invalid archive format: ", err)
		return nil, err
	}

	archiveFile, err := ioutil.ReadFile(filename)
	if err != nil {
		clilog.Error.Println("Error reading archive: %s", err)
		return nil, err
	}

	return archiveFile, nil
}

//ReadBundle confirms if the file format is a zip file
func ReadBundle(filename string) error {
	if !strings.HasSuffix(filename, ".zip") {
		clilog.Error.Println("proxy bundle must be a zip file")
		return errors.New("source must be a zipfile")
	}

	file, err := os.Open(filename)
	if err != nil {
		clilog.Error.Println("cannot open/read API Proxy Bundle: ", err)
		return err
	}

	defer file.Close()

	fi, err := file.Stat()
	if err != nil {
		clilog.Error.Println("error accessing file: ", err)
		return err
	}

	_, err = zip.NewReader(file, fi.Size())

	if err != nil {
		clilog.Error.Println("invalid API Proxy Bundle: ", err)
		return err
	}

	return nil
}

//WriteByteArrayToFile accepts []bytes and writes to a file
func WriteByteArrayToFile(exportFile string, fileAppend bool, payload []byte) error {
	var fileFlags = os.O_CREATE | os.O_WRONLY

	if fileAppend {
		fileFlags |= os.O_APPEND
	} else {
		fileFlags |= os.O_TRUNC
	}

	f, err := os.OpenFile(exportFile, fileFlags, 0644)
	if err != nil {
		return err
	}

	defer f.Close()

	_, err = f.Write(payload)
	if err != nil {
		clilog.Error.Println("error writing to file: ", err)
		return err
	}
	return nil
}

//WriteArrayByteArrayToFile accepts [][]bytes and writes to a file
func WriteArrayByteArrayToFile(exportFile string, fileAppend bool, payload [][]byte) error {
	var fileFlags = os.O_CREATE | os.O_WRONLY

	if fileAppend {
		fileFlags |= os.O_APPEND
	}

	f, err := os.OpenFile(exportFile, fileFlags, 0644)
	if err != nil {
		return err
	}

	defer f.Close()

	//begin json array
	_, err = f.Write([]byte("["))
	if err != nil {
		clilog.Error.Println("error writing to file ", err)
		return err
	}

	payloadFromArray := bytes.Join(payload, []byte(","))
	//add json array terminate
	payloadFromArray = append(payloadFromArray, byte(']'))

	_, err = f.Write(payloadFromArray)

	if err != nil {
		clilog.Error.Println("error writing to file: ", err)
		return err
	}

	return nil
}

//GetAsyncEntity stores results for each entity in a list
func GetAsyncEntity(entityURL string, wg *sync.WaitGroup, mu *sync.Mutex) {
	//this is a two step process - 1) get entity details 2) store in byte[][]
	defer wg.Done()

	var respBody []byte

	//don't print to sysout
	respBody, err := HttpClient(false, entityURL)

	if err != nil {
		clilog.Error.Fatalf("error with entity: %s", entityURL)
		clilog.Error.Println(err)

		return
	}

	mu.Lock()
	entityPayloadList = append(entityPayloadList, respBody)
	mu.Unlock()
	clilog.Info.Printf("Completed entity: %s", entityURL)
}

func GetEntityPayloadList() [][]byte {
	return entityPayloadList
}

func ClearEntityPayloadList() {
	entityPayloadList = entityPayloadList[:0]
}

//FetchAsyncBundle can download a shared flow or a proxy bundle
func FetchAsyncBundle(entityType string, folder string, name string, revision string, wg *sync.WaitGroup) {
	//this method is meant to be called asynchronously
	defer wg.Done()

	_ = FetchBundle(entityType, folder, name, revision)
}

//FetchBundle can download a shared flow or proxy bundle
func FetchBundle(entityType string, folder string, name string, revision string) error {
	u, _ := url.Parse(BaseURL)
	q := u.Query()
	q.Set("format", "bundle")
	u.RawQuery = q.Encode()
	u.Path = path.Join(u.Path, GetApigeeOrg(), entityType, name, "revisions", revision)

	proxyName := name + "_" + revision

	err := DownloadResource(u.String(), proxyName, ".zip")
	if err != nil {
		clilog.Error.Fatalf("error with entity: %s", name)
		clilog.Error.Println(err)
		return err
	}

	if len(folder) > 0 {
		_ = os.Rename(proxyName+".zip", path.Join(folder, proxyName+".zip"))
	}

	return nil
}

//ImportBundleAsync imports a sharedflow or api proxy bundle meantot be called asynchronously
func ImportBundleAsync(entityType string, name string, bundlePath string, wg *sync.WaitGroup) {
	defer wg.Done()

	_ = ImportBundle(entityType, name, bundlePath)
}

//ImportBundle imports a sharedflow or api proxy bundle
func ImportBundle(entityType string, name string, bundlePath string) error {
	err := ReadBundle(bundlePath)
	if err != nil {
		return err
	}

	//when importing from a folder, proxy name = file name
	if name == "" {
		_, fileName := filepath.Split(bundlePath)
		names := strings.Split(fileName, ".")
		name = names[0]
	}

	u, _ := url.Parse(BaseURL)
	u.Path = path.Join(u.Path, GetApigeeOrg(), entityType)

	q := u.Query()
	q.Set("name", name)
	q.Set("action", "import")
	u.RawQuery = q.Encode()

	err = ReadBundle(bundlePath)
	if err != nil {
		clilog.Error.Println(err)
		return err
	}

	_, err = PostHttpOctet(true, false, u.String(), bundlePath)
	if err != nil {
		clilog.Error.Println(err)
		return err
	}

	clilog.Info.Printf("Completed entity: %s", u.String())
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"strconv"
	"strings"

	"github.com/cactus/go-statsd-client/statsd"
	"github.com/coreos/go-etcd/etcd"
)

type stringslice []string

func (self *stringslice) String() string {
	return fmt.Sprintf("%d", *self)
}

func (self *stringslice) Set(value string) error {
	*self = append(*self, value)
	return nil
}

var port uint
var blocksize uint64
var etcd_nodes stringslice
var stats string
var logfile string

func init() {
	flag.UintVar(&port, "port", 9000, "Port to run HTTP server on")
	flag.Uint64Var(&blocksize, "blocksize", 64, "Block size")
	flag.Var(&etcd_nodes, "etcd", "Etcd server")
	flag.StringVar(&stats, "statsd", "127.0.0.1:8125", "Statsd server")
	flag.StringVar(&logfile, "log", "/tmp/nexter.log", "Log file name")
	flag.Parse()
}

type Req interface{}

type IncReq struct {
	id  int
	ret chan uint64
}

type DelReq struct {
	id int
}

type Nexter struct {
	reqchan chan Req
	done    chan bool
	stats   *statsd.Client
}

func (self *Nexter) countloop(ch chan uint64, id int, client *etcd.Client) {
	var start uint64
	var end uint64
	path := "nexter/" + strconv.Itoa(id)

	for {
		node, err := client.Get(path, false, false)
		if err != nil {
			ee, ok := err.(*etcd.EtcdError)
			if ok && ee.ErrorCode == 100 { // node does not exist
				_, err := client.Create(path, "0", 0)
				if err != nil {
					ee, ok := err.(*etcd.EtcdError)
					// Catch race condition where another node did the same client.Create()
					if ok && ee.ErrorCode == 105 { // Node has been created
						continue
					} else {
						log.Fatal(err)
					}
				}
				continue
			} else {
				log.Fatal(err)
			}
		} else { // No error, get start of series from etcd node
			start, err = strconv.ParseUint(node.Node.Value, 10, 0)
			if err != nil {
				log.Fatal(err)
			}
			end = start + blocksize
		}
		_, err = client.CompareAndSwap(path, strconv.FormatUint(end, 10), 0, strconv.FormatUint(start, 10), 0)
		if err != nil {
			log.Println("CAS failure", path, start, end)
			continue
		}
		self.stats.Gauge("nexter."+strconv.Itoa(id), int64(end), 1.0)
		log.Println("Allocated", id, start, end)
		for c := start; c < end; c += 1 {
			ch <- c
		}
	}
}

func (self *Nexter) loop() {
	counters := make(map[int]chan uint64)
	client := etcd.NewClient(etcd_nodes)
	for {
		select {
		case req := <-self.reqchan:
			switch req.(type) {
			case *IncReq:
				countreq := req.(*IncReq)
				counter, ok := counters[countreq.id]
				if !ok {
					counter = make(chan uint64)
					counters[countreq.id] = counter
					go self.countloop(counter, countreq.id, client)
				}
				go func() { countreq.ret <- <-counter }()
			case *DelReq:
				delreq := req.(*DelReq)
				delete(counters, delreq.id)
				path := "nexter/" + strconv.Itoa(delreq.id)
				client.Delete(path, true)
			default:
			}
		case <-self.done:
			break
		}
	}
}

func (self *Nexter) GetCount(id int) uint64 {
	ret := make(chan uint64)
	self.reqchan <- &IncReq{id, ret}
	return <-ret
}
func (self *Nexter) Delete(id int) {
	self.reqchan <- &DelReq{id}
}
func (self *Nexter) Stop() {
	self.done <- true
}

func NewNexter() (*Nexter, error) {
	stats, err := statsd.New(stats, "")
	if err != nil {
		return nil, err
	}
	nexter := &Nexter{make(chan Req), make(chan bool), stats}
	go nexter.loop()
	return nexter, nil
}

func main() {
	logf, err := os.OpenFile(logfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		log.Println("Error opening file: %v", err)
	}
	log.SetOutput(logf)
	log.Println("Starting Nexter...")
	nexter, err := NewNexter()
	if err != nil {
		log.Fatal(err)
	}
	http.HandleFunc("/nexter/", func(w http.ResponseWriter, r *http.Request) {
		url := r.URL.Path
		splits := strings.Split(url, "/")
		if len(splits) < 3 {
			http.Error(w, "Missing property id", http.StatusBadRequest)
			return
		}
		id_string := splits[2]
		id, err := strconv.Atoi(id_string)
		if err != nil {
			http.Error(w, "Property id is not a number", http.StatusBadRequest)
			return
		}

		num := nexter.GetCount(id)
		dec := json.NewEncoder(w)
		dec.Encode(num)
	})
	port_string := strconv.FormatUint(uint64(port), 10)
	http.ListenAndServe(":"+port_string, nil)
}
<commit_msg>Log each integer<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"strconv"
	"strings"

	"github.com/cactus/go-statsd-client/statsd"
	"github.com/coreos/go-etcd/etcd"
)

type stringslice []string

func (self *stringslice) String() string {
	return fmt.Sprintf("%d", *self)
}

func (self *stringslice) Set(value string) error {
	*self = append(*self, value)
	return nil
}

var port uint
var blocksize uint64
var etcd_nodes stringslice
var stats string
var logfile string

func init() {
	flag.UintVar(&port, "port", 9000, "Port to run HTTP server on")
	flag.Uint64Var(&blocksize, "blocksize", 64, "Block size")
	flag.Var(&etcd_nodes, "etcd", "Etcd server")
	flag.StringVar(&stats, "statsd", "127.0.0.1:8125", "Statsd server")
	flag.StringVar(&logfile, "log", "/tmp/nexter.log", "Log file name")
	flag.Parse()
}

type Req interface{}

type IncReq struct {
	id  int
	ret chan uint64
}

type DelReq struct {
	id int
}

type Nexter struct {
	reqchan chan Req
	done    chan bool
	stats   *statsd.Client
}

func (self *Nexter) countloop(ch chan uint64, id int, client *etcd.Client) {
	var start uint64
	var end uint64
	path := "nexter/" + strconv.Itoa(id)

	for {
		node, err := client.Get(path, false, false)
		if err != nil {
			ee, ok := err.(*etcd.EtcdError)
			if ok && ee.ErrorCode == 100 { // node does not exist
				_, err := client.Create(path, "0", 0)
				if err != nil {
					ee, ok := err.(*etcd.EtcdError)
					// Catch race condition where another node did the same client.Create()
					if ok && ee.ErrorCode == 105 { // Node has been created
						continue
					} else {
						log.Fatal(err)
					}
				}
				continue
			} else {
				log.Fatal(err)
			}
		} else { // No error, get start of series from etcd node
			start, err = strconv.ParseUint(node.Node.Value, 10, 0)
			if err != nil {
				log.Fatal(err)
			}
			end = start + blocksize
		}
		_, err = client.CompareAndSwap(path, strconv.FormatUint(end, 10), 0, strconv.FormatUint(start, 10), 0)
		if err != nil {
			log.Println("CAS failure", path, start, end)
			continue
		}
		self.stats.Gauge("nexter."+strconv.Itoa(id), int64(end), 1.0)
		log.Println("Allocated", id, start, end)
		for c := start; c < end; c += 1 {
			ch <- c
			log.Println("Send", id, c)
		}
	}
}

func (self *Nexter) loop() {
	counters := make(map[int]chan uint64)
	client := etcd.NewClient(etcd_nodes)
	for {
		select {
		case req := <-self.reqchan:
			switch req.(type) {
			case *IncReq:
				countreq := req.(*IncReq)
				counter, ok := counters[countreq.id]
				if !ok {
					counter = make(chan uint64)
					counters[countreq.id] = counter
					go self.countloop(counter, countreq.id, client)
				}
				go func() { countreq.ret <- <-counter }()
			case *DelReq:
				delreq := req.(*DelReq)
				delete(counters, delreq.id)
				path := "nexter/" + strconv.Itoa(delreq.id)
				client.Delete(path, true)
			default:
			}
		case <-self.done:
			break
		}
	}
}

func (self *Nexter) GetCount(id int) uint64 {
	ret := make(chan uint64)
	self.reqchan <- &IncReq{id, ret}
	return <-ret
}
func (self *Nexter) Delete(id int) {
	self.reqchan <- &DelReq{id}
}
func (self *Nexter) Stop() {
	self.done <- true
}

func NewNexter() (*Nexter, error) {
	stats, err := statsd.New(stats, "")
	if err != nil {
		return nil, err
	}
	nexter := &Nexter{make(chan Req), make(chan bool), stats}
	go nexter.loop()
	return nexter, nil
}

func main() {
	logf, err := os.OpenFile(logfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		log.Println("Error opening file: %v", err)
	}
	log.SetOutput(logf)
	log.Println("Starting Nexter...")
	nexter, err := NewNexter()
	if err != nil {
		log.Fatal(err)
	}
	http.HandleFunc("/nexter/", func(w http.ResponseWriter, r *http.Request) {
		url := r.URL.Path
		splits := strings.Split(url, "/")
		if len(splits) < 3 {
			http.Error(w, "Missing property id", http.StatusBadRequest)
			return
		}
		id_string := splits[2]
		id, err := strconv.Atoi(id_string)
		if err != nil {
			http.Error(w, "Property id is not a number", http.StatusBadRequest)
			return
		}

		num := nexter.GetCount(id)
		dec := json.NewEncoder(w)
		dec.Encode(num)
	})
	port_string := strconv.FormatUint(uint64(port), 10)
	http.ListenAndServe(":"+port_string, nil)
}
<|endoftext|>


<commit_before>package formatter

import (
	"strings"

	"github.com/docker/docker/api/types"
	"github.com/docker/go-units"
	"github.com/moncho/dry/docker"
)

const (
	imageIDHeader = "IMAGE ID"
	repository    = "REPOSITORY"
	tag           = "TAG"
	digest        = "DIGEST"
	createdSince  = "CREATEDSINCE"
	size          = "SIZE"
)

//ImageFormatter knows how to pretty-print the information of an image
type ImageFormatter struct {
	trunc  bool
	header []string
	image  types.ImageSummary
}

//NewImageFormatter creates an image formatter
func NewImageFormatter(image types.ImageSummary, trunc bool) *ImageFormatter {
	return &ImageFormatter{trunc: trunc, image: image}
}

func (formatter *ImageFormatter) addHeader(header string) {
	if formatter.header == nil {
		formatter.header = []string{}
	}
	formatter.header = append(formatter.header, strings.ToUpper(header))
}

//ID prettifies the id
func (formatter *ImageFormatter) ID() string {
	formatter.addHeader(imageIDHeader)
	if formatter.trunc {
		return docker.TruncateID(docker.ImageID(formatter.image.ID))
	}
	return docker.ImageID(formatter.image.ID)
}

//Repository prettifies the repository
func (formatter *ImageFormatter) Repository() string {
	formatter.addHeader(repository)
	if len(formatter.image.RepoTags) > 0 {
		tagPos := strings.LastIndex(formatter.image.RepoTags[0], ":")
		if tagPos > 0 {
			return formatter.image.RepoTags[0][:tagPos]
		}
		return formatter.image.RepoTags[0]
	} else if len(formatter.image.RepoDigests) > 0 {
		tagPos := strings.LastIndex(formatter.image.RepoDigests[0], "@")
		if tagPos > 0 {
			return formatter.image.RepoDigests[0][:tagPos]
		}
		return formatter.image.RepoDigests[0]
	}

	return "<none>"
}

//Tag prettifies the tag
func (formatter *ImageFormatter) Tag() string {
	formatter.addHeader(tag)
	if len(formatter.image.RepoTags) > 0 {
		tagPos := strings.LastIndex(formatter.image.RepoTags[0], ":")
		return formatter.image.RepoTags[0][tagPos+1:]
	}
	return "<none>"

}

//Digest prettifies the image digestv
func (formatter *ImageFormatter) Digest() string {
	formatter.addHeader(digest)
	if len(formatter.image.RepoDigests) == 0 {
		return ""
	}
	return formatter.image.RepoDigests[0]
}

//CreatedSince prettifies the image creation date
func (formatter *ImageFormatter) CreatedSince() string {
	formatter.addHeader(createdSince)

	return docker.DurationForHumans(int64(formatter.image.Created))
}

//Size prettifies the image size
func (formatter *ImageFormatter) Size() string {

	formatter.addHeader(size)
	//srw := units.HumanSize(float64(formatter.image.Size))
	//sf := srw

	if formatter.image.VirtualSize > 0 {
		sv := units.HumanSize(float64(formatter.image.VirtualSize))
		//sf = fmt.Sprintf("%s (virtual %s)", srw, sv)
		return sv
	}
	return ""
}
<commit_msg>Avoid unnecessary conversion<commit_after>package formatter

import (
	"strings"

	"github.com/docker/docker/api/types"
	"github.com/docker/go-units"
	"github.com/moncho/dry/docker"
)

const (
	imageIDHeader = "IMAGE ID"
	repository    = "REPOSITORY"
	tag           = "TAG"
	digest        = "DIGEST"
	createdSince  = "CREATEDSINCE"
	size          = "SIZE"
)

//ImageFormatter knows how to pretty-print the information of an image
type ImageFormatter struct {
	trunc  bool
	header []string
	image  types.ImageSummary
}

//NewImageFormatter creates an image formatter
func NewImageFormatter(image types.ImageSummary, trunc bool) *ImageFormatter {
	return &ImageFormatter{trunc: trunc, image: image}
}

func (formatter *ImageFormatter) addHeader(header string) {
	if formatter.header == nil {
		formatter.header = []string{}
	}
	formatter.header = append(formatter.header, strings.ToUpper(header))
}

//ID prettifies the id
func (formatter *ImageFormatter) ID() string {
	formatter.addHeader(imageIDHeader)
	if formatter.trunc {
		return docker.TruncateID(docker.ImageID(formatter.image.ID))
	}
	return docker.ImageID(formatter.image.ID)
}

//Repository prettifies the repository
func (formatter *ImageFormatter) Repository() string {
	formatter.addHeader(repository)
	if len(formatter.image.RepoTags) > 0 {
		tagPos := strings.LastIndex(formatter.image.RepoTags[0], ":")
		if tagPos > 0 {
			return formatter.image.RepoTags[0][:tagPos]
		}
		return formatter.image.RepoTags[0]
	} else if len(formatter.image.RepoDigests) > 0 {
		tagPos := strings.LastIndex(formatter.image.RepoDigests[0], "@")
		if tagPos > 0 {
			return formatter.image.RepoDigests[0][:tagPos]
		}
		return formatter.image.RepoDigests[0]
	}

	return "<none>"
}

//Tag prettifies the tag
func (formatter *ImageFormatter) Tag() string {
	formatter.addHeader(tag)
	if len(formatter.image.RepoTags) > 0 {
		tagPos := strings.LastIndex(formatter.image.RepoTags[0], ":")
		return formatter.image.RepoTags[0][tagPos+1:]
	}
	return "<none>"

}

//Digest prettifies the image digestv
func (formatter *ImageFormatter) Digest() string {
	formatter.addHeader(digest)
	if len(formatter.image.RepoDigests) == 0 {
		return ""
	}
	return formatter.image.RepoDigests[0]
}

//CreatedSince prettifies the image creation date
func (formatter *ImageFormatter) CreatedSince() string {
	formatter.addHeader(createdSince)

	return docker.DurationForHumans(formatter.image.Created)
}

//Size prettifies the image size
func (formatter *ImageFormatter) Size() string {

	formatter.addHeader(size)
	//srw := units.HumanSize(float64(formatter.image.Size))
	//sf := srw

	if formatter.image.VirtualSize > 0 {
		sv := units.HumanSize(float64(formatter.image.VirtualSize))
		//sf = fmt.Sprintf("%s (virtual %s)", srw, sv)
		return sv
	}
	return ""
}
<|endoftext|>


<commit_before>package ams

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"reflect"
	"testing"
	"time"
)

func TestLocator_ToUploadURL(t *testing.T) {
	locator := Locator{
		Path: "https://fake.url/upload?with=sas_tokens",
	}
	u, err := locator.ToUploadURL("test.mp4")
	if err != nil {
		t.Error(err)
	}
	expected := "https://fake.url/upload/test.mp4?with=sas_tokens"
	actual := u.String()

	if actual != expected {
		t.Error("unexpected UploadURL. expected: %v, actual: %v", expected, actual)
	}
}

func TestClient_CreateLocator(t *testing.T) {
	accessPolicyID := "sample-access-policy-id"
	assetID := "sample-asset-id"
	startTime := time.Now()
	locatorType := LocatorSAS

	expected := &Locator{
		ID:                     "sample-locator-id",
		ExpirationDateTime:     formatTime(time.Now()),
		Type:                   locatorType,
		Path:                   "https://fake.url/upload?with=sas_tokens",
		BaseURI:                "https://fake.url",
		ContentAccessComponent: "",
		AccessPolicyID:         accessPolicyID,
		AssetID:                assetID,
		StartTime:              formatTime(startTime),
		Name:                   "Sample Locator",
	}

	m := http.NewServeMux()
	m.HandleFunc("/Locators", func(w http.ResponseWriter, r *http.Request) {
		testRequestMethod(t, r, http.MethodPost)
		testAMSHeader(t, r.Header, false)

		var params struct {
			AccessPolicyID string `json:"AccessPolicyId"`
			AssetID        string `json:"AssetId"`
			StartTime      string `json:"StartTime"`
			Type           int    `json:"Type"`
		}

		if err := json.NewDecoder(r.Body).Decode(&params); err != nil {
			t.Fatal(err)
		}

		if params.AccessPolicyID != accessPolicyID {
			t.Errorf("unexpected AccessPolicyId. expected: %v, actual: %v", accessPolicyID, params.AccessPolicyID)
		}
		if params.AssetID != assetID {
			t.Errorf("unexpected AssetId. expected: %v, actual: %v", assetID, params.AssetID)
		}
		if params.StartTime != formatTime(startTime) {
			t.Errorf("unexpected StartTime. expected: %v, actual: %v", formatTime(startTime), params.StartTime)
		}
		if params.Type != locatorType {
			t.Errorf("unexpected Type. expected: %v, actual: %v", locatorType, params.Type)
		}

		w.WriteHeader(http.StatusCreated)
		if err := json.NewEncoder(w).Encode(expected); err != nil {
			t.Fatal(err)
		}
	})

	s := httptest.NewServer(m)
	client := testClient(t, s.URL)

	actual, err := client.CreateLocator(context.TODO(), accessPolicyID, assetID, startTime, locatorType)
	if err != nil {
		t.Error(err)
	}
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("unexpected locator. expected: %#v, actual: %#v", expected, actual)
	}
}

func TestClient_DeleteLocator(t *testing.T) {
	locatorID := "delete-locator-id"

	m := http.NewServeMux()
	m.HandleFunc(fmt.Sprintf("/Locators('%v')", locatorID),
		testJSONHandler(t, http.MethodDelete, false, http.StatusNoContent, nil),
	)
	s := httptest.NewServer(m)
	client := testClient(t, s.URL)
	if err := client.DeleteLocator(context.TODO(), locatorID); err != nil {
		t.Error(err)
	}
}
<commit_msg>fix: Error to Errorf<commit_after>package ams

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"reflect"
	"testing"
	"time"
)

func TestLocator_ToUploadURL(t *testing.T) {
	locator := Locator{
		Path: "https://fake.url/upload?with=sas_tokens",
	}
	u, err := locator.ToUploadURL("test.mp4")
	if err != nil {
		t.Error(err)
	}
	expected := "https://fake.url/upload/test.mp4?with=sas_tokens"
	actual := u.String()

	if actual != expected {
		t.Errorf("unexpected UploadURL. expected: %v, actual: %v", expected, actual)
	}
}

func TestClient_CreateLocator(t *testing.T) {
	accessPolicyID := "sample-access-policy-id"
	assetID := "sample-asset-id"
	startTime := time.Now()
	locatorType := LocatorSAS

	expected := &Locator{
		ID:                     "sample-locator-id",
		ExpirationDateTime:     formatTime(time.Now()),
		Type:                   locatorType,
		Path:                   "https://fake.url/upload?with=sas_tokens",
		BaseURI:                "https://fake.url",
		ContentAccessComponent: "",
		AccessPolicyID:         accessPolicyID,
		AssetID:                assetID,
		StartTime:              formatTime(startTime),
		Name:                   "Sample Locator",
	}

	m := http.NewServeMux()
	m.HandleFunc("/Locators", func(w http.ResponseWriter, r *http.Request) {
		testRequestMethod(t, r, http.MethodPost)
		testAMSHeader(t, r.Header, false)

		var params struct {
			AccessPolicyID string `json:"AccessPolicyId"`
			AssetID        string `json:"AssetId"`
			StartTime      string `json:"StartTime"`
			Type           int    `json:"Type"`
		}

		if err := json.NewDecoder(r.Body).Decode(&params); err != nil {
			t.Fatal(err)
		}

		if params.AccessPolicyID != accessPolicyID {
			t.Errorf("unexpected AccessPolicyId. expected: %v, actual: %v", accessPolicyID, params.AccessPolicyID)
		}
		if params.AssetID != assetID {
			t.Errorf("unexpected AssetId. expected: %v, actual: %v", assetID, params.AssetID)
		}
		if params.StartTime != formatTime(startTime) {
			t.Errorf("unexpected StartTime. expected: %v, actual: %v", formatTime(startTime), params.StartTime)
		}
		if params.Type != locatorType {
			t.Errorf("unexpected Type. expected: %v, actual: %v", locatorType, params.Type)
		}

		w.WriteHeader(http.StatusCreated)
		if err := json.NewEncoder(w).Encode(expected); err != nil {
			t.Fatal(err)
		}
	})

	s := httptest.NewServer(m)
	client := testClient(t, s.URL)

	actual, err := client.CreateLocator(context.TODO(), accessPolicyID, assetID, startTime, locatorType)
	if err != nil {
		t.Error(err)
	}
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("unexpected locator. expected: %#v, actual: %#v", expected, actual)
	}
}

func TestClient_DeleteLocator(t *testing.T) {
	locatorID := "delete-locator-id"

	m := http.NewServeMux()
	m.HandleFunc(fmt.Sprintf("/Locators('%v')", locatorID),
		testJSONHandler(t, http.MethodDelete, false, http.StatusNoContent, nil),
	)
	s := httptest.NewServer(m)
	client := testClient(t, s.URL)
	if err := client.DeleteLocator(context.TODO(), locatorID); err != nil {
		t.Error(err)
	}
}
<|endoftext|>


<commit_before>package appfile

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/hashicorp/go-multierror"
	"github.com/hashicorp/terraform/config/module"
	"github.com/hashicorp/terraform/dag"
)

const (
	// CompileVersion is the current version that we're on for
	// compilation formats. This can be used in the future to change
	// the directory structure and on-disk format of compiled appfiles.
	CompileVersion = 1

	CompileFilename        = "Appfile.compiled"
	CompileDepsFolder      = "deps"
	CompileVersionFilename = "version"
)

// Compiled represents a "Compiled" Appfile. A compiled Appfile is one
// that has loaded all of its dependency Appfiles, completed its imports,
// verified it is valid, etc.
//
// Appfile compilation is a process that requires network activity and
// has to occur once. The idea is that after compilation, a fully compiled
// Appfile can then be loaded in the future without network connectivity.
// Additionally, since we can assume it is valid, we can load it very quickly.
type Compiled struct {
	// File is the raw Appfile
	File *File

	// Graph is the DAG that has all the dependencies. This is already
	// verified to have no cycles. Each vertex is a *CompiledGraphVertex.
	Graph *dag.AcyclicGraph
}

func (c *Compiled) Validate() error {
	var result error
	if cycles := c.Graph.Cycles(); len(cycles) > 0 {
		for _, cycle := range cycles {
			vertices := make([]string, len(cycle))
			for i, v := range cycle {
				vertices[i] = dag.VertexName(v)
			}

			result = multierror.Append(result, fmt.Errorf(
				"Dependency cycle: %s", strings.Join(vertices, ", ")))
		}
	}

	return result
}

func (c *Compiled) String() string {
	var buf bytes.Buffer
	buf.WriteString(fmt.Sprintf("Compiled Appfile: %s\n\n", c.File.Path))
	buf.WriteString("Dep Graph:\n")
	buf.WriteString(c.Graph.String())
	buf.WriteString("\n")
	return buf.String()
}

// CompiledGraphVertex is the type of the vertex within the Graph of Compiled.
type CompiledGraphVertex struct {
	// File is the raw Appfile that this represents
	File *File

	// Dir is the directory of the data root for this dependency. This
	// is only non-empty for dependencies (the root vertex does not have
	// this value).
	Dir string

	// Don't use this outside of this package.
	NameValue string
}

func (v *CompiledGraphVertex) Name() string {
	return v.NameValue
}

// CompileOpts are the options for compilation.
type CompileOpts struct {
	// Dir is the directory where all the compiled data will be stored.
	// For use of Otto with a compiled Appfile, this directory must not
	// be deleted.
	Dir string

	// Callback is an optional way to receive notifications of events
	// during the compilation process. The CompileEvent argument should be
	// type switched to determine what it is.
	Callback func(CompileEvent)
}

// CompileEvent is a potential event that a Callback can receive during
// Compilation.
type CompileEvent interface{}

// CompileEventDep is the event that is called when a dependency is
// being loaded.
type CompileEventDep struct {
	Source string
}

// LoadCompiled loads and verifies a compiled Appfile (*Compiled) from
// disk.
func LoadCompiled(dir string) (*Compiled, error) {
	f, err := os.Open(filepath.Join(dir, CompileFilename))
	if err != nil {
		return nil, err
	}
	defer f.Close()

	var c Compiled
	dec := json.NewDecoder(f)
	if err := dec.Decode(&c); err != nil {
		return nil, err
	}

	return &c, nil
}

// Compile compiles an Appfile.
//
// This may require network connectivity if there are imports or
// non-local dependencies. The repositories that dependencies point to
// will be fully loaded into the given directory, and the compiled Appfile
// will be saved there.
//
// LoadCompiled can be used to load a pre-compiled Appfile.
//
// If you have no interest in reloading a compiled Appfile, you can
// recursively delete the compilation directory after this is completed.
// Note that certain functions of Otto such as development environments
// will depend on those directories existing, however.
func Compile(f *File, opts *CompileOpts) (*Compiled, error) {
	// First clear the directory. In the future, we can keep it around
	// and do incremental compilations.
	if err := os.RemoveAll(opts.Dir); err != nil {
		return nil, err
	}
	if err := os.MkdirAll(opts.Dir, 0755); err != nil {
		return nil, err
	}

	// Write the version of the compilation that we'll be completing.
	if err := compileVersion(opts.Dir); err != nil {
		return nil, fmt.Errorf("Error writing compiled Appfile version: %s", err)
	}

	// Start building our compiled Appfile
	compiled := &Compiled{File: f, Graph: new(dag.AcyclicGraph)}

	// Add our root vertex for this Appfile
	vertex := &CompiledGraphVertex{File: f, NameValue: f.Application.Name}
	compiled.Graph.Add(vertex)

	// Build the storage we'll use for storing downloaded dependencies,
	// then use that to trigger the recursive call to download all our
	// dependencies.
	storage := &module.FolderStorage{
		StorageDir: filepath.Join(opts.Dir, CompileDepsFolder)}
	if err := compileDependencies(storage, compiled.Graph, opts, vertex); err != nil {
		return nil, err
	}

	// Validate the compiled file tree.
	if err := compiled.Validate(); err != nil {
		return nil, err
	}

	// Write the compiled Appfile data
	if err := compileWrite(opts.Dir, compiled); err != nil {
		return nil, err
	}

	return compiled, nil
}

func compileDependencies(
	storage module.Storage,
	graph *dag.AcyclicGraph,
	opts *CompileOpts,
	root *CompiledGraphVertex) error {
	// Make a map to keep track of the dep source to vertex mapping
	vertexMap := make(map[string]*CompiledGraphVertex)

	// Store ourselves in the map
	key, err := module.Detect(".", filepath.Dir(root.File.Path))
	if err != nil {
		return err
	}
	vertexMap[key] = root

	// Make a queue for the other vertices we need to still get
	// dependencies for. We arbitrarily make the cap for this slice
	// 30, since that is a ton of dependencies and we don't expect the
	// average case to have more than this.
	queue := make([]*CompiledGraphVertex, 1, 30)
	queue[0] = root

	// While we still have dependencies to get, continue loading them.
	// TODO: parallelize
	for len(queue) > 0 {
		var current *CompiledGraphVertex
		current, queue = queue[len(queue)-1], queue[:len(queue)-1]

		log.Printf("[DEBUG] compiling dependencies for: %s", current.Name())
		for _, dep := range current.File.Application.Dependencies {
			key, err := module.Detect(dep.Source, filepath.Dir(current.File.Path))
			if err != nil {
				return fmt.Errorf(
					"Error loading source: %s", err)
			}

			vertex := vertexMap[key]
			if vertex == nil {
				log.Printf("[DEBUG] loading dependency: %s", key)

				// Call the callback if we have one
				if opts.Callback != nil {
					opts.Callback(&CompileEventDep{
						Source: key,
					})
				}

				// Download the dependency
				if err := storage.Get(key, key, true); err != nil {
					return err
				}
				dir, _, err := storage.Dir(key)
				if err != nil {
					return err
				}

				// Parse the Appfile
				f, err := ParseFile(filepath.Join(dir, "Appfile"))
				if err != nil {
					return fmt.Errorf(
						"Error parsing Appfile in %s: %s", key, err)
				}

				// Build the vertex for this
				vertex = &CompiledGraphVertex{
					File:      f,
					Dir:       dir,
					NameValue: f.Application.Name,
				}

				// Add the vertex since it is new, store the mapping, and
				// queue it to be loaded later.
				graph.Add(vertex)
				vertexMap[key] = vertex
				queue = append(queue, vertex)
			}

			// Connect the dependencies
			graph.Connect(dag.BasicEdge(current, vertex))
		}
	}

	return nil
}

func compileVersion(dir string) error {
	f, err := os.Create(filepath.Join(dir, CompileVersionFilename))
	if err != nil {
		return err
	}
	defer f.Close()

	_, err = fmt.Fprintf(f, "%d", CompileVersion)
	return err
}

func compileWrite(dir string, compiled *Compiled) error {
	// Pretty-print the JSON data so that it can be more easily inspected
	data, err := json.MarshalIndent(compiled, "", "    ")
	if err != nil {
		return err
	}

	// Write it out
	f, err := os.Create(filepath.Join(dir, CompileFilename))
	if err != nil {
		return err
	}
	defer f.Close()

	_, err = io.Copy(f, bytes.NewReader(data))
	return err
}
<commit_msg>appfile: just a comment<commit_after>package appfile

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/hashicorp/go-multierror"
	"github.com/hashicorp/terraform/config/module"
	"github.com/hashicorp/terraform/dag"
)

const (
	// CompileVersion is the current version that we're on for
	// compilation formats. This can be used in the future to change
	// the directory structure and on-disk format of compiled appfiles.
	CompileVersion = 1

	CompileFilename        = "Appfile.compiled"
	CompileDepsFolder      = "deps"
	CompileVersionFilename = "version"
)

// Compiled represents a "Compiled" Appfile. A compiled Appfile is one
// that has loaded all of its dependency Appfiles, completed its imports,
// verified it is valid, etc.
//
// Appfile compilation is a process that requires network activity and
// has to occur once. The idea is that after compilation, a fully compiled
// Appfile can then be loaded in the future without network connectivity.
// Additionally, since we can assume it is valid, we can load it very quickly.
type Compiled struct {
	// File is the raw Appfile
	File *File

	// Graph is the DAG that has all the dependencies. This is already
	// verified to have no cycles. Each vertex is a *CompiledGraphVertex.
	Graph *dag.AcyclicGraph
}

func (c *Compiled) Validate() error {
	var result error

	// First validate that there are no cycles in the dependency graph
	if cycles := c.Graph.Cycles(); len(cycles) > 0 {
		for _, cycle := range cycles {
			vertices := make([]string, len(cycle))
			for i, v := range cycle {
				vertices[i] = dag.VertexName(v)
			}

			result = multierror.Append(result, fmt.Errorf(
				"Dependency cycle: %s", strings.Join(vertices, ", ")))
		}
	}

	return result
}

func (c *Compiled) String() string {
	var buf bytes.Buffer
	buf.WriteString(fmt.Sprintf("Compiled Appfile: %s\n\n", c.File.Path))
	buf.WriteString("Dep Graph:\n")
	buf.WriteString(c.Graph.String())
	buf.WriteString("\n")
	return buf.String()
}

// CompiledGraphVertex is the type of the vertex within the Graph of Compiled.
type CompiledGraphVertex struct {
	// File is the raw Appfile that this represents
	File *File

	// Dir is the directory of the data root for this dependency. This
	// is only non-empty for dependencies (the root vertex does not have
	// this value).
	Dir string

	// Don't use this outside of this package.
	NameValue string
}

func (v *CompiledGraphVertex) Name() string {
	return v.NameValue
}

// CompileOpts are the options for compilation.
type CompileOpts struct {
	// Dir is the directory where all the compiled data will be stored.
	// For use of Otto with a compiled Appfile, this directory must not
	// be deleted.
	Dir string

	// Callback is an optional way to receive notifications of events
	// during the compilation process. The CompileEvent argument should be
	// type switched to determine what it is.
	Callback func(CompileEvent)
}

// CompileEvent is a potential event that a Callback can receive during
// Compilation.
type CompileEvent interface{}

// CompileEventDep is the event that is called when a dependency is
// being loaded.
type CompileEventDep struct {
	Source string
}

// LoadCompiled loads and verifies a compiled Appfile (*Compiled) from
// disk.
func LoadCompiled(dir string) (*Compiled, error) {
	f, err := os.Open(filepath.Join(dir, CompileFilename))
	if err != nil {
		return nil, err
	}
	defer f.Close()

	var c Compiled
	dec := json.NewDecoder(f)
	if err := dec.Decode(&c); err != nil {
		return nil, err
	}

	return &c, nil
}

// Compile compiles an Appfile.
//
// This may require network connectivity if there are imports or
// non-local dependencies. The repositories that dependencies point to
// will be fully loaded into the given directory, and the compiled Appfile
// will be saved there.
//
// LoadCompiled can be used to load a pre-compiled Appfile.
//
// If you have no interest in reloading a compiled Appfile, you can
// recursively delete the compilation directory after this is completed.
// Note that certain functions of Otto such as development environments
// will depend on those directories existing, however.
func Compile(f *File, opts *CompileOpts) (*Compiled, error) {
	// First clear the directory. In the future, we can keep it around
	// and do incremental compilations.
	if err := os.RemoveAll(opts.Dir); err != nil {
		return nil, err
	}
	if err := os.MkdirAll(opts.Dir, 0755); err != nil {
		return nil, err
	}

	// Write the version of the compilation that we'll be completing.
	if err := compileVersion(opts.Dir); err != nil {
		return nil, fmt.Errorf("Error writing compiled Appfile version: %s", err)
	}

	// Start building our compiled Appfile
	compiled := &Compiled{File: f, Graph: new(dag.AcyclicGraph)}

	// Add our root vertex for this Appfile
	vertex := &CompiledGraphVertex{File: f, NameValue: f.Application.Name}
	compiled.Graph.Add(vertex)

	// Build the storage we'll use for storing downloaded dependencies,
	// then use that to trigger the recursive call to download all our
	// dependencies.
	storage := &module.FolderStorage{
		StorageDir: filepath.Join(opts.Dir, CompileDepsFolder)}
	if err := compileDependencies(storage, compiled.Graph, opts, vertex); err != nil {
		return nil, err
	}

	// Validate the compiled file tree.
	if err := compiled.Validate(); err != nil {
		return nil, err
	}

	// Write the compiled Appfile data
	if err := compileWrite(opts.Dir, compiled); err != nil {
		return nil, err
	}

	return compiled, nil
}

func compileDependencies(
	storage module.Storage,
	graph *dag.AcyclicGraph,
	opts *CompileOpts,
	root *CompiledGraphVertex) error {
	// Make a map to keep track of the dep source to vertex mapping
	vertexMap := make(map[string]*CompiledGraphVertex)

	// Store ourselves in the map
	key, err := module.Detect(".", filepath.Dir(root.File.Path))
	if err != nil {
		return err
	}
	vertexMap[key] = root

	// Make a queue for the other vertices we need to still get
	// dependencies for. We arbitrarily make the cap for this slice
	// 30, since that is a ton of dependencies and we don't expect the
	// average case to have more than this.
	queue := make([]*CompiledGraphVertex, 1, 30)
	queue[0] = root

	// While we still have dependencies to get, continue loading them.
	// TODO: parallelize
	for len(queue) > 0 {
		var current *CompiledGraphVertex
		current, queue = queue[len(queue)-1], queue[:len(queue)-1]

		log.Printf("[DEBUG] compiling dependencies for: %s", current.Name())
		for _, dep := range current.File.Application.Dependencies {
			key, err := module.Detect(dep.Source, filepath.Dir(current.File.Path))
			if err != nil {
				return fmt.Errorf(
					"Error loading source: %s", err)
			}

			vertex := vertexMap[key]
			if vertex == nil {
				log.Printf("[DEBUG] loading dependency: %s", key)

				// Call the callback if we have one
				if opts.Callback != nil {
					opts.Callback(&CompileEventDep{
						Source: key,
					})
				}

				// Download the dependency
				if err := storage.Get(key, key, true); err != nil {
					return err
				}
				dir, _, err := storage.Dir(key)
				if err != nil {
					return err
				}

				// Parse the Appfile
				f, err := ParseFile(filepath.Join(dir, "Appfile"))
				if err != nil {
					return fmt.Errorf(
						"Error parsing Appfile in %s: %s", key, err)
				}

				// Build the vertex for this
				vertex = &CompiledGraphVertex{
					File:      f,
					Dir:       dir,
					NameValue: f.Application.Name,
				}

				// Add the vertex since it is new, store the mapping, and
				// queue it to be loaded later.
				graph.Add(vertex)
				vertexMap[key] = vertex
				queue = append(queue, vertex)
			}

			// Connect the dependencies
			graph.Connect(dag.BasicEdge(current, vertex))
		}
	}

	return nil
}

func compileVersion(dir string) error {
	f, err := os.Create(filepath.Join(dir, CompileVersionFilename))
	if err != nil {
		return err
	}
	defer f.Close()

	_, err = fmt.Fprintf(f, "%d", CompileVersion)
	return err
}

func compileWrite(dir string, compiled *Compiled) error {
	// Pretty-print the JSON data so that it can be more easily inspected
	data, err := json.MarshalIndent(compiled, "", "    ")
	if err != nil {
		return err
	}

	// Write it out
	f, err := os.Create(filepath.Join(dir, CompileFilename))
	if err != nil {
		return err
	}
	defer f.Close()

	_, err = io.Copy(f, bytes.NewReader(data))
	return err
}
<|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// Contributor: Julien Vehent jvehent@mozilla.com [:ulfr]
package pgp

import (
	"bytes"
	"camlistore.org/pkg/misc/gpgagent"
	"camlistore.org/pkg/misc/pinentry"
	"code.google.com/p/go.crypto/openpgp"
	"encoding/hex"
	"fmt"
	"io"
	"os"
	"strings"
)

var cachedPassphrase string

// Sign signs a string with a key identified by a key fingerprint or an email address
func Sign(data, keyid string, secringFile io.Reader) (sig string, err error) {
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("pgp.Sign(): %v", e)
		}
	}()
	keyring, err := openpgp.ReadKeyRing(secringFile)
	if err != nil {
		err = fmt.Errorf("Keyring access failed: '%v'", err)
		panic(err)
	}

	// find the entity in the keyring
	var signer *openpgp.Entity
	found := false
	for _, entity := range keyring {
		fingerprint := strings.ToUpper(hex.EncodeToString(entity.PrimaryKey.Fingerprint[:]))
		for _, ident := range entity.Identities {
			email := ident.UserId.Email
			if keyid == fingerprint || keyid == email {
				signer = entity
				found = true
				break
			}
		}
	}
	if !found {
		err = fmt.Errorf("Signer '%s' not found", keyid)
		panic(err)
	}

	// if private key is encrypted, attempt to decrypt it with the cached passphrase
	// then try with an agent or by asking the user for a passphrase
	if signer.PrivateKey.Encrypted {
		err = signer.PrivateKey.Decrypt([]byte(cachedPassphrase))
		if err != nil {
			var pass string
			// get private key passphrase
			signer, pass, err = decryptEntity(signer)
			if err != nil {
				panic(err)
			}
			if pass != "" {
				cachedPassphrase = pass
			}
		}
	}

	// calculate signature
	out := bytes.NewBuffer(nil)
	message := bytes.NewBufferString(data)
	err = openpgp.ArmoredDetachSign(out, signer, message, nil)
	if err != nil {
		err = fmt.Errorf("Signature failed: '%v'", err)
		panic(err)
	}

	// convert the writer back to string
	sig, err = deArmor(out.String())
	if err != nil {
		err = fmt.Errorf("Error converting signature to string: '%v'", err)
		panic(err)
	}

	return
}

// deArmor takes a multi line armored GPG signature, and turns it back
// into a single line signature (thank you, camlistore folks)
func deArmor(sig string) (str string, err error) {
	index1 := strings.Index(sig, "\n\n")
	index2 := strings.Index(sig, "\n-----")
	if index1 == -1 || index2 == -1 {
		err = fmt.Errorf("Failed to parse signature from gpg.")
		return
	}
	inner := sig[index1+2 : index2]
	str = strings.Replace(inner, "\n", "", -1)
	return
}

// decryptEntity calls gnupg-agent and pinentry to obtain a passphrase and
// decrypt the private key of a given entity (thank you, camlistore folks)
func decryptEntity(s *openpgp.Entity) (ds *openpgp.Entity, pass string, err error) {
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("pgp.decryptEntity(): %v", e)
		}
	}()
	ds = s
	// TODO: syscall.Mlock a region and keep pass phrase in it.
	pubk := &ds.PrivateKey.PublicKey
	desc := fmt.Sprintf("Need to unlock GPG key %s to use it for signing.",
		pubk.KeyIdShortString())

	conn, err := gpgagent.NewConn()
	switch err {
	case gpgagent.ErrNoAgent:
		fmt.Fprintf(os.Stderr, "Note: gpg-agent not found; resorting to on-demand password entry.\n")
	case nil:
		defer conn.Close()
		req := &gpgagent.PassphraseRequest{
			CacheKey: "mig:pgpsign:" + pubk.KeyIdShortString(),
			Prompt:   "Passphrase",
			Desc:     desc,
		}
		for tries := 0; tries < 2; tries++ {
			pass, err := conn.GetPassphrase(req)
			if err == nil {
				err = ds.PrivateKey.Decrypt([]byte(pass))
				if err == nil {
					return ds, pass, err
				}
				req.Error = "Passphrase failed to decrypt: " + err.Error()
				conn.RemoveFromCache(req.CacheKey)
				continue
			}
			if err == gpgagent.ErrCancel {
				panic("failed to decrypt key; action canceled")
			}
		}
	default:
		panic(err)
	}

	pinReq := &pinentry.Request{Desc: desc, Prompt: "Passphrase"}
	for tries := 0; tries < 2; tries++ {
		pass, err = pinReq.GetPIN()
		if err == nil {
			err = ds.PrivateKey.Decrypt([]byte(pass))
			if err == nil {
				return ds, pass, err
			}
			pinReq.Error = "Passphrase failed to decrypt: " + err.Error()
			continue
		}
		if err == pinentry.ErrCancel {
			panic("failed to decrypt key; action canceled")
		}
	}
	return ds, "", fmt.Errorf("decryptEntity(): failed to decrypt key %q", pubk.KeyIdShortString())
}
<commit_msg>[minor] Use 3 tries for pinetry (when not using local agent)<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// Contributor: Julien Vehent jvehent@mozilla.com [:ulfr]
package pgp

import (
	"bytes"
	"camlistore.org/pkg/misc/gpgagent"
	"camlistore.org/pkg/misc/pinentry"
	"code.google.com/p/go.crypto/openpgp"
	"encoding/hex"
	"fmt"
	"io"
	"os"
	"strings"
)

var cachedPassphrase string

// Sign signs a string with a key identified by a key fingerprint or an email address
func Sign(data, keyid string, secringFile io.Reader) (sig string, err error) {
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("pgp.Sign(): %v", e)
		}
	}()
	keyring, err := openpgp.ReadKeyRing(secringFile)
	if err != nil {
		err = fmt.Errorf("Keyring access failed: '%v'", err)
		panic(err)
	}

	// find the entity in the keyring
	var signer *openpgp.Entity
	found := false
	for _, entity := range keyring {
		fingerprint := strings.ToUpper(hex.EncodeToString(entity.PrimaryKey.Fingerprint[:]))
		for _, ident := range entity.Identities {
			email := ident.UserId.Email
			if keyid == fingerprint || keyid == email {
				signer = entity
				found = true
				break
			}
		}
	}
	if !found {
		err = fmt.Errorf("Signer '%s' not found", keyid)
		panic(err)
	}

	// if private key is encrypted, attempt to decrypt it with the cached passphrase
	// then try with an agent or by asking the user for a passphrase
	if signer.PrivateKey.Encrypted {
		err = signer.PrivateKey.Decrypt([]byte(cachedPassphrase))
		if err != nil {
			var pass string
			// get private key passphrase
			signer, pass, err = decryptEntity(signer)
			if err != nil {
				panic(err)
			}
			if pass != "" {
				cachedPassphrase = pass
			}
		}
	}

	// calculate signature
	out := bytes.NewBuffer(nil)
	message := bytes.NewBufferString(data)
	err = openpgp.ArmoredDetachSign(out, signer, message, nil)
	if err != nil {
		err = fmt.Errorf("Signature failed: '%v'", err)
		panic(err)
	}

	// convert the writer back to string
	sig, err = deArmor(out.String())
	if err != nil {
		err = fmt.Errorf("Error converting signature to string: '%v'", err)
		panic(err)
	}

	return
}

// deArmor takes a multi line armored GPG signature, and turns it back
// into a single line signature (thank you, camlistore folks)
func deArmor(sig string) (str string, err error) {
	index1 := strings.Index(sig, "\n\n")
	index2 := strings.Index(sig, "\n-----")
	if index1 == -1 || index2 == -1 {
		err = fmt.Errorf("Failed to parse signature from gpg.")
		return
	}
	inner := sig[index1+2 : index2]
	str = strings.Replace(inner, "\n", "", -1)
	return
}

// decryptEntity calls gnupg-agent and pinentry to obtain a passphrase and
// decrypt the private key of a given entity (thank you, camlistore folks)
func decryptEntity(s *openpgp.Entity) (ds *openpgp.Entity, pass string, err error) {
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("pgp.decryptEntity(): %v", e)
		}
	}()
	ds = s
	// TODO: syscall.Mlock a region and keep pass phrase in it.
	pubk := &ds.PrivateKey.PublicKey
	desc := fmt.Sprintf("Need to unlock GPG key %s to use it for signing.",
		pubk.KeyIdShortString())

	conn, err := gpgagent.NewConn()
	switch err {
	case gpgagent.ErrNoAgent:
		fmt.Fprintf(os.Stderr, "Note: gpg-agent not found; resorting to on-demand password entry.\n")
	case nil:
		defer conn.Close()
		req := &gpgagent.PassphraseRequest{
			CacheKey: "mig:pgpsign:" + pubk.KeyIdShortString(),
			Prompt:   "Passphrase",
			Desc:     desc,
		}
		for tries := 0; tries < 3; tries++ {
			pass, err := conn.GetPassphrase(req)
			if err == nil {
				err = ds.PrivateKey.Decrypt([]byte(pass))
				if err == nil {
					return ds, pass, err
				}
				req.Error = "Passphrase failed to decrypt: " + err.Error()
				conn.RemoveFromCache(req.CacheKey)
				continue
			}
			if err == gpgagent.ErrCancel {
				panic("failed to decrypt key; action canceled")
			}
		}
	default:
		panic(err)
	}

	pinReq := &pinentry.Request{Desc: desc, Prompt: "Passphrase"}
	for tries := 0; tries < 3; tries++ {
		pass, err = pinReq.GetPIN()
		if err == nil {

			err = ds.PrivateKey.Decrypt([]byte(pass))
			if err == nil {
				return ds, pass, err
			}
			pinReq.Error = "Passphrase failed to decrypt: " + err.Error()
			continue
		}
		if err == pinentry.ErrCancel {
			panic("failed to decrypt key; action canceled")
		}
	}
	return ds, "", fmt.Errorf("decryptEntity(): failed to decrypt key %q", pubk.KeyIdShortString())
}
<|endoftext|>