<commit_before>// Copyright 2013 Francisco Souza. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package testing provides a fake implementation of the Docker API, useful for
// testing purpose.
package testing

import (
	"crypto/rand"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/dotcloud/docker"
	"github.com/gorilla/mux"
	mathrand "math/rand"
	"net"
	"net/http"
	"strconv"
	"strings"
	"sync"
	"time"
)

// DockerServer represents a programmable, concurrent (not much), HTTP server
// implementing a fake version of the Docker remote API.
//
// It can used in standalone mode, listening for connections or as an arbitrary
// HTTP handler.
//
// For more details on the remote API, check http://goo.gl/yMI1S.
type DockerServer struct {
	containers []*docker.Container
	cMut       sync.RWMutex
	images     []docker.Image
	iMut       sync.RWMutex
	imgIDs     map[string]string
	listener   net.Listener
	mux        *mux.Router
	hook       func(*http.Request)
}

// NewServer returns a new instance of the fake server, in standalone mode. Use
// the method URL to get the URL of the server.
//
// Hook is a function that will be called on every request.
func NewServer(hook func(*http.Request)) (*DockerServer, error) {
	listener, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		return nil, err
	}
	server := DockerServer{listener: listener, imgIDs: make(map[string]string), hook: hook}
	server.buildMuxer()
	go http.Serve(listener, &server)
	return &server, nil
}

func (s *DockerServer) buildMuxer() {
	s.mux = mux.NewRouter()
	s.mux.Path("/v{version:[0-9.]+}/commit").Methods("POST").HandlerFunc(s.commitContainer)
	s.mux.Path("/v{version:[0-9.]+}/containers/json").Methods("GET").HandlerFunc(s.listContainers)
	s.mux.Path("/v{version:[0-9.]+}/containers/create").Methods("POST").HandlerFunc(s.createContainer)
	s.mux.Path("/v{version:[0-9.]+}/containers/{id:.*}/json").Methods("GET").HandlerFunc(s.inspectContainer)
	s.mux.Path("/v{version:[0-9.]+}/containers/{id:.*}/start").Methods("POST").HandlerFunc(s.startContainer)
	s.mux.Path("/v{version:[0-9.]+}/containers/{id:.*}/stop").Methods("POST").HandlerFunc(s.stopContainer)
	s.mux.Path("/v{version:[0-9.]+}/containers/{id:.*}/wait").Methods("POST").HandlerFunc(s.waitContainer)
	s.mux.Path("/v{version:[0-9.]+}/containers/{id:.*}/attach").Methods("POST").HandlerFunc(s.attachContainer)
	s.mux.Path("/v{version:[0-9.]+}/containers/{id:.*}").Methods("DELETE").HandlerFunc(s.removeContainer)
	s.mux.Path("/v{version:[0-9.]+}/images/create").Methods("POST").HandlerFunc(s.pullImage)
	s.mux.Path("/v{version:[0-9.]+}/images/json").Methods("GET").HandlerFunc(s.listImages)
	s.mux.Path("/v{version:[0-9.]+}/images/{id:.*}").Methods("DELETE").HandlerFunc(s.removeImage)
	s.mux.Path("/v{version:[0-9.]+}/images/{name:.*}/push").Methods("POST").HandlerFunc(s.pushImage)
}

// Stop stops the server.
func (s *DockerServer) Stop() {
	if s.listener != nil {
		s.listener.Close()
	}
}

// URL returns the HTTP URL of the server.
func (s *DockerServer) URL() string {
	if s.listener == nil {
		return ""
	}
	return "http://" + s.listener.Addr().String() + "/"
}

// ServeHTTP handles HTTP requests sent to the server.
func (s *DockerServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	s.mux.ServeHTTP(w, r)
	if s.hook != nil {
		s.hook(r)
	}
}

func (s *DockerServer) listContainers(w http.ResponseWriter, r *http.Request) {
	s.cMut.RLock()
	result := make([]docker.APIContainers, len(s.containers))
	for i, container := range s.containers {
		result[i] = docker.APIContainers{
			ID:      container.ID,
			Image:   container.Image,
			Command: fmt.Sprintf("%s %s", container.Path, strings.Join(container.Args, " ")),
			Created: container.Created.Unix(),
			Status:  container.State.String(),
			Ports:   container.NetworkSettings.PortMappingHuman(),
		}
	}
	s.cMut.RUnlock()
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(http.StatusOK)
	json.NewEncoder(w).Encode(result)
}

func (s *DockerServer) listImages(w http.ResponseWriter, r *http.Request) {
	s.cMut.RLock()
	result := make([]docker.APIImages, len(s.images))
	for i, image := range s.images {
		result[i] = docker.APIImages{
			ID:      image.ID,
			Created: image.Created.Unix(),
		}
	}
	s.cMut.RUnlock()
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(http.StatusOK)
	json.NewEncoder(w).Encode(result)
}

func (s *DockerServer) findImage(id string) (string, error) {
	s.iMut.RLock()
	defer s.iMut.RUnlock()
	image, ok := s.imgIDs[id]
	if ok {
		return image, nil
	}
	image, _, err := s.findImageByID(id)
	return image, err
}

func (s *DockerServer) findImageByID(id string) (string, int, error) {
	s.iMut.RLock()
	defer s.iMut.RUnlock()
	for i, image := range s.images {
		if image.ID == id {
			return image.ID, i, nil
		}
	}
	return "", -1, errors.New("No such image")
}

func (s *DockerServer) createContainer(w http.ResponseWriter, r *http.Request) {
	var config docker.Config
	defer r.Body.Close()
	err := json.NewDecoder(r.Body).Decode(&config)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	image, err := s.findImage(config.Image)
	if err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}
	w.WriteHeader(http.StatusCreated)
	portMapping := make(map[string]string, len(config.PortSpecs))
	for _, p := range config.PortSpecs {
		portMapping[p] = strconv.Itoa(mathrand.Int() % 65536)
	}
	container := docker.Container{
		ID:      s.generateID(),
		Created: time.Now(),
		Path:    config.Cmd[0],
		Args:    config.Cmd[1:],
		Config:  &config,
		State: docker.State{
			Running:   false,
			Pid:       mathrand.Int() % 50000,
			ExitCode:  0,
			StartedAt: time.Now(),
		},
		Image: image,
		NetworkSettings: &docker.NetworkSettings{
			IPAddress:   fmt.Sprintf("172.16.42.%d", mathrand.Int()%250+2),
			IPPrefixLen: 24,
			Gateway:     "172.16.42.1",
			Bridge:      "docker0",
			PortMapping: portMapping,
		},
	}
	s.cMut.Lock()
	s.containers = append(s.containers, &container)
	s.cMut.Unlock()
	var c = struct{ ID string }{ID: container.ID}
	json.NewEncoder(w).Encode(c)
}

func (s *DockerServer) generateID() string {
	var buf [16]byte
	rand.Read(buf[:])
	return fmt.Sprintf("%x", buf)
}

func (s *DockerServer) inspectContainer(w http.ResponseWriter, r *http.Request) {
	id := mux.Vars(r)["id"]
	container, _, err := s.findContainer(id)
	if err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(http.StatusOK)
	json.NewEncoder(w).Encode(container)
}

func (s *DockerServer) startContainer(w http.ResponseWriter, r *http.Request) {
	id := mux.Vars(r)["id"]
	container, _, err := s.findContainer(id)
	if err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}
	s.cMut.Lock()
	defer s.cMut.Unlock()
	if container.State.Running {
		http.Error(w, "Container already running", http.StatusBadRequest)
		return
	}
	container.State.Running = true
}

func (s *DockerServer) stopContainer(w http.ResponseWriter, r *http.Request) {
	id := mux.Vars(r)["id"]
	container, _, err := s.findContainer(id)
	if err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}
	s.cMut.Lock()
	defer s.cMut.Unlock()
	if !container.State.Running {
		http.Error(w, "Container not running", http.StatusBadRequest)
		return
	}
	w.WriteHeader(http.StatusNoContent)
	container.State.Running = false
}

func (s *DockerServer) attachContainer(w http.ResponseWriter, r *http.Request) {
	id := mux.Vars(r)["id"]
	container, _, err := s.findContainer(id)
	if err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}
	if container.State.Running {
		fmt.Fprintf(w, "Container %q is running\n", container.ID)
	} else {
		fmt.Fprintf(w, "Container %q is not running\n", container.ID)
	}
	fmt.Fprintln(w, "What happened?")
	fmt.Fprintln(w, "Something happened")
}

func (s *DockerServer) waitContainer(w http.ResponseWriter, r *http.Request) {
	id := mux.Vars(r)["id"]
	container, _, err := s.findContainer(id)
	if err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}
	for {
		s.cMut.RLock()
		if container.State.Running {
			s.cMut.RUnlock()
			break
		}
		s.cMut.RUnlock()
	}
	w.Write([]byte(`{"StatusCode":0}`))
}

func (s *DockerServer) removeContainer(w http.ResponseWriter, r *http.Request) {
	id := mux.Vars(r)["id"]
	_, index, err := s.findContainer(id)
	if err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}
	w.WriteHeader(http.StatusNoContent)
	s.cMut.Lock()
	defer s.cMut.Unlock()
	s.containers[index] = s.containers[len(s.containers)-1]
	s.containers = s.containers[:len(s.containers)-1]
}

func (s *DockerServer) commitContainer(w http.ResponseWriter, r *http.Request) {
	id := r.URL.Query().Get("container")
	container, _, err := s.findContainer(id)
	if err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}
	var config *docker.Config
	runConfig := r.URL.Query().Get("run")
	if runConfig != "" {
		config = new(docker.Config)
		err = json.Unmarshal([]byte(runConfig), config)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
	}
	w.WriteHeader(http.StatusOK)
	image := docker.Image{
		ID:        "img-" + container.ID,
		Parent:    container.Image,
		Container: container.ID,
		Comment:   r.URL.Query().Get("m"),
		Author:    r.URL.Query().Get("author"),
		Config:    config,
	}
	repository := r.URL.Query().Get("repo")
	s.iMut.Lock()
	s.images = append(s.images, image)
	if repository != "" {
		s.imgIDs[repository] = image.ID
	}
	s.iMut.Unlock()
	fmt.Fprintf(w, `{"ID":%q}`, image.ID)
}

func (s *DockerServer) findContainer(id string) (*docker.Container, int, error) {
	s.cMut.RLock()
	defer s.cMut.RUnlock()
	for i, container := range s.containers {
		if container.ID == id {
			return container, i, nil
		}
	}
	return nil, -1, errors.New("No such container")
}

func (s *DockerServer) pullImage(w http.ResponseWriter, r *http.Request) {
	repository := r.URL.Query().Get("fromImage")
	image := docker.Image{
		ID: s.generateID(),
	}
	s.iMut.Lock()
	s.images = append(s.images, image)
	if repository != "" {
		s.imgIDs[repository] = image.ID
	}
	s.iMut.Unlock()
}

func (s *DockerServer) pushImage(w http.ResponseWriter, r *http.Request) {
}

func (s *DockerServer) removeImage(w http.ResponseWriter, r *http.Request) {
	id := mux.Vars(r)["id"]
	_, index, err := s.findImageByID(id)
	if err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}
	w.WriteHeader(http.StatusNoContent)
	s.iMut.Lock()
	defer s.iMut.Unlock()
	s.images[index] = s.images[len(s.images)-1]
	s.images = s.images[:len(s.images)-1]
}
<commit_msg>testing: finish pushImage implementation<commit_after>// Copyright 2013 Francisco Souza. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package testing provides a fake implementation of the Docker API, useful for
// testing purpose.
package testing

import (
	"crypto/rand"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/dotcloud/docker"
	"github.com/gorilla/mux"
	mathrand "math/rand"
	"net"
	"net/http"
	"strconv"
	"strings"
	"sync"
	"time"
)

// DockerServer represents a programmable, concurrent (not much), HTTP server
// implementing a fake version of the Docker remote API.
//
// It can used in standalone mode, listening for connections or as an arbitrary
// HTTP handler.
//
// For more details on the remote API, check http://goo.gl/yMI1S.
type DockerServer struct {
	containers []*docker.Container
	cMut       sync.RWMutex
	images     []docker.Image
	iMut       sync.RWMutex
	imgIDs     map[string]string
	listener   net.Listener
	mux        *mux.Router
	hook       func(*http.Request)
}

// NewServer returns a new instance of the fake server, in standalone mode. Use
// the method URL to get the URL of the server.
//
// Hook is a function that will be called on every request.
func NewServer(hook func(*http.Request)) (*DockerServer, error) {
	listener, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		return nil, err
	}
	server := DockerServer{listener: listener, imgIDs: make(map[string]string), hook: hook}
	server.buildMuxer()
	go http.Serve(listener, &server)
	return &server, nil
}

func (s *DockerServer) buildMuxer() {
	s.mux = mux.NewRouter()
	s.mux.Path("/v{version:[0-9.]+}/commit").Methods("POST").HandlerFunc(s.commitContainer)
	s.mux.Path("/v{version:[0-9.]+}/containers/json").Methods("GET").HandlerFunc(s.listContainers)
	s.mux.Path("/v{version:[0-9.]+}/containers/create").Methods("POST").HandlerFunc(s.createContainer)
	s.mux.Path("/v{version:[0-9.]+}/containers/{id:.*}/json").Methods("GET").HandlerFunc(s.inspectContainer)
	s.mux.Path("/v{version:[0-9.]+}/containers/{id:.*}/start").Methods("POST").HandlerFunc(s.startContainer)
	s.mux.Path("/v{version:[0-9.]+}/containers/{id:.*}/stop").Methods("POST").HandlerFunc(s.stopContainer)
	s.mux.Path("/v{version:[0-9.]+}/containers/{id:.*}/wait").Methods("POST").HandlerFunc(s.waitContainer)
	s.mux.Path("/v{version:[0-9.]+}/containers/{id:.*}/attach").Methods("POST").HandlerFunc(s.attachContainer)
	s.mux.Path("/v{version:[0-9.]+}/containers/{id:.*}").Methods("DELETE").HandlerFunc(s.removeContainer)
	s.mux.Path("/v{version:[0-9.]+}/images/create").Methods("POST").HandlerFunc(s.pullImage)
	s.mux.Path("/v{version:[0-9.]+}/images/json").Methods("GET").HandlerFunc(s.listImages)
	s.mux.Path("/v{version:[0-9.]+}/images/{id:.*}").Methods("DELETE").HandlerFunc(s.removeImage)
	s.mux.Path("/v{version:[0-9.]+}/images/{name:.*}/push").Methods("POST").HandlerFunc(s.pushImage)
}

// Stop stops the server.
func (s *DockerServer) Stop() {
	if s.listener != nil {
		s.listener.Close()
	}
}

// URL returns the HTTP URL of the server.
func (s *DockerServer) URL() string {
	if s.listener == nil {
		return ""
	}
	return "http://" + s.listener.Addr().String() + "/"
}

// ServeHTTP handles HTTP requests sent to the server.
func (s *DockerServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	s.mux.ServeHTTP(w, r)
	if s.hook != nil {
		s.hook(r)
	}
}

func (s *DockerServer) listContainers(w http.ResponseWriter, r *http.Request) {
	s.cMut.RLock()
	result := make([]docker.APIContainers, len(s.containers))
	for i, container := range s.containers {
		result[i] = docker.APIContainers{
			ID:      container.ID,
			Image:   container.Image,
			Command: fmt.Sprintf("%s %s", container.Path, strings.Join(container.Args, " ")),
			Created: container.Created.Unix(),
			Status:  container.State.String(),
			Ports:   container.NetworkSettings.PortMappingHuman(),
		}
	}
	s.cMut.RUnlock()
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(http.StatusOK)
	json.NewEncoder(w).Encode(result)
}

func (s *DockerServer) listImages(w http.ResponseWriter, r *http.Request) {
	s.cMut.RLock()
	result := make([]docker.APIImages, len(s.images))
	for i, image := range s.images {
		result[i] = docker.APIImages{
			ID:      image.ID,
			Created: image.Created.Unix(),
		}
	}
	s.cMut.RUnlock()
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(http.StatusOK)
	json.NewEncoder(w).Encode(result)
}

func (s *DockerServer) findImage(id string) (string, error) {
	s.iMut.RLock()
	defer s.iMut.RUnlock()
	image, ok := s.imgIDs[id]
	if ok {
		return image, nil
	}
	image, _, err := s.findImageByID(id)
	return image, err
}

func (s *DockerServer) findImageByID(id string) (string, int, error) {
	s.iMut.RLock()
	defer s.iMut.RUnlock()
	for i, image := range s.images {
		if image.ID == id {
			return image.ID, i, nil
		}
	}
	return "", -1, errors.New("No such image")
}

func (s *DockerServer) createContainer(w http.ResponseWriter, r *http.Request) {
	var config docker.Config
	defer r.Body.Close()
	err := json.NewDecoder(r.Body).Decode(&config)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	image, err := s.findImage(config.Image)
	if err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}
	w.WriteHeader(http.StatusCreated)
	portMapping := make(map[string]string, len(config.PortSpecs))
	for _, p := range config.PortSpecs {
		portMapping[p] = strconv.Itoa(mathrand.Int() % 65536)
	}
	container := docker.Container{
		ID:      s.generateID(),
		Created: time.Now(),
		Path:    config.Cmd[0],
		Args:    config.Cmd[1:],
		Config:  &config,
		State: docker.State{
			Running:   false,
			Pid:       mathrand.Int() % 50000,
			ExitCode:  0,
			StartedAt: time.Now(),
		},
		Image: image,
		NetworkSettings: &docker.NetworkSettings{
			IPAddress:   fmt.Sprintf("172.16.42.%d", mathrand.Int()%250+2),
			IPPrefixLen: 24,
			Gateway:     "172.16.42.1",
			Bridge:      "docker0",
			PortMapping: portMapping,
		},
	}
	s.cMut.Lock()
	s.containers = append(s.containers, &container)
	s.cMut.Unlock()
	var c = struct{ ID string }{ID: container.ID}
	json.NewEncoder(w).Encode(c)
}

func (s *DockerServer) generateID() string {
	var buf [16]byte
	rand.Read(buf[:])
	return fmt.Sprintf("%x", buf)
}

func (s *DockerServer) inspectContainer(w http.ResponseWriter, r *http.Request) {
	id := mux.Vars(r)["id"]
	container, _, err := s.findContainer(id)
	if err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(http.StatusOK)
	json.NewEncoder(w).Encode(container)
}

func (s *DockerServer) startContainer(w http.ResponseWriter, r *http.Request) {
	id := mux.Vars(r)["id"]
	container, _, err := s.findContainer(id)
	if err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}
	s.cMut.Lock()
	defer s.cMut.Unlock()
	if container.State.Running {
		http.Error(w, "Container already running", http.StatusBadRequest)
		return
	}
	container.State.Running = true
}

func (s *DockerServer) stopContainer(w http.ResponseWriter, r *http.Request) {
	id := mux.Vars(r)["id"]
	container, _, err := s.findContainer(id)
	if err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}
	s.cMut.Lock()
	defer s.cMut.Unlock()
	if !container.State.Running {
		http.Error(w, "Container not running", http.StatusBadRequest)
		return
	}
	w.WriteHeader(http.StatusNoContent)
	container.State.Running = false
}

func (s *DockerServer) attachContainer(w http.ResponseWriter, r *http.Request) {
	id := mux.Vars(r)["id"]
	container, _, err := s.findContainer(id)
	if err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}
	if container.State.Running {
		fmt.Fprintf(w, "Container %q is running\n", container.ID)
	} else {
		fmt.Fprintf(w, "Container %q is not running\n", container.ID)
	}
	fmt.Fprintln(w, "What happened?")
	fmt.Fprintln(w, "Something happened")
}

func (s *DockerServer) waitContainer(w http.ResponseWriter, r *http.Request) {
	id := mux.Vars(r)["id"]
	container, _, err := s.findContainer(id)
	if err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}
	for {
		s.cMut.RLock()
		if container.State.Running {
			s.cMut.RUnlock()
			break
		}
		s.cMut.RUnlock()
	}
	w.Write([]byte(`{"StatusCode":0}`))
}

func (s *DockerServer) removeContainer(w http.ResponseWriter, r *http.Request) {
	id := mux.Vars(r)["id"]
	_, index, err := s.findContainer(id)
	if err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}
	w.WriteHeader(http.StatusNoContent)
	s.cMut.Lock()
	defer s.cMut.Unlock()
	s.containers[index] = s.containers[len(s.containers)-1]
	s.containers = s.containers[:len(s.containers)-1]
}

func (s *DockerServer) commitContainer(w http.ResponseWriter, r *http.Request) {
	id := r.URL.Query().Get("container")
	container, _, err := s.findContainer(id)
	if err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}
	var config *docker.Config
	runConfig := r.URL.Query().Get("run")
	if runConfig != "" {
		config = new(docker.Config)
		err = json.Unmarshal([]byte(runConfig), config)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
	}
	w.WriteHeader(http.StatusOK)
	image := docker.Image{
		ID:        "img-" + container.ID,
		Parent:    container.Image,
		Container: container.ID,
		Comment:   r.URL.Query().Get("m"),
		Author:    r.URL.Query().Get("author"),
		Config:    config,
	}
	repository := r.URL.Query().Get("repo")
	s.iMut.Lock()
	s.images = append(s.images, image)
	if repository != "" {
		s.imgIDs[repository] = image.ID
	}
	s.iMut.Unlock()
	fmt.Fprintf(w, `{"ID":%q}`, image.ID)
}

func (s *DockerServer) findContainer(id string) (*docker.Container, int, error) {
	s.cMut.RLock()
	defer s.cMut.RUnlock()
	for i, container := range s.containers {
		if container.ID == id {
			return container, i, nil
		}
	}
	return nil, -1, errors.New("No such container")
}

func (s *DockerServer) pullImage(w http.ResponseWriter, r *http.Request) {
	repository := r.URL.Query().Get("fromImage")
	image := docker.Image{
		ID: s.generateID(),
	}
	s.iMut.Lock()
	s.images = append(s.images, image)
	if repository != "" {
		s.imgIDs[repository] = image.ID
	}
	s.iMut.Unlock()
}

func (s *DockerServer) pushImage(w http.ResponseWriter, r *http.Request) {
	name := mux.Vars(r)["name"]
	s.iMut.RLock()
	if _, ok := s.imgIDs[name]; !ok {
		s.iMut.RUnlock()
		http.Error(w, "No such image", http.StatusNotFound)
		return
	}
	s.iMut.RUnlock()
	fmt.Fprintln(w, "Pushing...")
	fmt.Fprintln(w, "Pushed")
}

func (s *DockerServer) removeImage(w http.ResponseWriter, r *http.Request) {
	id := mux.Vars(r)["id"]
	_, index, err := s.findImageByID(id)
	if err != nil {
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}
	w.WriteHeader(http.StatusNoContent)
	s.iMut.Lock()
	defer s.iMut.Unlock()
	s.images[index] = s.images[len(s.images)-1]
	s.images = s.images[:len(s.images)-1]
}
<|endoftext|>


<commit_before>//   Copyright 2017 MSolution.IO
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package s3

import (
	"context"
	"database/sql"
	"strings"
	"sync"
	"time"

	"github.com/satori/go.uuid"
	"github.com/trackit/jsonlog"
	"gopkg.in/olivere/elastic.v5"

	"github.com/trackit/trackit-server/aws"
	"github.com/trackit/trackit-server/es"
)

const (
	kibibyte = 1 << 10
	mebibyte = 1 << 20
	gibibyte = 1 << 30

	esBulkInsertSize    = 8 * mebibyte
	esBulkInsertWorkers = 4

	opTypeIndex  = "index"
	opTypeCreate = "create"

	tagPrefix = `resourceTags/user:`
)

// ReportUpdateConclusion represents the results of a bill ingestion job.
type ReportUpdateConclusion struct {
	BillRepository       BillRepository
	LastImportedManifest time.Time
	Error                error
}

// reportUpdateConclusionChanToSlice accepts a <-chan ReportUpdateConclusion
// and a count, and builds a []ReportUpdateConclusion from the values read on
// the channel, stopping at 'count' values.
func reportUpdateConclusionChanToSlice(rucc <-chan ReportUpdateConclusion, count int) (rucs []ReportUpdateConclusion) {
	rucs = make([]ReportUpdateConclusion, count)
	for i := range rucs {
		if r, ok := <-rucc; ok {
			rucs[i] = r
		} else {
			rucs = rucs[:i]
			return
		}
	}
	return
}

// UpdateDueReports finds all BillRepositories in need of an update and updates
// them.
func UpdateDueReports(ctx context.Context, tx *sql.Tx) ([]ReportUpdateConclusion, error) {
	var wg sync.WaitGroup
	aas := make(map[int]aws.AwsAccount)
	brs, err := GetAwsBillRepositoriesWithDueUpdate(tx)
	if err != nil {
		return nil, err
	}
	wg.Add(len(brs))
	conclusionChan := make(chan ReportUpdateConclusion, len(brs))
	defer close(conclusionChan)
	for _, br := range brs {
		var aa aws.AwsAccount
		var ok bool
		var err error
		if aa, ok = aas[br.AwsAccountId]; !ok {
			aa, err = aws.GetAwsAccountWithId(br.AwsAccountId, tx)
			if err != nil {
				return nil, err
			}
			aas[br.AwsAccountId] = aa
		}
		go func(ctx context.Context, aa aws.AwsAccount, br BillRepository) {
			lim, err := UpdateReport(ctx, aa, br)
			conclusionChan <- ReportUpdateConclusion{
				BillRepository:       br,
				LastImportedManifest: lim,
				Error:                err,
			}
			wg.Done()
		}(ctx, aa, br)
	}
	wg.Wait()
	return reportUpdateConclusionChanToSlice(conclusionChan, len(brs)), nil
}

// contextKey is a key in a context, to prevent collision with other modules.
type contextKey uint

// ingestionContextKey is used to store an 'ingestionId' in a context.
const ingestionContextKey = contextKey(iota)

// contextWithIngestionId returns a context configured so that its logger logs
// an 'ingestionId'.
func contextWithIngestionId(ctx context.Context) context.Context {
	ingestionId := uuid.NewV1().String()
	ctx = context.WithValue(ctx, ingestionContextKey, ingestionId)
	logger := jsonlog.LoggerFromContextOrDefault(ctx)
	logger = logger.WithContextKey(ingestionContextKey, "ingestionId")
	logger = logger.WithContext(ctx)
	return jsonlog.ContextWithLogger(ctx, logger)
}

// UpdateReport updates the elasticsearch database with new data from usage and
// cost reports.
func UpdateReport(ctx context.Context, aa aws.AwsAccount, br BillRepository) (latestManifest time.Time, err error) {
	ctx = contextWithIngestionId(ctx)
	logger := jsonlog.LoggerFromContextOrDefault(ctx)
	logger.Info("Updating reports for AWS account.", map[string]interface{}{
		"awsAccount":     aa,
		"billRepository": br,
	})
	if bp, err := getBulkProcessor(ctx); err != nil {
		logger.Error("Failed to get bulk processor.", err.Error())
		return latestManifest, err
	} else {
		index := es.IndexNameForUserId(aa.UserId, IndexPrefixLineItem)
		latestManifest, err = ReadBills(
			ctx,
			aa,
			br,
			ingestLineItems(ctx, bp, index, br),
			manifestsModifiedAfter(br.LastImportedManifest),
		)
		logger.Info("Done ingesting data.", nil)
		return latestManifest, err
	}
}

// getBulkProcessor builds a bulk processor for ElasticSearch.
func getBulkProcessor(ctx context.Context) (*elastic.BulkProcessor, error) {
	bps := elastic.NewBulkProcessorService(es.Client)
	bps = bps.BulkActions(-1)
	bps = bps.BulkSize(esBulkInsertSize)
	bps = bps.Workers(esBulkInsertWorkers)
	bps = bps.Before(beforeBulk(ctx))
	bps = bps.After(afterBulk(ctx))
	return bps.Do(context.Background()) // use of background context is not an error
}

// ingestLineItems returns an OnLineItem handler which ingests LineItems in an
// ElasticSearch index.
func ingestLineItems(ctx context.Context, bp *elastic.BulkProcessor, index string, br BillRepository) OnLineItem {
	return func(li LineItem, ok bool) {
		if ok {
			li.BillRepositoryId = br.Id
			li = extractTags(li)
			rq := elastic.NewBulkIndexRequest()
			rq = rq.Index(index)
			rq = rq.OpType(opTypeCreate)
			rq = rq.Type(TypeLineItem)
			rq = rq.Id(li.EsId())
			rq = rq.Doc(li)
			bp.Add(rq)
		} else {
			bp.Flush()
			bp.Close()
		}
	}
}

// manifestsStartingAfter returns a manifest predicate which is true for all
// manifests starting after a given date.
func manifestsModifiedAfter(t time.Time) ManifestPredicate {
	return func(m manifest, oneMonthBefore bool) bool {
		if (oneMonthBefore) {
			if time.Time(m.LastModified).AddDate(0, 1, 0).After(t) {
				return true
			} else {
				return false
			}
		} else {
			if time.Time(m.LastModified).After(t) {
				return true
			} else {
				return false
			}
		}
	}
}

// extractTags extracts tags from a LineItem's Any field. It retrieves user
// tags only and stores them in the Tags map with a clean key.
func extractTags(li LineItem) LineItem {
	var tags []LineItemTags
	for k, v := range li.Any {
		if strings.HasPrefix(k, tagPrefix) {
			tags = append(tags, LineItemTags{strings.TrimPrefix(k, tagPrefix), v})
		}
	}
	li.Tags = tags
	li.Any = nil
	return li
}

func beforeBulk(ctx context.Context) func(int64, []elastic.BulkableRequest) {
	logger := jsonlog.LoggerFromContextOrDefault(ctx)
	return func(execId int64, reqs []elastic.BulkableRequest) {
		logger.Info("Performing bulk ElasticSearch requests.", map[string]interface{}{
			"executionId":   execId,
			"requestsCount": len(reqs),
		})
	}
}

func afterBulk(ctx context.Context) func(int64, []elastic.BulkableRequest, *elastic.BulkResponse, error) {
	logger := jsonlog.LoggerFromContextOrDefault(ctx)
	return func(execId int64, reqs []elastic.BulkableRequest, resp *elastic.BulkResponse, err error) {
		if err != nil {
			logger.Error("Failed bulk ElasticSearch requests.", map[string]interface{}{
				"executionId": execId,
				"error":       err.Error(),
			})
		} else {
			logger.Info("Finished bulk ElasticSearch requests.", map[string]interface{}{
				"executionId": execId,
				"took":        resp.Took,
			})
		}

	}
}
<commit_msg>put "taxes" in region and availabilityzone if the item is a tax<commit_after>//   Copyright 2017 MSolution.IO
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package s3

import (
	"context"
	"database/sql"
	"strings"
	"sync"
	"time"

	"github.com/satori/go.uuid"
	"github.com/trackit/jsonlog"
	"gopkg.in/olivere/elastic.v5"

	"github.com/trackit/trackit-server/aws"
	"github.com/trackit/trackit-server/es"
)

const (
	kibibyte = 1 << 10
	mebibyte = 1 << 20
	gibibyte = 1 << 30

	esBulkInsertSize    = 8 * mebibyte
	esBulkInsertWorkers = 4

	opTypeIndex  = "index"
	opTypeCreate = "create"

	tagPrefix = `resourceTags/user:`
)

// ReportUpdateConclusion represents the results of a bill ingestion job.
type ReportUpdateConclusion struct {
	BillRepository       BillRepository
	LastImportedManifest time.Time
	Error                error
}

// reportUpdateConclusionChanToSlice accepts a <-chan ReportUpdateConclusion
// and a count, and builds a []ReportUpdateConclusion from the values read on
// the channel, stopping at 'count' values.
func reportUpdateConclusionChanToSlice(rucc <-chan ReportUpdateConclusion, count int) (rucs []ReportUpdateConclusion) {
	rucs = make([]ReportUpdateConclusion, count)
	for i := range rucs {
		if r, ok := <-rucc; ok {
			rucs[i] = r
		} else {
			rucs = rucs[:i]
			return
		}
	}
	return
}

// UpdateDueReports finds all BillRepositories in need of an update and updates
// them.
func UpdateDueReports(ctx context.Context, tx *sql.Tx) ([]ReportUpdateConclusion, error) {
	var wg sync.WaitGroup
	aas := make(map[int]aws.AwsAccount)
	brs, err := GetAwsBillRepositoriesWithDueUpdate(tx)
	if err != nil {
		return nil, err
	}
	wg.Add(len(brs))
	conclusionChan := make(chan ReportUpdateConclusion, len(brs))
	defer close(conclusionChan)
	for _, br := range brs {
		var aa aws.AwsAccount
		var ok bool
		var err error
		if aa, ok = aas[br.AwsAccountId]; !ok {
			aa, err = aws.GetAwsAccountWithId(br.AwsAccountId, tx)
			if err != nil {
				return nil, err
			}
			aas[br.AwsAccountId] = aa
		}
		go func(ctx context.Context, aa aws.AwsAccount, br BillRepository) {
			lim, err := UpdateReport(ctx, aa, br)
			conclusionChan <- ReportUpdateConclusion{
				BillRepository:       br,
				LastImportedManifest: lim,
				Error:                err,
			}
			wg.Done()
		}(ctx, aa, br)
	}
	wg.Wait()
	return reportUpdateConclusionChanToSlice(conclusionChan, len(brs)), nil
}

// contextKey is a key in a context, to prevent collision with other modules.
type contextKey uint

// ingestionContextKey is used to store an 'ingestionId' in a context.
const ingestionContextKey = contextKey(iota)

// contextWithIngestionId returns a context configured so that its logger logs
// an 'ingestionId'.
func contextWithIngestionId(ctx context.Context) context.Context {
	ingestionId := uuid.NewV1().String()
	ctx = context.WithValue(ctx, ingestionContextKey, ingestionId)
	logger := jsonlog.LoggerFromContextOrDefault(ctx)
	logger = logger.WithContextKey(ingestionContextKey, "ingestionId")
	logger = logger.WithContext(ctx)
	return jsonlog.ContextWithLogger(ctx, logger)
}

// UpdateReport updates the elasticsearch database with new data from usage and
// cost reports.
func UpdateReport(ctx context.Context, aa aws.AwsAccount, br BillRepository) (latestManifest time.Time, err error) {
	ctx = contextWithIngestionId(ctx)
	logger := jsonlog.LoggerFromContextOrDefault(ctx)
	logger.Info("Updating reports for AWS account.", map[string]interface{}{
		"awsAccount":     aa,
		"billRepository": br,
	})
	if bp, err := getBulkProcessor(ctx); err != nil {
		logger.Error("Failed to get bulk processor.", err.Error())
		return latestManifest, err
	} else {
		index := es.IndexNameForUserId(aa.UserId, IndexPrefixLineItem)
		latestManifest, err = ReadBills(
			ctx,
			aa,
			br,
			ingestLineItems(ctx, bp, index, br),
			manifestsModifiedAfter(br.LastImportedManifest),
		)
		logger.Info("Done ingesting data.", nil)
		return latestManifest, err
	}
}

// getBulkProcessor builds a bulk processor for ElasticSearch.
func getBulkProcessor(ctx context.Context) (*elastic.BulkProcessor, error) {
	bps := elastic.NewBulkProcessorService(es.Client)
	bps = bps.BulkActions(-1)
	bps = bps.BulkSize(esBulkInsertSize)
	bps = bps.Workers(esBulkInsertWorkers)
	bps = bps.Before(beforeBulk(ctx))
	bps = bps.After(afterBulk(ctx))
	return bps.Do(context.Background()) // use of background context is not an error
}

// ingestLineItems returns an OnLineItem handler which ingests LineItems in an
// ElasticSearch index.
func ingestLineItems(ctx context.Context, bp *elastic.BulkProcessor, index string, br BillRepository) OnLineItem {
	return func(li LineItem, ok bool) {
		if ok {
			if li.LineItemType == "Tax" {
				li.AvailabilityZone = "taxes"
				li.Region = "taxes"
			}
			li.BillRepositoryId = br.Id
			li = extractTags(li)
			rq := elastic.NewBulkIndexRequest()
			rq = rq.Index(index)
			rq = rq.OpType(opTypeCreate)
			rq = rq.Type(TypeLineItem)
			rq = rq.Id(li.EsId())
			rq = rq.Doc(li)
			bp.Add(rq)
		} else {
			bp.Flush()
			bp.Close()
		}
	}
}

// manifestsStartingAfter returns a manifest predicate which is true for all
// manifests starting after a given date.
func manifestsModifiedAfter(t time.Time) ManifestPredicate {
	return func(m manifest, oneMonthBefore bool) bool {
		if oneMonthBefore {
			if time.Time(m.LastModified).AddDate(0, 1, 0).After(t) {
				return true
			} else {
				return false
			}
		} else {
			if time.Time(m.LastModified).After(t) {
				return true
			} else {
				return false
			}
		}
	}
}

// extractTags extracts tags from a LineItem's Any field. It retrieves user
// tags only and stores them in the Tags map with a clean key.
func extractTags(li LineItem) LineItem {
	var tags []LineItemTags
	for k, v := range li.Any {
		if strings.HasPrefix(k, tagPrefix) {
			tags = append(tags, LineItemTags{strings.TrimPrefix(k, tagPrefix), v})
		}
	}
	li.Tags = tags
	li.Any = nil
	return li
}

func beforeBulk(ctx context.Context) func(int64, []elastic.BulkableRequest) {
	logger := jsonlog.LoggerFromContextOrDefault(ctx)
	return func(execId int64, reqs []elastic.BulkableRequest) {
		logger.Info("Performing bulk ElasticSearch requests.", map[string]interface{}{
			"executionId":   execId,
			"requestsCount": len(reqs),
		})
	}
}

func afterBulk(ctx context.Context) func(int64, []elastic.BulkableRequest, *elastic.BulkResponse, error) {
	logger := jsonlog.LoggerFromContextOrDefault(ctx)
	return func(execId int64, reqs []elastic.BulkableRequest, resp *elastic.BulkResponse, err error) {
		if err != nil {
			logger.Error("Failed bulk ElasticSearch requests.", map[string]interface{}{
				"executionId": execId,
				"error":       err.Error(),
			})
		} else {
			logger.Info("Finished bulk ElasticSearch requests.", map[string]interface{}{
				"executionId": execId,
				"took":        resp.Took,
			})
		}

	}
}
<|endoftext|>


<commit_before>package termite

import (
	"exec"
	"fmt"
	"http"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/user"
	"rpc"
	"sync"
	"strings"
	"runtime"
	"time"
)

var _ = log.Println

type WorkerDaemon struct {
	Nobody *user.User
	secret []byte

	listener      net.Listener
	rpcServer     *rpc.Server
	contentCache  *ContentCache
	contentServer *ContentServer
	maxJobCount   int
	pending       *PendingConnections
	cacheDir      string
	tmpDir        string
	stats         *cpuStatSampler

	stopListener chan int

	mirrorMapMutex sync.Mutex
	cond           *sync.Cond
	mirrorMap      map[string]*Mirror
	shuttingDown   bool
	mustRestart    bool
	options        *WorkerOptions
}

type WorkerOptions struct {
	Secret   []byte
	TempDir  string
	CacheDir string
	Jobs     int

	// If set, change user to this for running.
	User             *string
	FileContentCount int

	// How often to reap filesystems. If 1, use 1 FS per task.
	ReapCount int

	// Delay between contacting the coordinator for making reports.
	ReportInterval float64
}

func (me *WorkerDaemon) getMirror(rpcConn, revConn net.Conn, reserveCount int) (*Mirror, os.Error) {
	if reserveCount <= 0 {
		return nil, os.NewError("must ask positive jobcount")
	}
	me.mirrorMapMutex.Lock()
	defer me.mirrorMapMutex.Unlock()
	used := 0
	for _, v := range me.mirrorMap {
		used += v.maxJobCount
	}

	remaining := me.maxJobCount - used
	if remaining <= 0 {
		return nil, os.NewError("no processes available")
	}
	if remaining < reserveCount {
		reserveCount = remaining
	}

	mirror := NewMirror(me, rpcConn, revConn)
	mirror.maxJobCount = reserveCount
	key := fmt.Sprintf("%v", rpcConn.RemoteAddr())
	me.mirrorMap[key] = mirror
	mirror.key = key
	return mirror, nil
}

func NewWorkerDaemon(options *WorkerOptions) *WorkerDaemon {
	if options.FileContentCount == 0 {
		options.FileContentCount = 1024
	}
	if options.ReapCount == 0 {
		options.ReapCount = 4
	}
	if options.ReportInterval == 0 {
		options.ReportInterval = 60.0
	}
	copied := *options

	cache := NewContentCache(options.CacheDir)
	cache.SetMemoryCacheSize(options.FileContentCount)
	me := &WorkerDaemon{
		secret:        options.Secret,
		contentCache:  cache,
		mirrorMap:     make(map[string]*Mirror),
		contentServer: &ContentServer{Cache: cache},
		pending:       NewPendingConnections(),
		maxJobCount:   options.Jobs,
		tmpDir:        options.TempDir,
		rpcServer:     rpc.NewServer(),
		stats:         newCpuStatSampler(),
		options:       &copied,
	}
	if os.Geteuid() == 0 && options.User != nil {
		nobody, err := user.Lookup(*options.User)
		if err != nil {
			log.Fatalf("can't lookup %q: %v", options.User, err)
		}
		me.Nobody = nobody
	}
	me.cond = sync.NewCond(&me.mirrorMapMutex)
	me.stopListener = make(chan int, 1)
	me.rpcServer.Register(me)
	return me
}

func (me *WorkerDaemon) PeriodicReport(coordinator string, port int) {
	if coordinator == "" {
		log.Println("No coordinator - not doing period reports.")
		return
	}
	me.report(coordinator, port)
	for {
		c := time.After(int64(me.options.ReportInterval * 1e9))
		<-c
		me.report(coordinator, port)
	}
}

func (me *WorkerDaemon) report(coordinator string, port int) {
	client, err := rpc.DialHTTP("tcp", coordinator)
	if err != nil {
		log.Println("dialing coordinator:", err)
		return
	}

	hostname, err := os.Hostname()
	if err != nil {
		log.Println("hostname", err)
		return
	}

	cname, err := net.LookupCNAME(hostname)
	if err != nil {
		log.Println("cname", err)
		return
	}
	cname = strings.TrimRight(cname, ".")
	req := Registration{
		Address: fmt.Sprintf("%v:%d", cname, port),
		Name:    fmt.Sprintf("%s:%d", hostname, port),
		Version: Version(),
	}

	rep := 0
	err = client.Call("Coordinator.Register", &req, &rep)
	if err != nil {
		log.Println("coordinator rpc error:", err)
	}
}

// TODO - should expose under ContentServer name?
func (me *WorkerDaemon) FileContent(req *ContentRequest, rep *ContentResponse) os.Error {
	return me.contentServer.FileContent(req, rep)
}

func (me *WorkerDaemon) CreateMirror(req *CreateMirrorRequest, rep *CreateMirrorResponse) os.Error {
	if me.shuttingDown {
		return os.NewError("Worker is shutting down.")
	}

	rpcConn := me.pending.WaitConnection(req.RpcId)
	revConn := me.pending.WaitConnection(req.RevRpcId)
	mirror, err := me.getMirror(rpcConn, revConn, req.MaxJobCount)
	if err != nil {
		rpcConn.Close()
		revConn.Close()
		return err
	}
	mirror.writableRoot = req.WritableRoot

	rep.GrantedJobCount = mirror.maxJobCount
	return nil
}

func (me *WorkerDaemon) DropMirror(mirror *Mirror) {
	me.mirrorMapMutex.Lock()
	defer me.mirrorMapMutex.Unlock()

	log.Println("dropping mirror", mirror.key)
	me.mirrorMap[mirror.key] = nil, false
	me.cond.Broadcast()
	runtime.GC()
}

func (me *WorkerDaemon) serveConn(conn net.Conn) {
	log.Println("Authenticated connection from", conn.RemoteAddr())
	if !me.pending.Accept(conn) {
		go me.rpcServer.ServeConn(conn)
	}
}

func (me *WorkerDaemon) RunWorkerServer(port int, coordinator string) {
	me.listener = AuthenticatedListener(port, me.secret)
	_, portString, _ := net.SplitHostPort(me.listener.Addr().String())

	fmt.Sscanf(portString, "%d", &port)
	go me.PeriodicReport(coordinator, port)

	for {
		conn, err := me.listener.Accept()
		if err != nil {
			log.Println("me.listener", err)
			break
		}
		log.Println("Authenticated connection from", conn.RemoteAddr())
		if !me.pending.Accept(conn) {
			go me.rpcServer.ServeConn(conn)
		}
	}

	if me.mustRestart {
		me.restart(coordinator)
	}
}

func (me *WorkerDaemon) Shutdown(req *ShutdownRequest, rep *ShutdownResponse) os.Error {
	log.Println("Received Shutdown.")
	if req.Restart {
		me.mustRestart = true
	}
	me.mirrorMapMutex.Lock()
	defer me.mirrorMapMutex.Unlock()

	me.shuttingDown = true
	for _, m := range me.mirrorMap {
		m.Shutdown()
	}
	log.Println("Asked all mirrors to shut down.")
	for len(me.mirrorMap) > 0 {
		log.Println("Live mirror count:", len(me.mirrorMap))
		me.cond.Wait()
	}
	log.Println("All mirrors have shut down.")
	me.listener.Close()
	return nil
}

func (me *WorkerDaemon) restart(coord string) {
	cl := http.Client{}
	req, err := cl.Get(fmt.Sprintf("http://%s/bin/worker", coord))
	if err != nil {
		log.Fatal("http get error.")
	}

	// We download into a tempdir, so we maintain the binary name.
	dir, err := ioutil.TempDir("", "worker-download")
	if err != nil {
		log.Fatal("TempDir:", err)
	}

	f, err := os.Create(dir + "/worker")
	if err != nil {
		log.Fatal("os.Create:", err)
	}
	io.Copy(f, req.Body)
	f.Close()
	os.Chmod(f.Name(), 0755)
	log.Println("Starting downloaded worker.")
	cmd := exec.Command(f.Name(), os.Args[1:]...)
	cmd.Start()
}
<commit_msg>Exit periodic report goroutine if worker has shutdown.<commit_after>package termite

import (
	"exec"
	"fmt"
	"http"
	"io"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/user"
	"rpc"
	"sync"
	"strings"
	"runtime"
	"time"
)

var _ = log.Println

type WorkerDaemon struct {
	Nobody *user.User
	secret []byte

	listener      net.Listener
	rpcServer     *rpc.Server
	contentCache  *ContentCache
	contentServer *ContentServer
	maxJobCount   int
	pending       *PendingConnections
	cacheDir      string
	tmpDir        string
	stats         *cpuStatSampler

	stopListener chan int

	mirrorMapMutex sync.Mutex
	cond           *sync.Cond
	mirrorMap      map[string]*Mirror
	shuttingDown   bool
	mustRestart    bool
	options        *WorkerOptions
}

type WorkerOptions struct {
	Secret   []byte
	TempDir  string
	CacheDir string
	Jobs     int

	// If set, change user to this for running.
	User             *string
	FileContentCount int

	// How often to reap filesystems. If 1, use 1 FS per task.
	ReapCount int

	// Delay between contacting the coordinator for making reports.
	ReportInterval float64
}

func (me *WorkerDaemon) getMirror(rpcConn, revConn net.Conn, reserveCount int) (*Mirror, os.Error) {
	if reserveCount <= 0 {
		return nil, os.NewError("must ask positive jobcount")
	}
	me.mirrorMapMutex.Lock()
	defer me.mirrorMapMutex.Unlock()
	used := 0
	for _, v := range me.mirrorMap {
		used += v.maxJobCount
	}

	remaining := me.maxJobCount - used
	if remaining <= 0 {
		return nil, os.NewError("no processes available")
	}
	if remaining < reserveCount {
		reserveCount = remaining
	}

	mirror := NewMirror(me, rpcConn, revConn)
	mirror.maxJobCount = reserveCount
	key := fmt.Sprintf("%v", rpcConn.RemoteAddr())
	me.mirrorMap[key] = mirror
	mirror.key = key
	return mirror, nil
}

func NewWorkerDaemon(options *WorkerOptions) *WorkerDaemon {
	if options.FileContentCount == 0 {
		options.FileContentCount = 1024
	}
	if options.ReapCount == 0 {
		options.ReapCount = 4
	}
	if options.ReportInterval == 0 {
		options.ReportInterval = 60.0
	}
	copied := *options

	cache := NewContentCache(options.CacheDir)
	cache.SetMemoryCacheSize(options.FileContentCount)
	me := &WorkerDaemon{
		secret:        options.Secret,
		contentCache:  cache,
		mirrorMap:     make(map[string]*Mirror),
		contentServer: &ContentServer{Cache: cache},
		pending:       NewPendingConnections(),
		maxJobCount:   options.Jobs,
		tmpDir:        options.TempDir,
		rpcServer:     rpc.NewServer(),
		stats:         newCpuStatSampler(),
		options:       &copied,
	}
	if os.Geteuid() == 0 && options.User != nil {
		nobody, err := user.Lookup(*options.User)
		if err != nil {
			log.Fatalf("can't lookup %q: %v", options.User, err)
		}
		me.Nobody = nobody
	}
	me.cond = sync.NewCond(&me.mirrorMapMutex)
	me.stopListener = make(chan int, 1)
	me.rpcServer.Register(me)
	return me
}

func (me *WorkerDaemon) PeriodicReport(coordinator string, port int) {
	if coordinator == "" {
		log.Println("No coordinator - not doing period reports.")
		return
	}
	for !me.shuttingDown {
		me.report(coordinator, port)
		c := time.After(int64(me.options.ReportInterval * 1e9))
		<-c
	}
}

func (me *WorkerDaemon) report(coordinator string, port int) {
	client, err := rpc.DialHTTP("tcp", coordinator)
	if err != nil {
		log.Println("dialing coordinator:", err)
		return
	}

	hostname, err := os.Hostname()
	if err != nil {
		log.Println("hostname", err)
		return
	}

	cname, err := net.LookupCNAME(hostname)
	if err != nil {
		log.Println("cname", err)
		return
	}
	cname = strings.TrimRight(cname, ".")
	req := Registration{
		Address: fmt.Sprintf("%v:%d", cname, port),
		Name:    fmt.Sprintf("%s:%d", hostname, port),
		Version: Version(),
	}

	rep := 0
	err = client.Call("Coordinator.Register", &req, &rep)
	if err != nil {
		log.Println("coordinator rpc error:", err)
	}
}

// TODO - should expose under ContentServer name?
func (me *WorkerDaemon) FileContent(req *ContentRequest, rep *ContentResponse) os.Error {
	return me.contentServer.FileContent(req, rep)
}

func (me *WorkerDaemon) CreateMirror(req *CreateMirrorRequest, rep *CreateMirrorResponse) os.Error {
	if me.shuttingDown {
		return os.NewError("Worker is shutting down.")
	}

	rpcConn := me.pending.WaitConnection(req.RpcId)
	revConn := me.pending.WaitConnection(req.RevRpcId)
	mirror, err := me.getMirror(rpcConn, revConn, req.MaxJobCount)
	if err != nil {
		rpcConn.Close()
		revConn.Close()
		return err
	}
	mirror.writableRoot = req.WritableRoot

	rep.GrantedJobCount = mirror.maxJobCount
	return nil
}

func (me *WorkerDaemon) DropMirror(mirror *Mirror) {
	me.mirrorMapMutex.Lock()
	defer me.mirrorMapMutex.Unlock()

	log.Println("dropping mirror", mirror.key)
	me.mirrorMap[mirror.key] = nil, false
	me.cond.Broadcast()
	runtime.GC()
}

func (me *WorkerDaemon) serveConn(conn net.Conn) {
	log.Println("Authenticated connection from", conn.RemoteAddr())
	if !me.pending.Accept(conn) {
		go me.rpcServer.ServeConn(conn)
	}
}

func (me *WorkerDaemon) RunWorkerServer(port int, coordinator string) {
	me.listener = AuthenticatedListener(port, me.secret)
	_, portString, _ := net.SplitHostPort(me.listener.Addr().String())

	fmt.Sscanf(portString, "%d", &port)
	go me.PeriodicReport(coordinator, port)

	for {
		conn, err := me.listener.Accept()
		if err != nil {
			log.Println("me.listener", err)
			break
		}
		log.Println("Authenticated connection from", conn.RemoteAddr())
		if !me.pending.Accept(conn) {
			go me.rpcServer.ServeConn(conn)
		}
	}

	if me.mustRestart {
		me.restart(coordinator)
	}
}

func (me *WorkerDaemon) Shutdown(req *ShutdownRequest, rep *ShutdownResponse) os.Error {
	log.Println("Received Shutdown.")
	if req.Restart {
		me.mustRestart = true
	}
	me.mirrorMapMutex.Lock()
	defer me.mirrorMapMutex.Unlock()

	me.shuttingDown = true
	for _, m := range me.mirrorMap {
		m.Shutdown()
	}
	log.Println("Asked all mirrors to shut down.")
	for len(me.mirrorMap) > 0 {
		log.Println("Live mirror count:", len(me.mirrorMap))
		me.cond.Wait()
	}
	log.Println("All mirrors have shut down.")
	me.listener.Close()
	return nil
}

func (me *WorkerDaemon) restart(coord string) {
	cl := http.Client{}
	req, err := cl.Get(fmt.Sprintf("http://%s/bin/worker", coord))
	if err != nil {
		log.Fatal("http get error.")
	}

	// We download into a tempdir, so we maintain the binary name.
	dir, err := ioutil.TempDir("", "worker-download")
	if err != nil {
		log.Fatal("TempDir:", err)
	}

	f, err := os.Create(dir + "/worker")
	if err != nil {
		log.Fatal("os.Create:", err)
	}
	io.Copy(f, req.Body)
	f.Close()
	os.Chmod(f.Name(), 0755)
	log.Println("Starting downloaded worker.")
	cmd := exec.Command(f.Name(), os.Args[1:]...)
	cmd.Start()
}
<|endoftext|>


<commit_before>// Copyright 2017 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package remote

import (
	"context"
	"sort"

	"github.com/prometheus/common/model"

	"github.com/prometheus/prometheus/pkg/labels"
	"github.com/prometheus/prometheus/prompb"
	"github.com/prometheus/prometheus/storage"
)

// Querier returns a new Querier on the storage.
func (r *Storage) Querier(mint, maxt int64) (storage.Querier, error) {
	r.mtx.Lock()
	defer r.mtx.Unlock()

	queriers := make([]storage.Querier, 0, len(r.clients))
	for _, c := range r.clients {
		queriers = append(queriers, &querier{
			mint:           mint,
			maxt:           maxt,
			client:         c,
			externalLabels: r.externalLabels,
		})
	}
	return storage.NewMergeQuerier(queriers), nil
}

// Querier is an adapter to make a Client usable as a storage.Querier.
type querier struct {
	mint, maxt     int64
	client         *Client
	externalLabels model.LabelSet
}

// Select returns a set of series that matches the given label matchers.
func (q *querier) Select(matchers ...*labels.Matcher) storage.SeriesSet {
	m, added := q.addExternalLabels(matchers)

	res, err := q.client.Read(context.TODO(), q.mint, q.maxt, labelMatchersToProto(m))
	if err != nil {
		return errSeriesSet{err: err}
	}

	series := make([]storage.Series, 0, len(res))
	for _, ts := range res {
		labels := labelPairsToLabels(ts.Labels)
		removeLabels(labels, added)
		series = append(series, &concreteSeries{
			labels:  labels,
			samples: ts.Samples,
		})
	}
	sort.Sort(byLabel(series))
	return &concreteSeriesSet{
		series: series,
	}
}

func labelMatchersToProto(matchers []*labels.Matcher) []*prompb.LabelMatcher {
	pbMatchers := make([]*prompb.LabelMatcher, 0, len(matchers))
	for _, m := range matchers {
		var mType prompb.LabelMatcher_Type
		switch m.Type {
		case labels.MatchEqual:
			mType = prompb.LabelMatcher_EQ
		case labels.MatchNotEqual:
			mType = prompb.LabelMatcher_NEQ
		case labels.MatchRegexp:
			mType = prompb.LabelMatcher_RE
		case labels.MatchNotRegexp:
			mType = prompb.LabelMatcher_NRE
		default:
			panic("invalid matcher type")
		}
		pbMatchers = append(pbMatchers, &prompb.LabelMatcher{
			Type:  mType,
			Name:  string(m.Name),
			Value: string(m.Value),
		})
	}
	return pbMatchers
}

func labelPairsToLabels(labelPairs []*prompb.Label) labels.Labels {
	result := make(labels.Labels, 0, len(labelPairs))
	for _, l := range labelPairs {
		result = append(result, labels.Label{
			Name:  l.Name,
			Value: l.Value,
		})
	}
	sort.Sort(result)
	return result
}

type byLabel []storage.Series

func (a byLabel) Len() int           { return len(a) }
func (a byLabel) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byLabel) Less(i, j int) bool { return labels.Compare(a[i].Labels(), a[j].Labels()) < 0 }

// LabelValues returns all potential values for a label name.
func (q *querier) LabelValues(name string) ([]string, error) {
	// TODO implement?
	return nil, nil
}

// Close releases the resources of the Querier.
func (q *querier) Close() error {
	return nil
}

// errSeriesSet implements storage.SeriesSet, just returning an error.
type errSeriesSet struct {
	err error
}

func (errSeriesSet) Next() bool {
	return false
}

func (errSeriesSet) At() storage.Series {
	return nil
}

func (e errSeriesSet) Err() error {
	return e.err
}

// concreteSeriesSet implements storage.SeriesSet.
type concreteSeriesSet struct {
	cur    int
	series []storage.Series
}

func (c *concreteSeriesSet) Next() bool {
	c.cur++
	return c.cur < len(c.series)
}

func (c *concreteSeriesSet) At() storage.Series {
	return c.series[c.cur]
}

func (c *concreteSeriesSet) Err() error {
	return nil
}

// concreteSeries implementes storage.Series.
type concreteSeries struct {
	labels  labels.Labels
	samples []*prompb.Sample
}

func (c *concreteSeries) Labels() labels.Labels {
	return c.labels
}

func (c *concreteSeries) Iterator() storage.SeriesIterator {
	return &concreteSeriesIterator{
		series: c,
	}
}

// concreteSeriesIterator implements storage.SeriesIterator.
type concreteSeriesIterator struct {
	cur    int
	series *concreteSeries
}

func (c *concreteSeriesIterator) Seek(t int64) bool {
	c.cur = sort.Search(len(c.series.samples), func(n int) bool {
		return c.series.samples[c.cur].Timestamp > t
	})
	return c.cur == 0
}

func (c *concreteSeriesIterator) At() (t int64, v float64) {
	s := c.series.samples[c.cur]
	return s.Timestamp, s.Value
}

func (c *concreteSeriesIterator) Next() bool {
	c.cur++
	return c.cur < len(c.series.samples)
}

func (c *concreteSeriesIterator) Err() error {
	return nil
}

// addExternalLabels adds matchers for each external label. External labels
// that already have a corresponding user-supplied matcher are skipped, as we
// assume that the user explicitly wants to select a different value for them.
// We return the new set of matchers, along with a map of labels for which
// matchers were added, so that these can later be removed from the result
// time series again.
func (q *querier) addExternalLabels(matchers []*labels.Matcher) ([]*labels.Matcher, model.LabelSet) {
	el := make(model.LabelSet, len(q.externalLabels))
	for k, v := range q.externalLabels {
		el[k] = v
	}
	for _, m := range matchers {
		if _, ok := el[model.LabelName(m.Name)]; ok {
			delete(el, model.LabelName(m.Name))
		}
	}
	for k, v := range el {
		m, err := labels.NewMatcher(labels.MatchEqual, string(k), string(v))
		if err != nil {
			panic(err)
		}
		matchers = append(matchers, m)
	}
	return matchers, el
}

func removeLabels(l labels.Labels, toDelete model.LabelSet) {
	for i := 0; i < len(l); {
		if _, ok := toDelete[model.LabelName(l[i].Name)]; ok {
			l = l[:i+copy(l[i:], l[i+1:])]
		} else {
			i++
		}
	}
}

//// MatrixToIterators returns series iterators for a given matrix.
//func MatrixToIterators(m model.Matrix, err error) ([]local.SeriesIterator, error) {
//	if err != nil {
//		return nil, err
//	}
//
//	its := make([]local.SeriesIterator, 0, len(m))
//	for _, ss := range m {
//		its = append(its, sampleStreamIterator{
//			ss: ss,
//		})
//	}
//	return its, nil
//}
<commit_msg>Make concreteSeriersIterator behave.<commit_after>// Copyright 2017 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package remote

import (
	"context"
	"sort"

	"github.com/prometheus/common/model"

	"github.com/prometheus/prometheus/pkg/labels"
	"github.com/prometheus/prometheus/prompb"
	"github.com/prometheus/prometheus/storage"
)

// Querier returns a new Querier on the storage.
func (r *Storage) Querier(mint, maxt int64) (storage.Querier, error) {
	r.mtx.Lock()
	defer r.mtx.Unlock()

	queriers := make([]storage.Querier, 0, len(r.clients))
	for _, c := range r.clients {
		queriers = append(queriers, &querier{
			mint:           mint,
			maxt:           maxt,
			client:         c,
			externalLabels: r.externalLabels,
		})
	}
	return storage.NewMergeQuerier(queriers), nil
}

// Querier is an adapter to make a Client usable as a storage.Querier.
type querier struct {
	mint, maxt     int64
	client         *Client
	externalLabels model.LabelSet
}

// Select returns a set of series that matches the given label matchers.
func (q *querier) Select(matchers ...*labels.Matcher) storage.SeriesSet {
	m, added := q.addExternalLabels(matchers)

	res, err := q.client.Read(context.TODO(), q.mint, q.maxt, labelMatchersToProto(m))
	if err != nil {
		return errSeriesSet{err: err}
	}

	series := make([]storage.Series, 0, len(res))
	for _, ts := range res {
		labels := labelPairsToLabels(ts.Labels)
		removeLabels(labels, added)
		series = append(series, &concreteSeries{
			labels:  labels,
			samples: ts.Samples,
		})
	}
	sort.Sort(byLabel(series))
	return &concreteSeriesSet{
		series: series,
	}
}

func labelMatchersToProto(matchers []*labels.Matcher) []*prompb.LabelMatcher {
	pbMatchers := make([]*prompb.LabelMatcher, 0, len(matchers))
	for _, m := range matchers {
		var mType prompb.LabelMatcher_Type
		switch m.Type {
		case labels.MatchEqual:
			mType = prompb.LabelMatcher_EQ
		case labels.MatchNotEqual:
			mType = prompb.LabelMatcher_NEQ
		case labels.MatchRegexp:
			mType = prompb.LabelMatcher_RE
		case labels.MatchNotRegexp:
			mType = prompb.LabelMatcher_NRE
		default:
			panic("invalid matcher type")
		}
		pbMatchers = append(pbMatchers, &prompb.LabelMatcher{
			Type:  mType,
			Name:  string(m.Name),
			Value: string(m.Value),
		})
	}
	return pbMatchers
}

func labelPairsToLabels(labelPairs []*prompb.Label) labels.Labels {
	result := make(labels.Labels, 0, len(labelPairs))
	for _, l := range labelPairs {
		result = append(result, labels.Label{
			Name:  l.Name,
			Value: l.Value,
		})
	}
	sort.Sort(result)
	return result
}

type byLabel []storage.Series

func (a byLabel) Len() int           { return len(a) }
func (a byLabel) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a byLabel) Less(i, j int) bool { return labels.Compare(a[i].Labels(), a[j].Labels()) < 0 }

// LabelValues returns all potential values for a label name.
func (q *querier) LabelValues(name string) ([]string, error) {
	// TODO implement?
	return nil, nil
}

// Close releases the resources of the Querier.
func (q *querier) Close() error {
	return nil
}

// errSeriesSet implements storage.SeriesSet, just returning an error.
type errSeriesSet struct {
	err error
}

func (errSeriesSet) Next() bool {
	return false
}

func (errSeriesSet) At() storage.Series {
	return nil
}

func (e errSeriesSet) Err() error {
	return e.err
}

// concreteSeriesSet implements storage.SeriesSet.
type concreteSeriesSet struct {
	cur    int
	series []storage.Series
}

func (c *concreteSeriesSet) Next() bool {
	c.cur++
	return c.cur < len(c.series)
}

func (c *concreteSeriesSet) At() storage.Series {
	return c.series[c.cur]
}

func (c *concreteSeriesSet) Err() error {
	return nil
}

// concreteSeries implementes storage.Series.
type concreteSeries struct {
	labels  labels.Labels
	samples []*prompb.Sample
}

func (c *concreteSeries) Labels() labels.Labels {
	return c.labels
}

func (c *concreteSeries) Iterator() storage.SeriesIterator {
	return newConcreteSeriersIterator(c)
}

// concreteSeriesIterator implements storage.SeriesIterator.
type concreteSeriesIterator struct {
	cur    int
	series *concreteSeries
}

func newConcreteSeriersIterator(series *concreteSeries) storage.SeriesIterator {
	return &concreteSeriesIterator{
		cur:    -1,
		series: series,
	}
}

func (c *concreteSeriesIterator) Seek(t int64) bool {
	c.cur = sort.Search(len(c.series.samples), func(n int) bool {
		return c.series.samples[n].Timestamp >= t
	})
	return c.cur < len(c.series.samples)
}

func (c *concreteSeriesIterator) At() (t int64, v float64) {
	s := c.series.samples[c.cur]
	return s.Timestamp, s.Value
}

func (c *concreteSeriesIterator) Next() bool {
	c.cur++
	return c.cur < len(c.series.samples)
}

func (c *concreteSeriesIterator) Err() error {
	return nil
}

// addExternalLabels adds matchers for each external label. External labels
// that already have a corresponding user-supplied matcher are skipped, as we
// assume that the user explicitly wants to select a different value for them.
// We return the new set of matchers, along with a map of labels for which
// matchers were added, so that these can later be removed from the result
// time series again.
func (q *querier) addExternalLabels(matchers []*labels.Matcher) ([]*labels.Matcher, model.LabelSet) {
	el := make(model.LabelSet, len(q.externalLabels))
	for k, v := range q.externalLabels {
		el[k] = v
	}
	for _, m := range matchers {
		if _, ok := el[model.LabelName(m.Name)]; ok {
			delete(el, model.LabelName(m.Name))
		}
	}
	for k, v := range el {
		m, err := labels.NewMatcher(labels.MatchEqual, string(k), string(v))
		if err != nil {
			panic(err)
		}
		matchers = append(matchers, m)
	}
	return matchers, el
}

func removeLabels(l labels.Labels, toDelete model.LabelSet) {
	for i := 0; i < len(l); {
		if _, ok := toDelete[model.LabelName(l[i].Name)]; ok {
			l = l[:i+copy(l[i:], l[i+1:])]
		} else {
			i++
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 gopm authors.
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package doc

import (
	"bytes"
	"go/ast"
	"go/build"
	"go/doc"
	"go/parser"
	"go/token"
	"io"
	"io/ioutil"
	"os"
	"path"
	"runtime"
	"strings"
	"unicode"
	"unicode/utf8"

	"github.com/Unknwon/com"
)

type sliceWriter struct{ p *[]byte }

func (w sliceWriter) Write(p []byte) (int, error) {
	*w.p = append(*w.p, p...)
	return len(p), nil
}

func (w *walker) readDir(dir string) ([]os.FileInfo, error) {
	if dir != w.ImportPath {
		panic("unexpected")
	}
	fis := make([]os.FileInfo, 0, len(w.srcs))
	for _, src := range w.srcs {
		fis = append(fis, src)
	}
	return fis, nil
}

func (w *walker) openFile(path string) (io.ReadCloser, error) {
	if strings.HasPrefix(path, w.ImportPath+"/") {
		if src, ok := w.srcs[path[len(w.ImportPath)+1:]]; ok {
			return ioutil.NopCloser(bytes.NewReader(src.data)), nil
		}
	}
	panic("unexpected")
}

func simpleImporter(imports map[string]*ast.Object, path string) (*ast.Object, error) {
	pkg := imports[path]
	if pkg == nil {
		// Guess the package name without importing it. Start with the last
		// element of the path.
		name := path[strings.LastIndex(path, "/")+1:]

		// Trim commonly used prefixes and suffixes containing illegal name
		// runes.
		name = strings.TrimSuffix(name, ".go")
		name = strings.TrimSuffix(name, "-go")
		name = strings.TrimPrefix(name, "go.")
		name = strings.TrimPrefix(name, "go-")
		name = strings.TrimPrefix(name, "biogo.")

		// It's also common for the last element of the path to contain an
		// extra "go" prefix, but not always. TODO: examine unresolved ids to
		// detect when trimming the "go" prefix is appropriate.

		pkg = ast.NewObj(ast.Pkg, name)
		pkg.Data = ast.NewScope(nil)
		imports[path] = pkg
	}
	return pkg, nil
}

// build gets imports from source files.
func (w *walker) build(srcs []*source, nod *Node) ([]string, error) {
	// Add source files to walker, I skipped references here.
	w.srcs = make(map[string]*source)
	for _, src := range srcs {
		w.srcs[src.name] = src
	}

	w.fset = token.NewFileSet()

	// Find the package and associated files.
	ctxt := build.Context{
		GOOS:          runtime.GOOS,
		GOARCH:        runtime.GOARCH,
		CgoEnabled:    true,
		JoinPath:      path.Join,
		IsAbsPath:     path.IsAbs,
		SplitPathList: func(list string) []string { return strings.Split(list, ":") },
		IsDir:         func(path string) bool { panic("unexpected") },
		HasSubdir:     func(root, dir string) (rel string, ok bool) { panic("unexpected") },
		ReadDir:       func(dir string) (fi []os.FileInfo, err error) { return w.readDir(dir) },
		OpenFile:      func(path string) (r io.ReadCloser, err error) { return w.openFile(path) },
		Compiler:      "gc",
	}

	bpkg, err := ctxt.ImportDir(w.ImportPath, 0)
	// Continue if there are no Go source files; we still want the directory info.
	_, nogo := err.(*build.NoGoError)
	if err != nil {
		if nogo {
			err = nil
		} else {
			com.ColorLog("[WARN] Error occurs when check imports[ %s ]\n", err)
			return nil, nil
		}
	}

	// Parse the Go files

	files := make(map[string]*ast.File)
	for _, name := range append(bpkg.GoFiles, bpkg.CgoFiles...) {
		file, err := parser.ParseFile(w.fset, name, w.srcs[name].data, parser.ParseComments)
		if err != nil {
			//beego.Error("doc.walker.build():", err)
			continue
		}
		files[name] = file
	}

	w.ImportPath = strings.Replace(w.ImportPath, "\\", "/", -1)
	var imports []string
	for _, v := range bpkg.Imports {
		// Skip strandard library.
		if !IsGoRepoPath(v) &&
			(GetProjectPath(v) != GetProjectPath(w.ImportPath)) {
			imports = append(imports, v)
		}
	}

	apkg, _ := ast.NewPackage(w.fset, files, simpleImporter, nil)

	mode := doc.Mode(0)
	if w.ImportPath == "builtin" {
		mode |= doc.AllDecls
	}

	pdoc := doc.New(apkg, w.ImportPath, mode)

	if nod != nil {
		nod.Synopsis = Synopsis(pdoc.Doc)
		if i := strings.Index(nod.Synopsis, "\n"); i > -1 {
			nod.Synopsis = nod.Synopsis[:i]
		}
	}

	return imports, err
}

var badSynopsisPrefixes = []string{
	"Autogenerated by Thrift Compiler",
	"Automatically generated ",
	"Auto-generated by ",
	"Copyright ",
	"COPYRIGHT ",
	`THE SOFTWARE IS PROVIDED "AS IS"`,
	"TODO: ",
	"vim:",
}

// Synopsis extracts the first sentence from s. All runs of whitespace are
// replaced by a single space.
func Synopsis(s string) string {

	parts := strings.SplitN(s, "\n\n", 2)
	s = parts[0]

	var buf []byte
	const (
		other = iota
		period
		space
	)
	last := space
Loop:
	for i := 0; i < len(s); i++ {
		b := s[i]
		switch b {
		case ' ', '\t', '\r', '\n':
			switch last {
			case period:
				break Loop
			case other:
				buf = append(buf, ' ')
				last = space
			}
		case '.':
			last = period
			buf = append(buf, b)
		default:
			last = other
			buf = append(buf, b)
		}
	}

	// Ensure that synopsis fits an App Engine datastore text property.
	const m = 400
	if len(buf) > m {
		buf = buf[:m]
		if i := bytes.LastIndex(buf, []byte{' '}); i >= 0 {
			buf = buf[:i]
		}
		buf = append(buf, " ..."...)
	}

	s = string(buf)

	r, n := utf8.DecodeRuneInString(s)
	if n < 0 || unicode.IsPunct(r) || unicode.IsSymbol(r) {
		// ignore Markdown headings, editor settings, Go build constraints, and * in poorly formatted block comments.
		s = ""
	} else {
		for _, prefix := range badSynopsisPrefixes {
			if strings.HasPrefix(s, prefix) {
				s = ""
				break
			}
		}
	}

	return s
}
<commit_msg>Fixed bug for gopm build<commit_after>// Copyright 2013 gopm authors.
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package doc

import (
	"bytes"
	"go/ast"
	"go/build"
	"go/doc"
	"go/parser"
	"go/token"
	"io"
	"io/ioutil"
	"os"
	"path"
	"runtime"
	"strings"
	"unicode"
	"unicode/utf8"

	"github.com/Unknwon/com"

	"github.com/gpmgo/gopm/log"
)

type sliceWriter struct{ p *[]byte }

func (w sliceWriter) Write(p []byte) (int, error) {
	*w.p = append(*w.p, p...)
	return len(p), nil
}

func (w *walker) readDir(dir string) ([]os.FileInfo, error) {
	if dir != w.ImportPath {
		panic("unexpected")
	}
	fis := make([]os.FileInfo, 0, len(w.srcs))
	for _, src := range w.srcs {
		fis = append(fis, src)
	}
	return fis, nil
}

func (w *walker) openFile(path string) (io.ReadCloser, error) {
	if strings.HasPrefix(path, w.ImportPath+"/") {
		if src, ok := w.srcs[path[len(w.ImportPath)+1:]]; ok {
			return ioutil.NopCloser(bytes.NewReader(src.data)), nil
		}
	}
	panic("unexpected")
}

func simpleImporter(imports map[string]*ast.Object, path string) (*ast.Object, error) {
	pkg := imports[path]
	if pkg == nil {
		// Guess the package name without importing it. Start with the last
		// element of the path.
		name := path[strings.LastIndex(path, "/")+1:]

		// Trim commonly used prefixes and suffixes containing illegal name
		// runes.
		name = strings.TrimSuffix(name, ".go")
		name = strings.TrimSuffix(name, "-go")
		name = strings.TrimPrefix(name, "go.")
		name = strings.TrimPrefix(name, "go-")
		name = strings.TrimPrefix(name, "biogo.")

		// It's also common for the last element of the path to contain an
		// extra "go" prefix, but not always. TODO: examine unresolved ids to
		// detect when trimming the "go" prefix is appropriate.

		pkg = ast.NewObj(ast.Pkg, name)
		pkg.Data = ast.NewScope(nil)
		imports[path] = pkg
	}
	return pkg, nil
}

// build gets imports from source files.
func (w *walker) build(srcs []*source, nod *Node) ([]string, error) {
	// Add source files to walker, I skipped references here.
	w.srcs = make(map[string]*source)
	for _, src := range srcs {
		w.srcs[src.name] = src
	}

	w.fset = token.NewFileSet()

	// Find the package and associated files.
	ctxt := build.Context{
		GOOS:          runtime.GOOS,
		GOARCH:        runtime.GOARCH,
		CgoEnabled:    true,
		JoinPath:      path.Join,
		IsAbsPath:     path.IsAbs,
		SplitPathList: func(list string) []string { return strings.Split(list, ":") },
		IsDir:         func(path string) bool { panic("unexpected") },
		HasSubdir:     func(root, dir string) (rel string, ok bool) { panic("unexpected") },
		ReadDir:       func(dir string) (fi []os.FileInfo, err error) { return w.readDir(dir) },
		OpenFile:      func(path string) (r io.ReadCloser, err error) { return w.openFile(path) },
		Compiler:      "gc",
	}

	bpkg, err := ctxt.ImportDir(w.ImportPath, 0)
	// Continue if there are no Go source files; we still want the directory info.
	_, nogo := err.(*build.NoGoError)
	if err != nil {
		if nogo {
			err = nil
		} else {
			log.Error("walker", "Error occurs when check imports:")
			log.Error("", "\t"+err.Error())
			return nil, nil
		}
	}

	// Parse the Go files

	files := make(map[string]*ast.File)
	for _, name := range append(bpkg.GoFiles, bpkg.CgoFiles...) {
		file, err := parser.ParseFile(w.fset, name, w.srcs[name].data, parser.ParseComments)
		if err != nil {
			//beego.Error("doc.walker.build():", err)
			continue
		}
		files[name] = file
	}

	w.ImportPath = strings.Replace(w.ImportPath, "\\", "/", -1)
	var imports []string
	for _, v := range bpkg.Imports {
		// Skip strandard library.
		if !IsGoRepoPath(v) &&
			(GetProjectPath(v) != GetProjectPath(w.ImportPath)) {
			imports = append(imports, v)
		}
	}

	apkg, _ := ast.NewPackage(w.fset, files, simpleImporter, nil)

	mode := doc.Mode(0)
	if w.ImportPath == "builtin" {
		mode |= doc.AllDecls
	}

	pdoc := doc.New(apkg, w.ImportPath, mode)

	if nod != nil {
		nod.Synopsis = Synopsis(pdoc.Doc)
		if i := strings.Index(nod.Synopsis, "\n"); i > -1 {
			nod.Synopsis = nod.Synopsis[:i]
		}
	}

	return imports, err
}

var badSynopsisPrefixes = []string{
	"Autogenerated by Thrift Compiler",
	"Automatically generated ",
	"Auto-generated by ",
	"Copyright ",
	"COPYRIGHT ",
	`THE SOFTWARE IS PROVIDED "AS IS"`,
	"TODO: ",
	"vim:",
}

// Synopsis extracts the first sentence from s. All runs of whitespace are
// replaced by a single space.
func Synopsis(s string) string {

	parts := strings.SplitN(s, "\n\n", 2)
	s = parts[0]

	var buf []byte
	const (
		other = iota
		period
		space
	)
	last := space
Loop:
	for i := 0; i < len(s); i++ {
		b := s[i]
		switch b {
		case ' ', '\t', '\r', '\n':
			switch last {
			case period:
				break Loop
			case other:
				buf = append(buf, ' ')
				last = space
			}
		case '.':
			last = period
			buf = append(buf, b)
		default:
			last = other
			buf = append(buf, b)
		}
	}

	// Ensure that synopsis fits an App Engine datastore text property.
	const m = 400
	if len(buf) > m {
		buf = buf[:m]
		if i := bytes.LastIndex(buf, []byte{' '}); i >= 0 {
			buf = buf[:i]
		}
		buf = append(buf, " ..."...)
	}

	s = string(buf)

	r, n := utf8.DecodeRuneInString(s)
	if n < 0 || unicode.IsPunct(r) || unicode.IsSymbol(r) {
		// ignore Markdown headings, editor settings, Go build constraints, and * in poorly formatted block comments.
		s = ""
	} else {
		for _, prefix := range badSynopsisPrefixes {
			if strings.HasPrefix(s, prefix) {
				s = ""
				break
			}
		}
	}

	return s
}
<|endoftext|>


<commit_before>// +build integration

package test

import (
	"encoding/json"
	"errors"
	"fmt"
	"os/exec"
	"strings"
	"testing"
	"time"

	"github.com/micro/micro/v2/client/cli/token"
)

func TestServerAuth(t *testing.T) {
	trySuite(t, testServerAuth, retryCount)
}

func testServerAuth(t *t) {
	t.Parallel()
	serv := newServer(t)
	defer serv.close()
	if err := serv.launch(); err != nil {
		return
	}

	basicAuthSuite(serv, t)
}

func TestServerAuthJWT(t *testing.T) {
	trySuite(t, testServerAuthJWT, retryCount)
}

func testServerAuthJWT(t *t) {
	t.Parallel()
	serv := newServer(t, options{
		auth: "jwt",
	})
	defer serv.close()
	if err := serv.launch(); err != nil {
		return
	}

	basicAuthSuite(serv, t)
}

func basicAuthSuite(serv testServer, t *t) {
	login(serv, t, "default", "password")

	// Execute first command in read to wait for store service
	// to start up
	if err := try("Calling micro auth list accounts", t, func() ([]byte, error) {
		readCmd := exec.Command("micro", serv.envFlag(), "auth", "list", "accounts")
		outp, err := readCmd.CombinedOutput()
		if err != nil {
			return outp, err
		}
		if !strings.Contains(string(outp), "admin") ||
			!strings.Contains(string(outp), "default") {
			return outp, fmt.Errorf("Output should contain default admin account")
		}
		return outp, nil
	}, 15*time.Second); err != nil {
		return
	}

	if err := try("Calling micro auth list rules", t, func() ([]byte, error) {
		readCmd := exec.Command("micro", serv.envFlag(), "auth", "list", "rules")
		outp, err := readCmd.CombinedOutput()
		if err != nil {
			return outp, err
		}
		if !strings.Contains(string(outp), "default") {
			return outp, fmt.Errorf("Output should contain default rule")
		}
		return outp, nil
	}, 8*time.Second); err != nil {
		return
	}

	if err := try("Try to get token with default account", t, func() ([]byte, error) {
		readCmd := exec.Command("micro", serv.envFlag(), "call", "go.micro.auth", "Auth.Token", `{"id":"default","secret":"password"}`)
		outp, err := readCmd.CombinedOutput()
		if err != nil {
			return outp, err
		}
		rsp := map[string]interface{}{}
		err = json.Unmarshal(outp, &rsp)
		token, ok := rsp["token"].(map[string]interface{})
		if !ok {
			return outp, errors.New("Can't find token")
		}
		if _, ok = token["access_token"].(string); !ok {
			return outp, fmt.Errorf("Can't find access token")
		}
		if _, ok = token["refresh_token"].(string); !ok {
			return outp, fmt.Errorf("Can't find access token")
		}
		if _, ok = token["refresh_token"].(string); !ok {
			return outp, fmt.Errorf("Can't find refresh token")
		}
		if _, ok = token["expiry"].(string); !ok {
			return outp, fmt.Errorf("Can't find access token")
		}
		return outp, nil
	}, 8*time.Second); err != nil {
		return
	}
}

func TestServerLoginJWT(t *testing.T) {
	trySuite(t, testServerAuthJWT, retryCount)
}

func testServerLoginJWT(t *t) {
	t.Parallel()
	serv := newServer(t, options{
		auth: "jwt",
	})
	defer serv.close()
	if err := serv.launch(); err != nil {
		return
	}

	login(serv, t, "default", "password")
}

// Test bad tokens by messing up refresh token and trying to log in
// - this used to make even login fail which resulted in a UX deadlock
func TestServerBadTokenJWT(t *testing.T) {
	trySuite(t, testServerAuthJWT, retryCount)
}

func testServerBadTokenJWT(t *t) {
	t.Parallel()
	serv := newServer(t, options{
		auth: "jwt",
	})
	defer serv.close()
	if err := serv.launch(); err != nil {
		return
	}

	login(serv, t, "default", "password")

	// Micro status should work
	if err := try("Micro status", t, func() ([]byte, error) {
		return exec.Command("micro", serv.envFlag(), "status").CombinedOutput()
	}, 3*time.Second); err != nil {
		return
	}

	// Modify rules so only logged in users can do anything

	// Add new rule that only lets logged in users do anything
	outp, err := exec.Command("micro", serv.envFlag(), "auth", "create", "rule", "--access=granted", "--scope='*'", "--resource='*:*:*'", "onlyloggedin").CombinedOutput()
	if err != nil {
		t.Fatal(string(outp))
		return
	}
	// Remove default rule
	outp, err = exec.Command("micro", serv.envFlag(), "auth", "delete", "rule", "default").CombinedOutput()
	if err != nil {
		t.Fatal(string(outp))
		return
	}

	// Micro status should still work as our user is already logged in
	if err := try("Micro status", t, func() ([]byte, error) {
		return exec.Command("micro", serv.envFlag(), "status").CombinedOutput()
	}, 3*time.Second); err != nil {
		return
	}

	// Now get the token and mess it up

	tok, err := token.Get(serv.envName())
	if err != nil {
		t.Fatal(err)
		return
	}

	tok.AccessToken = ""
	tok.Expiry = time.Time{}
	tok.RefreshToken = "some-random-junk"

	err = token.Save(serv.envName(), tok)
	if err != nil {
		t.Fatal(err)
		return
	}

	// Micro status should fail
	if err := try("Micro status", t, func() ([]byte, error) {
		outp, err := exec.Command("micro", serv.envFlag(), "status").CombinedOutput()
		if err == nil {
			return outp, errors.New("Micro status should fail")
		}
		return outp, err
	}, 3*time.Second); err != nil {
		return
	}

	login(serv, t, "default", "password")

	// Micro status should still work again after login
	if err := try("Micro status", t, func() ([]byte, error) {
		return exec.Command("micro", serv.envFlag(), "status").CombinedOutput()
	}, 3*time.Second); err != nil {
		return
	}
}
<commit_msg>Auth tutor tests (#1099)<commit_after>// +build integration

package test

import (
	"encoding/json"
	"errors"
	"fmt"
	"os/exec"
	"strings"
	"testing"
	"time"

	"github.com/micro/micro/v2/client/cli/token"
)

func TestServerAuth(t *testing.T) {
	trySuite(t, testServerAuth, retryCount)
}

func testServerAuth(t *t) {
	t.Parallel()
	serv := newServer(t)
	defer serv.close()
	if err := serv.launch(); err != nil {
		return
	}

	basicAuthSuite(serv, t)
}

func TestServerAuthJWT(t *testing.T) {
	trySuite(t, testServerAuthJWT, retryCount)
}

func testServerAuthJWT(t *t) {
	t.Parallel()
	serv := newServer(t, options{
		auth: "jwt",
	})
	defer serv.close()
	if err := serv.launch(); err != nil {
		return
	}

	basicAuthSuite(serv, t)
}

func basicAuthSuite(serv testServer, t *t) {
	login(serv, t, "default", "password")

	// Execute first command in read to wait for store service
	// to start up
	if err := try("Calling micro auth list accounts", t, func() ([]byte, error) {
		readCmd := exec.Command("micro", serv.envFlag(), "auth", "list", "accounts")
		outp, err := readCmd.CombinedOutput()
		if err != nil {
			return outp, err
		}
		if !strings.Contains(string(outp), "admin") ||
			!strings.Contains(string(outp), "default") {
			return outp, fmt.Errorf("Output should contain default admin account")
		}
		return outp, nil
	}, 15*time.Second); err != nil {
		return
	}

	if err := try("Calling micro auth list rules", t, func() ([]byte, error) {
		readCmd := exec.Command("micro", serv.envFlag(), "auth", "list", "rules")
		outp, err := readCmd.CombinedOutput()
		if err != nil {
			return outp, err
		}
		if !strings.Contains(string(outp), "default") {
			return outp, fmt.Errorf("Output should contain default rule")
		}
		return outp, nil
	}, 8*time.Second); err != nil {
		return
	}

	if err := try("Try to get token with default account", t, func() ([]byte, error) {
		readCmd := exec.Command("micro", serv.envFlag(), "call", "go.micro.auth", "Auth.Token", `{"id":"default","secret":"password"}`)
		outp, err := readCmd.CombinedOutput()
		if err != nil {
			return outp, err
		}
		rsp := map[string]interface{}{}
		err = json.Unmarshal(outp, &rsp)
		token, ok := rsp["token"].(map[string]interface{})
		if !ok {
			return outp, errors.New("Can't find token")
		}
		if _, ok = token["access_token"].(string); !ok {
			return outp, fmt.Errorf("Can't find access token")
		}
		if _, ok = token["refresh_token"].(string); !ok {
			return outp, fmt.Errorf("Can't find access token")
		}
		if _, ok = token["refresh_token"].(string); !ok {
			return outp, fmt.Errorf("Can't find refresh token")
		}
		if _, ok = token["expiry"].(string); !ok {
			return outp, fmt.Errorf("Can't find access token")
		}
		return outp, nil
	}, 8*time.Second); err != nil {
		return
	}
}

func TestServerLoginJWT(t *testing.T) {
	trySuite(t, testServerAuthJWT, retryCount)
}

func testServerLoginJWT(t *t) {
	t.Parallel()
	serv := newServer(t, options{
		auth: "jwt",
	})
	defer serv.close()
	if err := serv.launch(); err != nil {
		return
	}

	login(serv, t, "default", "password")
}

// Test bad tokens by messing up refresh token and trying to log in
// - this used to make even login fail which resulted in a UX deadlock
func TestServerBadTokenJWT(t *testing.T) {
	trySuite(t, testServerAuthJWT, retryCount)
}

func testServerBadTokenJWT(t *t) {
	t.Parallel()
	serv := newServer(t, options{
		auth: "jwt",
	})
	defer serv.close()
	if err := serv.launch(); err != nil {
		return
	}

	login(serv, t, "default", "password")

	// Micro status should work
	if err := try("Micro status", t, func() ([]byte, error) {
		return exec.Command("micro", serv.envFlag(), "status").CombinedOutput()
	}, 3*time.Second); err != nil {
		return
	}

	// Modify rules so only logged in users can do anything

	// Add new rule that only lets logged in users do anything
	outp, err := exec.Command("micro", serv.envFlag(), "auth", "create", "rule", "--access=granted", "--scope='*'", "--resource='*:*:*'", "onlyloggedin").CombinedOutput()
	if err != nil {
		t.Fatal(string(outp))
		return
	}
	// Remove default rule
	outp, err = exec.Command("micro", serv.envFlag(), "auth", "delete", "rule", "default").CombinedOutput()
	if err != nil {
		t.Fatal(string(outp))
		return
	}

	// Micro status should still work as our user is already logged in
	if err := try("Micro status", t, func() ([]byte, error) {
		return exec.Command("micro", serv.envFlag(), "status").CombinedOutput()
	}, 3*time.Second); err != nil {
		return
	}

	// Now get the token and mess it up

	tok, err := token.Get(serv.envName())
	if err != nil {
		t.Fatal(err)
		return
	}

	tok.AccessToken = ""
	tok.Expiry = time.Time{}
	tok.RefreshToken = "some-random-junk"

	err = token.Save(serv.envName(), tok)
	if err != nil {
		t.Fatal(err)
		return
	}

	// Micro status should fail
	if err := try("Micro status", t, func() ([]byte, error) {
		outp, err := exec.Command("micro", serv.envFlag(), "status").CombinedOutput()
		if err == nil {
			return outp, errors.New("Micro status should fail")
		}
		return outp, err
	}, 3*time.Second); err != nil {
		return
	}

	login(serv, t, "default", "password")

	// Micro status should still work again after login
	if err := try("Micro status", t, func() ([]byte, error) {
		return exec.Command("micro", serv.envFlag(), "status").CombinedOutput()
	}, 3*time.Second); err != nil {
		return
	}
}

func TestServerLockdown(t *testing.T) {
	trySuite(t, testServerAuth, retryCount)
}

func testServerLockdown(t *t) {
	t.Parallel()
	serv := newServer(t)
	defer serv.close()
	if err := serv.launch(); err != nil {
		return
	}

	lockdownSuite(serv, t)
}

func TestServerLockdownJWT(t *testing.T) {
	trySuite(t, testServerAuthJWT, retryCount)
}

func testServerLockdownJWT(t *t) {
	t.Parallel()
	serv := newServer(t, options{
		auth: "jwt",
	})
	defer serv.close()
	if err := serv.launch(); err != nil {
		return
	}

	basicAuthSuite(serv, t)
}

func lockdownSuite(serv testServer, t *t) {
	// Execute first command in read to wait for store service
	// to start up
	if err := try("Calling micro auth list rules", t, func() ([]byte, error) {
		readCmd := exec.Command("micro", serv.envFlag(), "auth", "list", "rules")
		outp, err := readCmd.CombinedOutput()
		if err != nil {
			return outp, err
		}
		if !strings.Contains(string(outp), "default") {
			return outp, fmt.Errorf("Output should contain default rule")
		}
		return outp, nil
	}, 15*time.Second); err != nil {
		return
	}

	email := "me@email.com"
	pass := "mystrongpass"

	outp, err := exec.Command("micro", serv.envFlag(), "auth", "create", "account", "--secret", pass, "--scopes", "admin", email).CombinedOutput()
	if err != nil {
		t.Fatal(string(outp), err)
		return
	}

	outp, err = exec.Command("micro", serv.envFlag(), "auth", "create", "rule", "--access=granted", "--scope='*'", "--resource='*:*:*'", "onlyloggedin").CombinedOutput()
	if err != nil {
		t.Fatal(string(outp), err)
		return
	}

	outp, err = exec.Command("micro", serv.envFlag(), "auth", "create", "rule", "--access=granted", "--scope=''", "authpublic").CombinedOutput()
	if err != nil {
		t.Fatal(string(outp), err)
		return
	}

	outp, err = exec.Command("micro", serv.envFlag(), "auth", "delete", "rule", "default").CombinedOutput()
	if err != nil {
		t.Fatal(string(outp), err)
		return
	}

	outp, err = exec.Command("micro", serv.envFlag(), "auth", "delete", "account", "default").CombinedOutput()
	if err != nil {
		t.Fatal(string(outp), err)
		return
	}

	if err := try("Listing rules should fail before login", t, func() ([]byte, error) {
		outp, err := exec.Command("micro", serv.envFlag(), "auth", "list", "rules").CombinedOutput()
		if err == nil {
			return outp, errors.New("List rules should fail")
		}
		return outp, err
	}, 31*time.Second); err != nil {
		return
	}

	login(serv, t, "me@email.com", "mystrongpass")

	if err := try("Listing rules should pass after login", t, func() ([]byte, error) {
		outp, err := exec.Command("micro", serv.envFlag(), "auth", "list", "rules").CombinedOutput()
		if err != nil {
			return outp, err
		}
		if !strings.Contains(string(outp), "onlyloggedin") || !strings.Contains(string(outp), "authpublic") {
			return outp, errors.New("Can't find rules")
		}
		return outp, err
	}, 31*time.Second); err != nil {
		return
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	jseg "github.com/garyhouston/jpegsegs"
	"log"
	"os"
)

// Print the JPEG markers and segment lengths.
func main() {
	if len(os.Args) != 2 {
		fmt.Printf("Usage: %s file\n", os.Args[0])
		return
	}
	reader, err := os.Open(os.Args[1])
	if err != nil {
		log.Fatal(err)
	}
	defer reader.Close()
	scanner, err := jseg.NewScanner(reader)
	if err != nil {
		log.Fatal(err)
	}
	dataCount := uint32(0)
	resetCount := uint32(0)
	for {
		marker, buf, err := scanner.Scan()
		if err != nil {
			log.Fatal(err)
		}
		if marker == 0 {
			dataCount += uint32(len(buf))
			continue
		}
		if buf == nil && (marker >= jseg.RST0 && marker <= jseg.RST0+7) {
				resetCount++
				continue
		}
		if dataCount > 0 || resetCount > 0 {
			fmt.Printf("%d bytes of image data", dataCount)
			if resetCount > 0 {
				fmt.Printf(" and %d reset markers", resetCount)
			}
			fmt.Println()
		}
		if buf == nil {
			fmt.Println(marker.Name())
			if marker == jseg.EOI {
				break
			}
			continue
		}
		if marker == jseg.APP0+2 {
			isMPF, _ := jseg.GetMPFHeader(buf)
			if isMPF {
				fmt.Printf("%s, %d bytes (MPF segment)\n", marker.Name(), len(buf))
				continue
			}
		}
		fmt.Printf("%s, %d bytes\n", marker.Name(), len(buf))
	}
}
<commit_msg>Print multiple images encoded with MPF if present.<commit_after>package main

// Print JPEG markers and segment lengths.

import (
	"errors"
	"fmt"
	jseg "github.com/garyhouston/jpegsegs"
	tiff "github.com/garyhouston/tiff66"
	"io"
	"log"
	"os"
)

// Process a single image. A file using the MPF extensions can contain
// multiple images. Returns the MPF segment and MPF offset, if found.
func scanImage(reader io.ReadSeeker) ([]byte, uint32, error) {
	var mpfSegment []byte
	mpfOffset := uint32(0)
	scanner, err := jseg.NewScanner(reader)
	if err != nil {
		return nil, 0, err
	}
	fmt.Println("SOI")
	dataCount := uint32(0)
	resetCount := uint32(0)
	for {
		marker, buf, err := scanner.Scan()
		if err != nil {
			return nil, 0, err
		}
		if marker == 0 {
			dataCount += uint32(len(buf))
			continue
		}
		if buf == nil && (marker >= jseg.RST0 && marker <= jseg.RST0+7) {
			resetCount++
			continue
		}
		if dataCount > 0 || resetCount > 0 {
			fmt.Printf("%d bytes of image data", dataCount)
			if resetCount > 0 {
				fmt.Printf(" and %d reset markers", resetCount)
			}
			fmt.Println()
		}
		if buf == nil {
			fmt.Println(marker.Name())
			if marker == jseg.EOI {
				return mpfSegment, mpfOffset, nil
			}
			continue
		}
		if marker == jseg.APP0+2 {
			isMPF, next := jseg.GetMPFHeader(buf)
			if isMPF {
				fmt.Printf("%s, %d bytes (MPF segment)\n", marker.Name(), len(buf))
				mpfSegment = make([]byte, len(buf[next:]))
				copy(mpfSegment, buf[next:])
				// MPF offsets are relative to the start of the
				// MPF header, which is 4 bytes past the start
				// of buf.
				pos, err := reader.Seek(0, io.SeekCurrent)
				if err != nil {
					return nil, 0, err
				}
				mpfOffset = uint32(pos) - uint32(len(buf)-4)
				continue
			}
		}
		fmt.Printf("%s, %d bytes\n", marker.Name(), len(buf))
	}
}

func scanMPFImages(reader io.ReadSeeker, mpfSegment []byte, mpfOffset uint32) error {
	if mpfSegment != nil {
		mpfTree, err := jseg.GetMPFTree(mpfSegment)
		if err != nil {
			return err
		}
		if mpfTree.Space != tiff.MPFIndexSpace {
			return errors.New("MPF segment doesn't contain Index")
		}
		order := mpfTree.Order
		count := uint32(0)
		sizes := []uint32(nil)
		offsets := []uint32(nil)
		for _, f := range mpfTree.Fields {
			switch f.Tag {
			case jseg.MPFNumberOfImages:
				count = f.Long(0, order)
				sizes = make([]uint32, count)
				offsets = make([]uint32, count)
			case jseg.MPFEntry:
				for i := uint32(0); i < count; i++ {
					sizes[i] = f.Long(i*4+1, order)
					offsets[i] = f.Long(i*4+2, order)
				}
			}
		}
		for i := uint32(0); i < count; i++ {
			if offsets[i] > 0 {
				fmt.Printf("MPF image %d at offset %d, size %d\n", i+1, mpfOffset+offsets[i], sizes[i])
				if _, err = reader.Seek(int64(offsets[i]+mpfOffset), io.SeekStart); err != nil {
					return err
				}
				if _, _, err := scanImage(reader); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

func main() {
	if len(os.Args) != 2 {
		fmt.Printf("Usage: %s file\n", os.Args[0])
		return
	}
	reader, err := os.Open(os.Args[1])
	if err != nil {
		log.Fatal(err)
	}
	defer reader.Close()
	mpfSegment, mpfOffset, err := scanImage(reader)
	if err != nil {
		log.Fatal(err)
	}
	err = scanMPFImages(reader, mpfSegment, mpfOffset)
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package kdtree

import (
	"geo"
	"graph"
)

const (
	TotalBits       = 34
	TypeSize        = 64
	VertexIndexBits = 18
	EdgeOffsetBits  = 5
	StepOffsetBits  = 11
	MaxVertexIndex  = 0x3FFFF // (1 << VertexIndexBits) - 1
	MaxEdgeOffset   = 0x1F    // (1 << EdgeOffsetBits) - 1
	MaxStepOffset   = 0x7FF   // (1 << StepOffsetBits) - 1
)

// encoded step: vertex index (18bit) + edge offset (8bit) + step offset (8bit)
type KdTree struct {
	Graph        graph.Graph
	EncodedSteps []uint64
	Coordinates  []geo.Coordinate
	// It is inefficient to create a sub slice of EncodedSteps due to the used encoding.
	// Thus, we use start and end pointer instead.
	EncodedStepsStart int
	EncodedStepsEnd   int
}

type ClusterKdTree struct {
	Overlay *KdTree
	Cluster []*KdTree
	BBoxes  []geo.BBox
}

func (t *KdTree) Len() int {
	return len(t.Coordinates)
}

func (t *KdTree) Swap(i, j int) {
	t.Coordinates[i], t.Coordinates[j] = t.Coordinates[j], t.Coordinates[i]
	tmp := t.EncodedStep(j)
	t.SetEncodedStep(j, t.EncodedStep(i))
	t.SetEncodedStep(i, tmp)
}

func (t *KdTree) EncodedStepLen() int {
	if t.EncodedStepsEnd > 0 {
		return t.EncodedStepsEnd + 1
	}
	l := (len(t.EncodedSteps) * TypeSize) / TotalBits
	if l > 0 && t.EncodedStep(l-1) == (1<<TotalBits)-1 {
		return l - 1
	}
	return l
}

func (t *KdTree) EncodedStep(i int) uint64 {
	index := t.EncodedStepsStart + i*TotalBits/TypeSize
	offset := i * TotalBits % TypeSize
	if offset+TotalBits <= TypeSize {
		// contained in one uint64
		mask := (uint64(1) << TotalBits) - 1
		return (t.EncodedSteps[index] >> uint(offset)) & mask
	}
	// split over two uint64
	first := uint(TypeSize - offset)
	second := uint(TotalBits - first)

	fMask := ((uint64(1) << first) - 1)
	result := ((t.EncodedSteps[index] >> uint(offset)) & fMask) << second

	//sShift := TypeSize - second
	sMask := (uint64(1) << second) - 1
	result |= t.EncodedSteps[index+1] & sMask
	return result
}

func (t *KdTree) SetEncodedStep(i int, s uint64) {
	index := t.EncodedStepsStart + i*TotalBits/TypeSize
	offset := i * TotalBits % TypeSize
	if offset+TotalBits <= TypeSize {
		// contained in one uint64
		mask := (uint64(1) << TotalBits) - 1
		t.EncodedSteps[index] ^= t.EncodedSteps[index] & (mask << uint(offset))
		t.EncodedSteps[index] |= s << uint(offset)
	} else {
		// split over two uint64
		first := uint(TypeSize - offset)
		second := uint(TotalBits - first)

		fMask := (uint64(1) << first) - 1
		t.EncodedSteps[index] ^= t.EncodedSteps[index] & (fMask << uint(offset))
		t.EncodedSteps[index] |= (s >> second) << uint(offset)

		sMask := (uint64(1) << second) - 1
		t.EncodedSteps[index+1] ^= t.EncodedSteps[index+1] & sMask
		t.EncodedSteps[index+1] |= s & sMask
	}
}

func (t *KdTree) AppendEncodedStep(s uint64) {
	l := t.EncodedStepLen()
	index := l * TotalBits / TypeSize
	offset := l * TotalBits % TypeSize
	if index >= len(t.EncodedSteps) {
		t.EncodedSteps = append(t.EncodedSteps, (1<<64)-1)
	}
	if offset+TotalBits >= TypeSize && index+1 >= len(t.EncodedSteps) {
		t.EncodedSteps = append(t.EncodedSteps, (1<<64)-1)
	}
	t.SetEncodedStep(l, s)
}
<commit_msg>cleaned up the k-d tree<commit_after>package kdtree

import (
	"geo"
	"graph"
)

// Parameters for the encoding that is used for storing k-d trees space efficient
// Encoded vertex: vertex index + MaxEdgeOffset + MaxStepOffset
// Encoded step:   vertex index + edge offset   + step offset
const (
	VertexIndexBits = 18
	EdgeOffsetBits  = 5
	StepOffsetBits  = 11
	TotalBits       = VertexIndexBits + EdgeOffsetBits + StepOffsetBits
	TypeSize        = 64
	MaxVertexIndex  = (1 << VertexIndexBits) - 1
	MaxEdgeOffset   = (1 << EdgeOffsetBits) - 1
	MaxStepOffset   = (1 << StepOffsetBits) - 1
)

type KdTree struct {
	Graph        graph.Graph
	EncodedSteps []uint64
	Coordinates  []geo.Coordinate
	// It is inefficient to create a sub slice of EncodedSteps due to the used encoding.
	// Thus, we use start and end pointer instead.
	EncodedStepsStart int
	EncodedStepsEnd   int
}

type ClusterKdTree struct {
	Overlay *KdTree
	Cluster []*KdTree
	BBoxes  []geo.BBox
}

func (t *KdTree) Len() int {
	return len(t.Coordinates)
}

func (t *KdTree) Swap(i, j int) {
	t.Coordinates[i], t.Coordinates[j] = t.Coordinates[j], t.Coordinates[i]
	tmp := t.EncodedStep(j)
	t.SetEncodedStep(j, t.EncodedStep(i))
	t.SetEncodedStep(i, tmp)
}

func (t *KdTree) EncodedStepLen() int {
	if t.EncodedStepsEnd > 0 {
		return t.EncodedStepsEnd + 1
	}
	l := (len(t.EncodedSteps) * TypeSize) / TotalBits
	if l > 0 && t.EncodedStep(l-1) == (1<<TotalBits)-1 {
		return l - 1
	}
	return l
}

func (t *KdTree) EncodedStep(i int) uint64 {
	index := t.EncodedStepsStart + i*TotalBits/TypeSize
	offset := i * TotalBits % TypeSize
	if offset+TotalBits <= TypeSize {
		// contained in one uint64
		mask := (uint64(1) << TotalBits) - 1
		return (t.EncodedSteps[index] >> uint(offset)) & mask
	}
	// split over two uint64
	first := uint(TypeSize - offset)
	second := uint(TotalBits - first)

	fMask := ((uint64(1) << first) - 1)
	result := ((t.EncodedSteps[index] >> uint(offset)) & fMask) << second

	sMask := (uint64(1) << second) - 1
	result |= t.EncodedSteps[index+1] & sMask
	return result
}

func (t *KdTree) SetEncodedStep(i int, s uint64) {
	index := t.EncodedStepsStart + i*TotalBits/TypeSize
	offset := i * TotalBits % TypeSize
	if offset+TotalBits <= TypeSize {
		// contained in one uint64
		mask := (uint64(1) << TotalBits) - 1
		t.EncodedSteps[index] ^= t.EncodedSteps[index] & (mask << uint(offset))
		t.EncodedSteps[index] |= s << uint(offset)
	} else {
		// split over two uint64
		first := uint(TypeSize - offset)
		second := uint(TotalBits - first)

		fMask := (uint64(1) << first) - 1
		t.EncodedSteps[index] ^= t.EncodedSteps[index] & (fMask << uint(offset))
		t.EncodedSteps[index] |= (s >> second) << uint(offset)

		sMask := (uint64(1) << second) - 1
		t.EncodedSteps[index+1] ^= t.EncodedSteps[index+1] & sMask
		t.EncodedSteps[index+1] |= s & sMask
	}
}

func (t *KdTree) AppendEncodedStep(s uint64) {
	l := t.EncodedStepLen()
	index := l * TotalBits / TypeSize
	offset := l * TotalBits % TypeSize
	if index >= len(t.EncodedSteps) {
		t.EncodedSteps = append(t.EncodedSteps, (1<<64)-1)
	}
	if offset+TotalBits >= TypeSize && index+1 >= len(t.EncodedSteps) {
		t.EncodedSteps = append(t.EncodedSteps, (1<<64)-1)
	}
	t.SetEncodedStep(l, s)
}
<|endoftext|>