<commit_before>/*
   Copyright (c) 2016 VMware, Inc. All Rights Reserved.
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package replication

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/docker/distribution"
	"github.com/docker/distribution/manifest/schema1"
	"github.com/docker/distribution/manifest/schema2"
	"github.com/vmware/harbor/models"
	"github.com/vmware/harbor/utils/log"
	"github.com/vmware/harbor/utils/registry"
	"github.com/vmware/harbor/utils/registry/auth"
)

const (
	// StateCheck ...
	StateCheck = "check"
	// StatePullManifest ...
	StatePullManifest = "pull_manifest"
	// StateTransferBlob ...
	StateTransferBlob = "transfer_blob"
	// StatePushManifest ...
	StatePushManifest = "push_manifest"
)

// BaseHandler holds informations shared by other state handlers
type BaseHandler struct {
	project    string // project_name
	repository string // prject_name/repo_name
	tags       []string

	srcURL       string // url of source registry
	srcSecretKey string // secretKey ...

	dstURL string // url of target registry
	dstUsr string // username ...
	dstPwd string // password ...

	srcClient *registry.Repository
	dstClient *registry.Repository

	manifest distribution.Manifest // manifest of tags[0]
	blobs    []string              // blobs need to be transferred for tags[0]

	blobsExistence map[string]bool //key: digest of blob, value: existence

	logger *log.Logger
}

// InitBaseHandler initializes a BaseHandler: creating clients for source and destination registry,
// listing tags of the repository if parameter tags is nil.
func InitBaseHandler(repository, srcURL, srcSecretKey,
	dstURL, dstUsr, dstPwd string, tags []string, logger *log.Logger) (*BaseHandler, error) {

	logger.Infof("initializing: repository: %s, tags: %v, source URL: %s, destination URL: %s, destination user: %s",
		repository, tags, srcURL, dstURL, dstUsr)

	base := &BaseHandler{
		repository:     repository,
		tags:           tags,
		srcURL:         srcURL,
		srcSecretKey:   srcSecretKey,
		dstURL:         dstURL,
		dstUsr:         dstUsr,
		dstPwd:         dstPwd,
		blobsExistence: make(map[string]bool, 10),
		logger:         logger,
	}

	base.project = getProjectName(base.repository)

	//TODO using secret key
	srcCred := auth.NewBasicAuthCredential("admin", "Harbor12345")
	srcClient, err := registry.NewRepositoryWithCredential(base.repository, base.srcURL, srcCred)
	if err != nil {
		return nil, err
	}
	base.srcClient = srcClient

	dstCred := auth.NewBasicAuthCredential(base.dstUsr, base.dstPwd)
	dstClient, err := registry.NewRepositoryWithCredential(base.repository, base.dstURL, dstCred)
	if err != nil {
		return nil, err
	}
	base.dstClient = dstClient

	if len(base.tags) == 0 {
		tags, err := base.srcClient.ListTag()
		if err != nil {
			return nil, err
		}
		base.tags = tags
	}

	base.logger.Infof("initialization completed: project: %s, repository: %s, tags: %v, source URL: %s, destination URL: %s, destination user: %s",
		base.project, base.repository, base.tags, base.srcURL, base.dstURL, base.dstUsr)

	return base, nil
}

// Exit ...
func (b *BaseHandler) Exit() error {
	return nil
}

func getProjectName(repository string) string {
	repository = strings.TrimSpace(repository)
	repository = strings.TrimRight(repository, "/")
	return repository[:strings.LastIndex(repository, "/")]
}

// Checker checks the existence of project and the user's privlege to the project
type Checker struct {
	*BaseHandler
}

// Enter check existence of project, if it does not exist, create it,
// if it exists, check whether the user has write privilege to it.
func (c *Checker) Enter() (string, error) {
	exist, canWrite, err := c.projectExist()
	if err != nil {
		c.logger.Errorf("an error occurred while checking existence of project %s on %s with user %s : %v", c.project, c.dstURL, c.dstUsr, err)
		return "", err
	}
	if !exist {
		if err := c.createProject(); err != nil {
			c.logger.Errorf("an error occurred while creating project %s on %s with user %s : %v", c.project, c.dstURL, c.dstUsr, err)
			return "", err
		}
		c.logger.Infof("project %s is created on %s with user %s", c.project, c.dstURL, c.dstUsr)
		return StatePullManifest, nil
	}

	c.logger.Infof("project %s already exists on %s", c.project, c.dstURL)

	if !canWrite {
		err = fmt.Errorf("the user %s has no write privilege to project %s on %s", c.dstUsr, c.project, c.dstURL)
		c.logger.Errorf("%v", err)
		return "", err
	}
	c.logger.Infof("the user %s has write privilege to project %s on %s", c.dstUsr, c.project, c.dstURL)

	return StatePullManifest, nil
}

// check the existence of project, if it exists, returning whether the user has write privilege to it
func (c *Checker) projectExist() (exist, canWrite bool, err error) {
	url := strings.TrimRight(c.dstURL, "/") + "/api/projects/?project_name=" + c.project
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return
	}

	req.SetBasicAuth(c.dstUsr, c.dstPwd)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return
	}

	if resp.StatusCode == http.StatusNotFound {
		return
	}

	if resp.StatusCode == http.StatusUnauthorized {
		exist = true
		return
	}

	defer resp.Body.Close()
	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}

	if resp.StatusCode == http.StatusOK {
		projects := make([]models.Project, 1)
		if err = json.Unmarshal(data, &projects); err != nil {
			return
		}

		if len(projects) == 0 {
			return
		}

		exist = true
		// TODO handle canWrite when new API is ready
		canWrite = true

		return
	}

	err = fmt.Errorf("an error occurred while checking existen of project %s on %s with user %s: %d %s",
		c.project, c.dstURL, c.dstUsr, resp.StatusCode, string(data))

	return
}

func (c *Checker) createProject() error {
	// TODO handle publicity of project
	project := struct {
		ProjectName string `json:"project_name"`
		Public      bool   `json:"public"`
	}{
		ProjectName: c.project,
	}

	data, err := json.Marshal(project)
	if err != nil {
		return err
	}

	url := strings.TrimRight(c.dstURL, "/") + "/api/projects/"
	req, err := http.NewRequest("POST", url, bytes.NewReader(data))
	if err != nil {
		return err
	}

	req.SetBasicAuth(c.dstUsr, c.dstPwd)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}

	if resp.StatusCode != http.StatusCreated {
		defer resp.Body.Close()
		message, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			c.logger.Errorf("an error occurred while reading message from response: %v", err)
		}

		return fmt.Errorf("failed to create project %s on %s with user %s: %d %s",
			c.project, c.dstURL, c.dstUsr, resp.StatusCode, string(message))
	}

	return nil
}

// ManifestPuller pulls the manifest of a tag. And if no tag needs to be pulled,
// the next state that state machine should enter is "finished".
type ManifestPuller struct {
	*BaseHandler
}

// Enter pulls manifest of a tag and checks if all blobs exist in the destination registry
func (m *ManifestPuller) Enter() (string, error) {
	if len(m.tags) == 0 {
		m.logger.Infof("no tag needs to be replicated, next state is \"finished\"")
		return models.JobFinished, nil
	}

	name := m.repository
	tag := m.tags[0]

	acceptMediaTypes := []string{schema1.MediaTypeManifest, schema2.MediaTypeManifest}
	digest, mediaType, payload, err := m.srcClient.PullManifest(tag, acceptMediaTypes)
	if err != nil {
		m.logger.Errorf("an error occurred while pulling manifest of %s:%s from %s: %v", name, tag, m.srcURL, err)
		return "", err
	}
	m.logger.Infof("manifest of %s:%s pulled successfully from %s: %s", name, tag, m.srcURL, digest)

	if strings.Contains(mediaType, "application/json") {
		mediaType = schema1.MediaTypeManifest
	}

	manifest, _, err := registry.UnMarshal(mediaType, payload)
	if err != nil {
		m.logger.Errorf("an error occurred while parsing manifest of %s:%s from %s: %v", name, tag, m.srcURL, err)
		return "", err
	}

	m.manifest = manifest

	// all blobs(layers and config)
	var blobs []string

	for _, discriptor := range manifest.References() {
		blobs = append(blobs, discriptor.Digest.String())
	}

	// config is also need to be transferred if the schema of manifest is v2
	manifest2, ok := manifest.(*schema2.DeserializedManifest)
	if ok {
		blobs = append(blobs, manifest2.Target().Digest.String())
	}

	m.logger.Infof("all blobs of %s:%s from %s: %v", name, tag, m.srcURL, blobs)

	for _, blob := range blobs {
		exist, ok := m.blobsExistence[blob]
		if !ok {
			exist, err = m.dstClient.BlobExist(blob)
			if err != nil {
				m.logger.Errorf("an error occurred while checking existence of blob %s of %s:%s on %s: %v", blob, name, tag, m.dstURL, err)
				return "", err
			}
			m.blobsExistence[blob] = exist
		}

		if !exist {
			m.blobs = append(m.blobs, blob)
		} else {
			m.logger.Infof("blob %s of %s:%s already exists in %s", blob, name, tag, m.dstURL)
		}
	}
	m.logger.Infof("blobs of %s:%s need to be transferred to %s: %v", name, tag, m.dstURL, m.blobs)

	return StateTransferBlob, nil
}

// BlobTransfer transfers blobs of a tag
type BlobTransfer struct {
	*BaseHandler
}

// Enter pulls blobs and then pushs them to destination registry.
func (b *BlobTransfer) Enter() (string, error) {
	name := b.repository
	tag := b.tags[0]
	for _, blob := range b.blobs {
		b.logger.Infof("transferring blob %s of %s:%s to %s ...", blob, name, tag, b.dstURL)
		size, data, err := b.srcClient.PullBlob(blob)
		if err != nil {
			b.logger.Errorf("an error occurred while pulling blob %s of %s:%s from %s: %v", blob, name, tag, b.srcURL, err)
			return "", err
		}
		if err = b.dstClient.PushBlob(blob, size, data); err != nil {
			b.logger.Errorf("an error occurred while pushing blob %s of %s:%s to %s : %v", blob, name, tag, b.dstURL, err)
			return "", err
		}
		b.logger.Infof("blob %s of %s:%s transferred to %s completed", blob, name, tag, b.dstURL)
	}

	return StatePushManifest, nil
}

// ManifestPusher pushs the manifest to destination registry
type ManifestPusher struct {
	*BaseHandler
}

// Enter checks the existence of manifest in the source registry first, and if it
// exists, pushs it to destination registry. The checking operation is to avoid
// the situation that the tag is deleted during the blobs transfering
func (m *ManifestPusher) Enter() (string, error) {
	name := m.repository
	tag := m.tags[0]
	_, exist, err := m.srcClient.ManifestExist(tag)
	if err != nil {
		m.logger.Infof("an error occurred while checking the existence of manifest of %s:%s on %s: %v", name, tag, m.srcURL, err)
		return "", err
	}
	if !exist {
		m.logger.Infof("manifest of %s:%s does not exist on source registry %s, cancel manifest pushing", name, tag, m.srcURL)
	} else {
		m.logger.Infof("manifest of %s:%s exists on source registry %s, continue manifest pushing", name, tag, m.srcURL)
		mediaType, data, err := m.manifest.Payload()
		if err != nil {
			m.logger.Errorf("an error occurred while getting payload of manifest for %s:%s : %v", name, tag, err)
			return "", err
		}

		if _, err = m.dstClient.PushManifest(tag, mediaType, data); err != nil {
			m.logger.Errorf("an error occurred while pushing manifest of %s:%s to %s : %v", name, tag, m.dstURL, err)
			return "", err
		}
		m.logger.Infof("manifest of %s:%s has been pushed to %s", name, tag, m.dstURL)
	}

	m.tags = m.tags[1:]
	m.manifest = nil
	m.blobs = nil

	return StatePullManifest, nil
}
<commit_msg>integrated with new API of project<commit_after>/*
   Copyright (c) 2016 VMware, Inc. All Rights Reserved.
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package replication

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/docker/distribution"
	"github.com/docker/distribution/manifest/schema1"
	"github.com/docker/distribution/manifest/schema2"
	"github.com/vmware/harbor/models"
	"github.com/vmware/harbor/utils/log"
	"github.com/vmware/harbor/utils/registry"
	"github.com/vmware/harbor/utils/registry/auth"
)

const (
	// StateCheck ...
	StateCheck = "check"
	// StatePullManifest ...
	StatePullManifest = "pull_manifest"
	// StateTransferBlob ...
	StateTransferBlob = "transfer_blob"
	// StatePushManifest ...
	StatePushManifest = "push_manifest"
)

// BaseHandler holds informations shared by other state handlers
type BaseHandler struct {
	project    string // project_name
	repository string // prject_name/repo_name
	tags       []string

	srcURL       string // url of source registry
	srcSecretKey string // secretKey ...

	dstURL string // url of target registry
	dstUsr string // username ...
	dstPwd string // password ...

	srcClient *registry.Repository
	dstClient *registry.Repository

	manifest distribution.Manifest // manifest of tags[0]
	blobs    []string              // blobs need to be transferred for tags[0]

	blobsExistence map[string]bool //key: digest of blob, value: existence

	logger *log.Logger
}

// InitBaseHandler initializes a BaseHandler: creating clients for source and destination registry,
// listing tags of the repository if parameter tags is nil.
func InitBaseHandler(repository, srcURL, srcSecretKey,
	dstURL, dstUsr, dstPwd string, tags []string, logger *log.Logger) (*BaseHandler, error) {

	logger.Infof("initializing: repository: %s, tags: %v, source URL: %s, destination URL: %s, destination user: %s",
		repository, tags, srcURL, dstURL, dstUsr)

	base := &BaseHandler{
		repository:     repository,
		tags:           tags,
		srcURL:         srcURL,
		srcSecretKey:   srcSecretKey,
		dstURL:         dstURL,
		dstUsr:         dstUsr,
		dstPwd:         dstPwd,
		blobsExistence: make(map[string]bool, 10),
		logger:         logger,
	}

	base.project = getProjectName(base.repository)

	//TODO using secret key
	srcCred := auth.NewBasicAuthCredential("admin", "Harbor12345")
	srcClient, err := registry.NewRepositoryWithCredential(base.repository, base.srcURL, srcCred)
	if err != nil {
		return nil, err
	}
	base.srcClient = srcClient

	dstCred := auth.NewBasicAuthCredential(base.dstUsr, base.dstPwd)
	dstClient, err := registry.NewRepositoryWithCredential(base.repository, base.dstURL, dstCred)
	if err != nil {
		return nil, err
	}
	base.dstClient = dstClient

	if len(base.tags) == 0 {
		tags, err := base.srcClient.ListTag()
		if err != nil {
			return nil, err
		}
		base.tags = tags
	}

	base.logger.Infof("initialization completed: project: %s, repository: %s, tags: %v, source URL: %s, destination URL: %s, destination user: %s",
		base.project, base.repository, base.tags, base.srcURL, base.dstURL, base.dstUsr)

	return base, nil
}

// Exit ...
func (b *BaseHandler) Exit() error {
	return nil
}

func getProjectName(repository string) string {
	repository = strings.TrimSpace(repository)
	repository = strings.TrimRight(repository, "/")
	return repository[:strings.LastIndex(repository, "/")]
}

// Checker checks the existence of project and the user's privlege to the project
type Checker struct {
	*BaseHandler
}

// Enter check existence of project, if it does not exist, create it,
// if it exists, check whether the user has write privilege to it.
func (c *Checker) Enter() (string, error) {
	exist, canWrite, err := c.projectExist()
	if err != nil {
		c.logger.Errorf("an error occurred while checking existence of project %s on %s with user %s : %v", c.project, c.dstURL, c.dstUsr, err)
		return "", err
	}
	if !exist {
		if err := c.createProject(); err != nil {
			c.logger.Errorf("an error occurred while creating project %s on %s with user %s : %v", c.project, c.dstURL, c.dstUsr, err)
			return "", err
		}
		c.logger.Infof("project %s is created on %s with user %s", c.project, c.dstURL, c.dstUsr)
		return StatePullManifest, nil
	}

	c.logger.Infof("project %s already exists on %s", c.project, c.dstURL)

	if !canWrite {
		err = fmt.Errorf("the user %s has no write privilege to project %s on %s", c.dstUsr, c.project, c.dstURL)
		c.logger.Errorf("%v", err)
		return "", err
	}
	c.logger.Infof("the user %s has write privilege to project %s on %s", c.dstUsr, c.project, c.dstURL)

	return StatePullManifest, nil
}

// check the existence of project, if it exists, returning whether the user has write privilege to it
func (c *Checker) projectExist() (exist, canWrite bool, err error) {
	url := strings.TrimRight(c.dstURL, "/") + "/api/projects/?project_name=" + c.project
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return
	}

	req.SetBasicAuth(c.dstUsr, c.dstPwd)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return
	}

	if resp.StatusCode == http.StatusNotFound {
		return
	}

	if resp.StatusCode == http.StatusUnauthorized {
		exist = true
		return
	}

	defer resp.Body.Close()
	data, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}

	if resp.StatusCode == http.StatusOK {
		projects := make([]models.Project, 1)
		if err = json.Unmarshal(data, &projects); err != nil {
			return
		}

		if len(projects) == 0 {
			return
		}

		for _, project := range projects {
			if project.Name == c.project {
				exist = true
				canWrite = (project.Role == models.PROJECTADMIN ||
					project.Role == models.DEVELOPER)
				break
			}
		}

		return
	}

	err = fmt.Errorf("an error occurred while checking existen of project %s on %s with user %s: %d %s",
		c.project, c.dstURL, c.dstUsr, resp.StatusCode, string(data))

	return
}

func (c *Checker) createProject() error {
	// TODO handle publicity of project
	project := struct {
		ProjectName string `json:"project_name"`
		Public      bool   `json:"public"`
	}{
		ProjectName: c.project,
	}

	data, err := json.Marshal(project)
	if err != nil {
		return err
	}

	url := strings.TrimRight(c.dstURL, "/") + "/api/projects/"
	req, err := http.NewRequest("POST", url, bytes.NewReader(data))
	if err != nil {
		return err
	}

	req.SetBasicAuth(c.dstUsr, c.dstPwd)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}

	if resp.StatusCode != http.StatusCreated {
		defer resp.Body.Close()
		message, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			c.logger.Errorf("an error occurred while reading message from response: %v", err)
		}

		return fmt.Errorf("failed to create project %s on %s with user %s: %d %s",
			c.project, c.dstURL, c.dstUsr, resp.StatusCode, string(message))
	}

	return nil
}

// ManifestPuller pulls the manifest of a tag. And if no tag needs to be pulled,
// the next state that state machine should enter is "finished".
type ManifestPuller struct {
	*BaseHandler
}

// Enter pulls manifest of a tag and checks if all blobs exist in the destination registry
func (m *ManifestPuller) Enter() (string, error) {
	if len(m.tags) == 0 {
		m.logger.Infof("no tag needs to be replicated, next state is \"finished\"")
		return models.JobFinished, nil
	}

	name := m.repository
	tag := m.tags[0]

	acceptMediaTypes := []string{schema1.MediaTypeManifest, schema2.MediaTypeManifest}
	digest, mediaType, payload, err := m.srcClient.PullManifest(tag, acceptMediaTypes)
	if err != nil {
		m.logger.Errorf("an error occurred while pulling manifest of %s:%s from %s: %v", name, tag, m.srcURL, err)
		return "", err
	}
	m.logger.Infof("manifest of %s:%s pulled successfully from %s: %s", name, tag, m.srcURL, digest)

	if strings.Contains(mediaType, "application/json") {
		mediaType = schema1.MediaTypeManifest
	}

	manifest, _, err := registry.UnMarshal(mediaType, payload)
	if err != nil {
		m.logger.Errorf("an error occurred while parsing manifest of %s:%s from %s: %v", name, tag, m.srcURL, err)
		return "", err
	}

	m.manifest = manifest

	// all blobs(layers and config)
	var blobs []string

	for _, discriptor := range manifest.References() {
		blobs = append(blobs, discriptor.Digest.String())
	}

	// config is also need to be transferred if the schema of manifest is v2
	manifest2, ok := manifest.(*schema2.DeserializedManifest)
	if ok {
		blobs = append(blobs, manifest2.Target().Digest.String())
	}

	m.logger.Infof("all blobs of %s:%s from %s: %v", name, tag, m.srcURL, blobs)

	for _, blob := range blobs {
		exist, ok := m.blobsExistence[blob]
		if !ok {
			exist, err = m.dstClient.BlobExist(blob)
			if err != nil {
				m.logger.Errorf("an error occurred while checking existence of blob %s of %s:%s on %s: %v", blob, name, tag, m.dstURL, err)
				return "", err
			}
			m.blobsExistence[blob] = exist
		}

		if !exist {
			m.blobs = append(m.blobs, blob)
		} else {
			m.logger.Infof("blob %s of %s:%s already exists in %s", blob, name, tag, m.dstURL)
		}
	}
	m.logger.Infof("blobs of %s:%s need to be transferred to %s: %v", name, tag, m.dstURL, m.blobs)

	return StateTransferBlob, nil
}

// BlobTransfer transfers blobs of a tag
type BlobTransfer struct {
	*BaseHandler
}

// Enter pulls blobs and then pushs them to destination registry.
func (b *BlobTransfer) Enter() (string, error) {
	name := b.repository
	tag := b.tags[0]
	for _, blob := range b.blobs {
		b.logger.Infof("transferring blob %s of %s:%s to %s ...", blob, name, tag, b.dstURL)
		size, data, err := b.srcClient.PullBlob(blob)
		if err != nil {
			b.logger.Errorf("an error occurred while pulling blob %s of %s:%s from %s: %v", blob, name, tag, b.srcURL, err)
			return "", err
		}
		if err = b.dstClient.PushBlob(blob, size, data); err != nil {
			b.logger.Errorf("an error occurred while pushing blob %s of %s:%s to %s : %v", blob, name, tag, b.dstURL, err)
			return "", err
		}
		b.logger.Infof("blob %s of %s:%s transferred to %s completed", blob, name, tag, b.dstURL)
	}

	return StatePushManifest, nil
}

// ManifestPusher pushs the manifest to destination registry
type ManifestPusher struct {
	*BaseHandler
}

// Enter checks the existence of manifest in the source registry first, and if it
// exists, pushs it to destination registry. The checking operation is to avoid
// the situation that the tag is deleted during the blobs transfering
func (m *ManifestPusher) Enter() (string, error) {
	name := m.repository
	tag := m.tags[0]
	_, exist, err := m.srcClient.ManifestExist(tag)
	if err != nil {
		m.logger.Infof("an error occurred while checking the existence of manifest of %s:%s on %s: %v", name, tag, m.srcURL, err)
		return "", err
	}
	if !exist {
		m.logger.Infof("manifest of %s:%s does not exist on source registry %s, cancel manifest pushing", name, tag, m.srcURL)
	} else {
		m.logger.Infof("manifest of %s:%s exists on source registry %s, continue manifest pushing", name, tag, m.srcURL)
		mediaType, data, err := m.manifest.Payload()
		if err != nil {
			m.logger.Errorf("an error occurred while getting payload of manifest for %s:%s : %v", name, tag, err)
			return "", err
		}

		if _, err = m.dstClient.PushManifest(tag, mediaType, data); err != nil {
			m.logger.Errorf("an error occurred while pushing manifest of %s:%s to %s : %v", name, tag, m.dstURL, err)
			return "", err
		}
		m.logger.Infof("manifest of %s:%s has been pushed to %s", name, tag, m.dstURL)
	}

	m.tags = m.tags[1:]
	m.manifest = nil
	m.blobs = nil

	return StatePullManifest, nil
}
<|endoftext|>


<commit_before>package qmp

import (
	"encoding/json"
	"fmt"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/digitalocean/go-qemu/qmp"

	"github.com/lxc/lxd/shared"
)

var monitors = map[string]*Monitor{}
var monitorsLock sync.Mutex

// RingbufSize is the size of the agent serial ringbuffer in bytes
var RingbufSize = 16

// Monitor represents a QMP monitor.
type Monitor struct {
	path string
	qmp  *qmp.SocketMonitor

	agentReady   bool
	disconnected bool
	chDisconnect chan struct{}
	eventHandler func(name string, data map[string]interface{})
}

// Connect creates or retrieves an existing QMP monitor for the path.
func Connect(path string, eventHandler func(name string, data map[string]interface{})) (*Monitor, error) {
	monitorsLock.Lock()
	defer monitorsLock.Unlock()

	// Look for an existing monitor.
	monitor, ok := monitors[path]
	if ok {
		monitor.eventHandler = eventHandler
		return monitor, nil
	}

	// Setup the connection.
	qmpConn, err := qmp.NewSocketMonitor("unix", path, time.Second)
	if err != nil {
		return nil, err
	}

	err = qmpConn.Connect()
	if err != nil {
		return nil, err
	}

	// Setup the monitor struct.
	monitor = &Monitor{}
	monitor.path = path
	monitor.qmp = qmpConn
	monitor.chDisconnect = make(chan struct{}, 1)
	monitor.eventHandler = eventHandler

	// Spawn goroutines.
	err = monitor.run()
	if err != nil {
		return nil, err
	}

	// Register in global map.
	monitors[path] = monitor

	return monitor, nil
}

func (m *Monitor) run() error {
	// Start ringbuffer monitoring go routine.
	go func() {
		for {
			// Read the ringbuffer.
			resp, err := m.qmp.Run([]byte(fmt.Sprintf(`{"execute": "ringbuf-read", "arguments": {"device": "qemu_serial-chardev", "size": %d, "format": "utf8"}}`, RingbufSize)))
			if err != nil {
				m.Disconnect()
				return
			}

			// Decode the response.
			var respDecoded struct {
				Return string `json:"return"`
			}

			err = json.Unmarshal(resp, &respDecoded)
			if err != nil {
				continue
			}

			// Extract the last entry.
			entries := strings.Split(respDecoded.Return, "\n")
			if len(entries) > 1 {
				status := entries[len(entries)-2]

				if status == "STARTED" {
					m.agentReady = true
				} else if status == "STOPPED" {
					m.agentReady = false
				}
			}

			// Wait until next read or cancel.
			select {
			case <-m.chDisconnect:
				return
			case <-time.After(10 * time.Second):
				continue
			}
		}
	}()

	// Start event monitoring go routine.
	chEvents, err := m.qmp.Events()
	if err != nil {
		return err
	}

	go func() {
		for {
			select {
			case <-m.chDisconnect:
				return
			case e := <-chEvents:
				if e.Event == "" {
					continue
				}

				if m.eventHandler != nil {
					m.eventHandler(e.Event, e.Data)
				}
			}
		}
	}()

	return nil
}

// Wait returns a channel that will be closed on disconnection.
func (m *Monitor) Wait() (chan struct{}, error) {
	// Check if disconnected
	if m.disconnected {
		return nil, ErrMonitorDisconnect
	}

	return m.chDisconnect, nil
}

// Disconnect forces a disconnection from QEMU.
func (m *Monitor) Disconnect() {
	// Stop all go routines and disconnect from socket.
	if !m.disconnected {
		close(m.chDisconnect)
	}
	m.disconnected = true
	m.qmp.Disconnect()

	// Remove from the map.
	monitorsLock.Lock()
	defer monitorsLock.Unlock()
	delete(monitors, m.path)
}

// Status returns the current VM status.
func (m *Monitor) Status() (string, error) {
	// Check if disconnected
	if m.disconnected {
		return "", ErrMonitorDisconnect
	}

	// Query the status.
	respRaw, err := m.qmp.Run([]byte("{'execute': 'query-status'}"))
	if err != nil {
		m.Disconnect()
		return "", ErrMonitorDisconnect
	}

	// Process the response.
	var respDecoded struct {
		Return struct {
			Status string `json:"status"`
		} `json:"return"`
	}

	err = json.Unmarshal(respRaw, &respDecoded)
	if err != nil {
		return "", ErrMonitorBadReturn
	}

	return respDecoded.Return.Status, nil
}

// Console fetches the File for a particular console.
func (m *Monitor) Console(target string) (*os.File, error) {
	// Check if disconnected
	if m.disconnected {
		return nil, ErrMonitorDisconnect
	}

	// Query the consoles.
	respRaw, err := m.qmp.Run([]byte("{'execute': 'query-chardev'}"))
	if err != nil {
		m.Disconnect()
		return nil, ErrMonitorDisconnect
	}

	// Process the response.
	var respDecoded struct {
		Return []struct {
			Label    string `json:"label"`
			Filename string `json:"filename"`
		} `json:"return"`
	}

	err = json.Unmarshal(respRaw, &respDecoded)
	if err != nil {
		return nil, ErrMonitorBadReturn
	}

	// Look for the requested console.
	for _, v := range respDecoded.Return {
		if v.Label == target {
			ptsPath := strings.TrimPrefix(v.Filename, "pty:")

			if !shared.PathExists(ptsPath) {
				continue
			}

			// Open the PTS device
			console, err := os.OpenFile(ptsPath, os.O_RDWR, 0600)
			if err != nil {
				return nil, err
			}

			return console, nil
		}
	}

	return nil, ErrMonitorBadConsole
}

func (m *Monitor) runCmd(cmd string) error {
	// Check if disconnected
	if m.disconnected {
		return ErrMonitorDisconnect
	}

	// Query the status.
	_, err := m.qmp.Run([]byte(fmt.Sprintf("{'execute': '%s'}", cmd)))
	if err != nil {
		m.Disconnect()
		return ErrMonitorDisconnect
	}

	return nil
}

// Powerdown tells the VM to gracefully shutdown.
func (m *Monitor) Powerdown() error {
	return m.runCmd("system_powerdown")
}

// Start tells QEMU to start the emulation.
func (m *Monitor) Start() error {
	return m.runCmd("cont")
}

// Pause tells QEMU to temporarily stop the emulation.
func (m *Monitor) Pause() error {
	return m.runCmd("stop")
}

// Quit tells QEMU to exit immediately.
func (m *Monitor) Quit() error {
	return m.runCmd("quit")
}

// AgentReady indicates whether an agent has been detected.
func (m *Monitor) AgentReady() bool {
	return m.agentReady
}

// GetCPUs fetches the vCPU information for pinning.
func (m *Monitor) GetCPUs() ([]int, error) {
	// Check if disconnected
	if m.disconnected {
		return nil, ErrMonitorDisconnect
	}

	// Query the consoles.
	respRaw, err := m.qmp.Run([]byte("{'execute': 'query-cpus'}"))
	if err != nil {
		m.Disconnect()
		return nil, ErrMonitorDisconnect
	}

	// Process the response.
	var respDecoded struct {
		Return []struct {
			CPU int `json:"CPU"`
			PID int `json:"thread_id"`
		} `json:"return"`
	}

	err = json.Unmarshal(respRaw, &respDecoded)
	if err != nil {
		return nil, ErrMonitorBadReturn
	}

	// Make a slice of PIDs.
	pids := []int{}
	for _, cpu := range respDecoded.Return {
		pids = append(pids, cpu.PID)
	}

	return pids, nil
}
<commit_msg>lxd/instance/drivers/qmp/monitor: Allow serial char device name to be passed in<commit_after>package qmp

import (
	"encoding/json"
	"fmt"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/digitalocean/go-qemu/qmp"

	"github.com/lxc/lxd/shared"
)

var monitors = map[string]*Monitor{}
var monitorsLock sync.Mutex

// RingbufSize is the size of the agent serial ringbuffer in bytes
var RingbufSize = 16

// Monitor represents a QMP monitor.
type Monitor struct {
	path string
	qmp  *qmp.SocketMonitor

	agentReady    bool
	disconnected  bool
	chDisconnect  chan struct{}
	eventHandler  func(name string, data map[string]interface{})
	serialCharDev string
}

// Connect creates or retrieves an existing QMP monitor for the path.
func Connect(path string, serialCharDev string, eventHandler func(name string, data map[string]interface{})) (*Monitor, error) {
	monitorsLock.Lock()
	defer monitorsLock.Unlock()

	// Look for an existing monitor.
	monitor, ok := monitors[path]
	if ok {
		monitor.eventHandler = eventHandler
		return monitor, nil
	}

	// Setup the connection.
	qmpConn, err := qmp.NewSocketMonitor("unix", path, time.Second)
	if err != nil {
		return nil, err
	}

	err = qmpConn.Connect()
	if err != nil {
		return nil, err
	}

	// Setup the monitor struct.
	monitor = &Monitor{}
	monitor.path = path
	monitor.qmp = qmpConn
	monitor.chDisconnect = make(chan struct{}, 1)
	monitor.eventHandler = eventHandler
	monitor.serialCharDev = serialCharDev

	// Spawn goroutines.
	err = monitor.run()
	if err != nil {
		return nil, err
	}

	// Register in global map.
	monitors[path] = monitor

	return monitor, nil
}

func (m *Monitor) run() error {
	// Start ringbuffer monitoring go routine.
	go func() {
		for {
			// Read the ringbuffer.
			resp, err := m.qmp.Run([]byte(fmt.Sprintf(`{"execute": "ringbuf-read", "arguments": {"device": "%s", "size": %d, "format": "utf8"}}`, m.serialCharDev, RingbufSize)))
			if err != nil {
				m.Disconnect()
				return
			}

			// Decode the response.
			var respDecoded struct {
				Return string `json:"return"`
			}

			err = json.Unmarshal(resp, &respDecoded)
			if err != nil {
				continue
			}

			// Extract the last entry.
			entries := strings.Split(respDecoded.Return, "\n")
			if len(entries) > 1 {
				status := entries[len(entries)-2]

				if status == "STARTED" {
					m.agentReady = true
				} else if status == "STOPPED" {
					m.agentReady = false
				}
			}

			// Wait until next read or cancel.
			select {
			case <-m.chDisconnect:
				return
			case <-time.After(10 * time.Second):
				continue
			}
		}
	}()

	// Start event monitoring go routine.
	chEvents, err := m.qmp.Events()
	if err != nil {
		return err
	}

	go func() {
		for {
			select {
			case <-m.chDisconnect:
				return
			case e := <-chEvents:
				if e.Event == "" {
					continue
				}

				if m.eventHandler != nil {
					m.eventHandler(e.Event, e.Data)
				}
			}
		}
	}()

	return nil
}

// Wait returns a channel that will be closed on disconnection.
func (m *Monitor) Wait() (chan struct{}, error) {
	// Check if disconnected
	if m.disconnected {
		return nil, ErrMonitorDisconnect
	}

	return m.chDisconnect, nil
}

// Disconnect forces a disconnection from QEMU.
func (m *Monitor) Disconnect() {
	// Stop all go routines and disconnect from socket.
	if !m.disconnected {
		close(m.chDisconnect)
	}
	m.disconnected = true
	m.qmp.Disconnect()

	// Remove from the map.
	monitorsLock.Lock()
	defer monitorsLock.Unlock()
	delete(monitors, m.path)
}

// Status returns the current VM status.
func (m *Monitor) Status() (string, error) {
	// Check if disconnected
	if m.disconnected {
		return "", ErrMonitorDisconnect
	}

	// Query the status.
	respRaw, err := m.qmp.Run([]byte("{'execute': 'query-status'}"))
	if err != nil {
		m.Disconnect()
		return "", ErrMonitorDisconnect
	}

	// Process the response.
	var respDecoded struct {
		Return struct {
			Status string `json:"status"`
		} `json:"return"`
	}

	err = json.Unmarshal(respRaw, &respDecoded)
	if err != nil {
		return "", ErrMonitorBadReturn
	}

	return respDecoded.Return.Status, nil
}

// Console fetches the File for a particular console.
func (m *Monitor) Console(target string) (*os.File, error) {
	// Check if disconnected
	if m.disconnected {
		return nil, ErrMonitorDisconnect
	}

	// Query the consoles.
	respRaw, err := m.qmp.Run([]byte("{'execute': 'query-chardev'}"))
	if err != nil {
		m.Disconnect()
		return nil, ErrMonitorDisconnect
	}

	// Process the response.
	var respDecoded struct {
		Return []struct {
			Label    string `json:"label"`
			Filename string `json:"filename"`
		} `json:"return"`
	}

	err = json.Unmarshal(respRaw, &respDecoded)
	if err != nil {
		return nil, ErrMonitorBadReturn
	}

	// Look for the requested console.
	for _, v := range respDecoded.Return {
		if v.Label == target {
			ptsPath := strings.TrimPrefix(v.Filename, "pty:")

			if !shared.PathExists(ptsPath) {
				continue
			}

			// Open the PTS device
			console, err := os.OpenFile(ptsPath, os.O_RDWR, 0600)
			if err != nil {
				return nil, err
			}

			return console, nil
		}
	}

	return nil, ErrMonitorBadConsole
}

func (m *Monitor) runCmd(cmd string) error {
	// Check if disconnected
	if m.disconnected {
		return ErrMonitorDisconnect
	}

	// Query the status.
	_, err := m.qmp.Run([]byte(fmt.Sprintf("{'execute': '%s'}", cmd)))
	if err != nil {
		m.Disconnect()
		return ErrMonitorDisconnect
	}

	return nil
}

// Powerdown tells the VM to gracefully shutdown.
func (m *Monitor) Powerdown() error {
	return m.runCmd("system_powerdown")
}

// Start tells QEMU to start the emulation.
func (m *Monitor) Start() error {
	return m.runCmd("cont")
}

// Pause tells QEMU to temporarily stop the emulation.
func (m *Monitor) Pause() error {
	return m.runCmd("stop")
}

// Quit tells QEMU to exit immediately.
func (m *Monitor) Quit() error {
	return m.runCmd("quit")
}

// AgentReady indicates whether an agent has been detected.
func (m *Monitor) AgentReady() bool {
	return m.agentReady
}

// GetCPUs fetches the vCPU information for pinning.
func (m *Monitor) GetCPUs() ([]int, error) {
	// Check if disconnected
	if m.disconnected {
		return nil, ErrMonitorDisconnect
	}

	// Query the consoles.
	respRaw, err := m.qmp.Run([]byte("{'execute': 'query-cpus'}"))
	if err != nil {
		m.Disconnect()
		return nil, ErrMonitorDisconnect
	}

	// Process the response.
	var respDecoded struct {
		Return []struct {
			CPU int `json:"CPU"`
			PID int `json:"thread_id"`
		} `json:"return"`
	}

	err = json.Unmarshal(respRaw, &respDecoded)
	if err != nil {
		return nil, ErrMonitorBadReturn
	}

	// Make a slice of PIDs.
	pids := []int{}
	for _, cpu := range respDecoded.Return {
		pids = append(pids, cpu.PID)
	}

	return pids, nil
}
<|endoftext|>


<commit_before>package raft

import (
	"code.google.com/p/go.net/context"
	"github.com/coreos/etcd/wait"
)

type SyncNode struct {
	n *Node
	w wait.WaitList
}

func NewSyncNode(n *Node) *SyncNode { panic("not implemented") }

type waitResp struct {
	e   Entry
	err error
}

func (n *SyncNode) Propose(ctx context.Context, id int64, data []byte) (Entry, error) {
	ch := n.w.Register(id)
	n.n.Propose(id, data)
	select {
	case x := <-ch:
		wr := x.(waitResp)
		return wr.e, wr.err
	case <-ctx.Done():
		n.w.Trigger(id, nil) // GC the Wait
		return Entry{}, ctx.Err()
	}
}

func (n *SyncNode) ReadState() (State, []Entry, []Message, error) {
	st, ents, msgs, err := n.n.ReadState()
	for _, e := range ents {
		if e.Index >= st.CommitIndex {
			n.w.Trigger(e.Id, waitResp{e: e, err: nil})
		}
	}
	return st, ents, msgs, err
}
<commit_msg>raft: remove sync<commit_after><|endoftext|>


<commit_before>package main

import (
	"strings"
)

// repeatedString is a string flag that can be specified multiple times.
type repeatedString []string

func (s repeatedString) String() string {
	return strings.Join(s, ",")
}

func (s *repeatedString) Set(v string) error {
	*s = append(*s, v)
	return nil
}

type funcFlag func(string) error

func (f funcFlag) Set(s string) error { return f(s) }
func (f funcFlag) String() string     { return "" }
func (f funcFlag) Type() string       { return "function" } // Purpose of this method unclear.
<commit_msg>Remove unused repeatedString type.<commit_after>package main

type funcFlag func(string) error

func (f funcFlag) Set(s string) error { return f(s) }
func (f funcFlag) String() string     { return "" }
func (f funcFlag) Type() string       { return "function" } // Purpose of this method unclear.
<|endoftext|>


<commit_before>package main

import (
	"context"
	"os"
	"os/signal"

	"github.com/anaminus/cobra"
	"github.com/anaminus/rbxmk/rbxmk/term"
	terminal "golang.org/x/term"
)

var Frag = NewFragments(initFragRoot())

// FragContent renders in HTML without the outer section or body.
func FragContent(fragref string) string {
	return Frag.ResolveWith(fragref, FragOptions{Inner: true})
}

var docState = NewDocState(Frag)

func Doc(fragref string) string     { return docState.Doc(fragref) }
func DocFlag(fragref string) string { return docState.DocFlag(fragref) }
func DocFragments() []string        { return docState.DocFragments() }
func UnresolvedFragments()          { docState.UnresolvedFragments() }

var Program = &cobra.Command{
	Use:           "rbxmk",
	Short:         Doc("Commands:Summary"),
	Long:          Doc("Commands:Summary"),
	SilenceUsage:  false,
	SilenceErrors: true,
}

func init() {
	fragTmplFuncs["frag"] = FragContent
	fragTmplFuncs["fraglist"] = Frag.List

	cobra.AddTemplateFunc("frag", func(fragref string) string {
		return Frag.ResolveWith(fragref, FragOptions{
			Renderer: term.Renderer{Width: -1}.Render,
			Inner:    true,
		})
	})
	cobra.AddTemplateFunc("width", func() int {
		width, _, _ := terminal.GetSize(int(os.Stdout.Fd()))
		return width
	})
	Program.SetUsageTemplate(usageTemplate)
}

func main() {
	ctx, stop := signal.NotifyContext(context.Background(), os.Kill)
	defer stop()

	Program.SetIn(os.Stdin)
	Program.SetOut(os.Stdout)
	Program.SetErr(os.Stderr)

	DocumentCommands()
	UnresolvedFragments()
	if err := Program.ExecuteContext(ctx); err != nil {
		Program.PrintErrln(err)
	}
}
<commit_msg>Add template function for expanding variables.<commit_after>package main

import (
	"context"
	"os"
	"os/signal"

	"github.com/anaminus/cobra"
	"github.com/anaminus/rbxmk/rbxmk/term"
	terminal "golang.org/x/term"
)

var Frag = NewFragments(initFragRoot())

// FragContent renders in HTML without the outer section or body.
func FragContent(fragref string) string {
	return Frag.ResolveWith(fragref, FragOptions{Inner: true})
}

var docState = NewDocState(Frag)

func Doc(fragref string) string     { return docState.Doc(fragref) }
func DocFlag(fragref string) string { return docState.DocFlag(fragref) }
func DocFragments() []string        { return docState.DocFragments() }
func UnresolvedFragments()          { docState.UnresolvedFragments() }

var Program = &cobra.Command{
	Use:           "rbxmk",
	Short:         Doc("Commands:Summary"),
	Long:          Doc("Commands:Summary"),
	SilenceUsage:  false,
	SilenceErrors: true,
}

// Template function that expands environment variables. Each argument is an
// alternating key and value. The last value is the string to expand.
func expand(p ...string) string {
	if len(p) == 0 {
		return ""
	}
	s, p := p[len(p)-1], p[:len(p)-1]
	m := make(map[string]string, len(p)/2)
	for i := 1; i < len(p); i += 2 {
		m[p[i-1]] = p[i]
	}
	return os.Expand(s, func(s string) string { return m[s] })
}

func init() {
	fragTmplFuncs["frag"] = FragContent
	fragTmplFuncs["fraglist"] = Frag.List
	fragTmplFuncs["expand"] = os.Expand

	cobra.AddTemplateFunc("frag", func(fragref string) string {
		return Frag.ResolveWith(fragref, FragOptions{
			Renderer: term.Renderer{Width: -1}.Render,
			Inner:    true,
		})
	})
	cobra.AddTemplateFunc("expand", expand)
	cobra.AddTemplateFunc("width", func() int {
		width, _, _ := terminal.GetSize(int(os.Stdout.Fd()))
		return width
	})
	Program.SetUsageTemplate(usageTemplate)
}

func main() {
	ctx, stop := signal.NotifyContext(context.Background(), os.Kill)
	defer stop()

	Program.SetIn(os.Stdin)
	Program.SetOut(os.Stdout)
	Program.SetErr(os.Stderr)

	DocumentCommands()
	UnresolvedFragments()
	if err := Program.ExecuteContext(ctx); err != nil {
		Program.PrintErrln(err)
	}
}
<|endoftext|>


<commit_before>/*

*/
package mandrake

import (
	"log"
	"encoding/json"

	"golang.org/x/exp/inotify"
	"github.com/hosom/gomandrake/config"
	"github.com/hosom/gomandrake/filemeta"
	"github.com/hosom/gomandrake/plugin"
)

// Mandrake is a wrapper struct for the bulk of the application logic
type Mandrake struct {
	AnalysisPipeline	chan string
	LoggingPipeline		chan string
	MonitoredDirectory	string
	Analyzers			[]plugin.AnalyzerCaller
	AnalyzerFilter		map[string][]plugin.AnalyzerCaller
	Loggers				[]plugin.LoggerCaller
}

// NewMandrake creates and returns a Mandrake struct utilizing a passed
// parsed configuration file to create the correct fields.
func NewMandrake(c config.Config) Mandrake {
	analyzers := []plugin.AnalyzerCaller{}
	filter := make(map[string][]plugin.AnalyzerCaller)
	for _, plug := range c.Analyzers {
		analyzer := plugin.NewAnalyzerCaller(plug)
		// Build a slice of all AnalyzerCaller structs
		analyzers = append(analyzers, analyzer)

		// Create a map to function as a mime_type filter for analyzers
		for _, mime := range analyzer.MimeFilter {
			filter[mime] = append(filter[mime], analyzer)
		}
	}

	loggers := []plugin.LoggerCaller{}
	for _, plug := range c.Loggers {
		logger := plugin.NewLoggerCaller(plug)
		loggers = append(loggers, logger)
	}

	return Mandrake{make(chan string), make(chan string), c.MonitoredDirectory, analyzers, filter, loggers}
}

// ListenAndServe starts the goroutines that perform all of the heavy lifting
// including Monitor() and DispatchAnalysis().
func (m Mandrake) ListenAndServe() {
	log.SetPrefix("[mandrake] ")
	log.Println(m.Analyzers[0])
	go m.DispatchLogging()
	go m.DispatchAnalysis()
	m.Monitor()
}

// DispatchAnalysis intelligently sends a new file to registered plugins so
// that it can be analyzed.
func (m Mandrake) DispatchAnalysis() {
	for fpath := range m.AnalysisPipeline {
		go m.Analysis(fpath)
	}
}

// Analysis is the method that kicks off all of the analysis plugins
// this is utilized so that each file can be analyzed in a goroutine
func (m Mandrake) Analysis(fpath string) {
	fmeta, err := filemeta.NewFileMeta(fpath)
	if err != nil {
		log.Println(err)
	}

	// Create JSON filemeta object to pass to plugins so that plugins
	// receive basic contextual information about the file.
	fs, err := json.Marshal(fmeta)
	// Finalize string form of JSON filemeta to pass to plugins
	fstring := string(fs)

	var analysis []map[string]interface{}

	for _, analyzer := range m.AnalyzerFilter["all"] {
		result, err := analyzer.Analyze(fstring)
		if err != nil {
			log.Print(err)
		}
		analysis = append(analysis, MapFromJSON(result))
	}

	for _, analyzer := range m.AnalyzerFilter[fmeta.Mime] {
		result, err := analyzer.Analyze(fstring)
		if err != nil {
			log.Print(err)
		}
		analysis = append(analysis, MapFromJSON(result))
	}

	report := MapFromJSON(fstring)
	report["analysis"] = analysis

	r, _ := json.Marshal(report)

	log.Printf("Analysis of %s complete", fpath)
	m.LoggingPipeline <- string(r)
	log.Printf("File analysis sent to logging pipeline.")
}

// DispatchLogging sends the call to the Logger plugins to log the completed
// record of analysis performed by Mandrake
func (m Mandrake) DispatchLogging() {
	for record := range m.LoggingPipeline {
		for _, logger := range m.Loggers {
			_, err := logger.Log(record)
			if err != nil {
				log.Print(err)
			}
		}
	}
}

// Monitor uses inotify to monitor the MonitoredDirectory for IN_CLOSE_WRITE
// events. Files written to the MonitoredDirectory will be sent to the
// analysis pipeline to be analyzed.
func (m Mandrake) Monitor() {
	log.Println("starting inotify watcher")
	watcher, err := inotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}

	log.Printf("adding watcher to %s directory", m.MonitoredDirectory)
	err = watcher.AddWatch(m.MonitoredDirectory, inotify.IN_CLOSE_WRITE)
	if err != nil {
		log.Fatal(err)
	}

	for {
		select {
		case ev := <- watcher.Event:
			m.AnalysisPipeline <- ev.Name
		case err := <- watcher.Error:
			log.Printf("inotify error: %s", err)
		}
	}
}

// MapFromJSON accepts an anonymous JSON object as a string and returns the
// resulting Map
func MapFromJSON(s string) map[string]interface{} {
	log.Printf("Performing mapping with string: %s", s)
	var f interface{}
	json.Unmarshal([]byte(s), &f)
	m := f.(map[string]interface{})
	log.Printf("Mapping complete: %s", m)
	return m
}<commit_msg>prevent panic on plugin failure<commit_after>/*

*/
package mandrake

import (
	"log"
	"encoding/json"

	"golang.org/x/exp/inotify"
	"github.com/hosom/gomandrake/config"
	"github.com/hosom/gomandrake/filemeta"
	"github.com/hosom/gomandrake/plugin"
)

// Mandrake is a wrapper struct for the bulk of the application logic
type Mandrake struct {
	AnalysisPipeline	chan string
	LoggingPipeline		chan string
	MonitoredDirectory	string
	Analyzers			[]plugin.AnalyzerCaller
	AnalyzerFilter		map[string][]plugin.AnalyzerCaller
	Loggers				[]plugin.LoggerCaller
}

// NewMandrake creates and returns a Mandrake struct utilizing a passed
// parsed configuration file to create the correct fields.
func NewMandrake(c config.Config) Mandrake {
	analyzers := []plugin.AnalyzerCaller{}
	filter := make(map[string][]plugin.AnalyzerCaller)
	for _, plug := range c.Analyzers {
		analyzer := plugin.NewAnalyzerCaller(plug)
		// Build a slice of all AnalyzerCaller structs
		analyzers = append(analyzers, analyzer)

		// Create a map to function as a mime_type filter for analyzers
		for _, mime := range analyzer.MimeFilter {
			filter[mime] = append(filter[mime], analyzer)
		}
	}

	loggers := []plugin.LoggerCaller{}
	for _, plug := range c.Loggers {
		logger := plugin.NewLoggerCaller(plug)
		loggers = append(loggers, logger)
	}

	return Mandrake{make(chan string), make(chan string), c.MonitoredDirectory, analyzers, filter, loggers}
}

// ListenAndServe starts the goroutines that perform all of the heavy lifting
// including Monitor() and DispatchAnalysis().
func (m Mandrake) ListenAndServe() {
	log.SetPrefix("[mandrake] ")
	log.Println(m.Analyzers[0])
	go m.DispatchLogging()
	go m.DispatchAnalysis()
	m.Monitor()
}

// DispatchAnalysis intelligently sends a new file to registered plugins so
// that it can be analyzed.
func (m Mandrake) DispatchAnalysis() {
	for fpath := range m.AnalysisPipeline {
		go m.Analysis(fpath)
	}
}

// Analysis is the method that kicks off all of the analysis plugins
// this is utilized so that each file can be analyzed in a goroutine
func (m Mandrake) Analysis(fpath string) {
	fmeta, err := filemeta.NewFileMeta(fpath)
	if err != nil {
		log.Println(err)
	}

	// Create JSON filemeta object to pass to plugins so that plugins
	// receive basic contextual information about the file.
	fs, err := json.Marshal(fmeta)
	// Finalize string form of JSON filemeta to pass to plugins
	fstring := string(fs)

	var analysis []map[string]interface{}

	for _, analyzer := range m.AnalyzerFilter["all"] {
		result, err := analyzer.Analyze(fstring)
		if err != nil {
			log.Print(err)
		}
		analysis = append(analysis, MapFromJSON(result))
	}

	for _, analyzer := range m.AnalyzerFilter[fmeta.Mime] {
		result, err := analyzer.Analyze(fstring)
		if err != nil {
			log.Print(err)
		}
		analysis = append(analysis, MapFromJSON(result))
	}

	report := MapFromJSON(fstring)
	report["analysis"] = analysis

	r, _ := json.Marshal(report)

	log.Printf("Analysis of %s complete", fpath)
	m.LoggingPipeline <- string(r)
	log.Printf("File analysis sent to logging pipeline.")
}

// DispatchLogging sends the call to the Logger plugins to log the completed
// record of analysis performed by Mandrake
func (m Mandrake) DispatchLogging() {
	for record := range m.LoggingPipeline {
		for _, logger := range m.Loggers {
			_, err := logger.Log(record)
			if err != nil {
				log.Print(err)
			}
		}
	}
}

// Monitor uses inotify to monitor the MonitoredDirectory for IN_CLOSE_WRITE
// events. Files written to the MonitoredDirectory will be sent to the
// analysis pipeline to be analyzed.
func (m Mandrake) Monitor() {
	log.Println("starting inotify watcher")
	watcher, err := inotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}

	log.Printf("adding watcher to %s directory", m.MonitoredDirectory)
	err = watcher.AddWatch(m.MonitoredDirectory, inotify.IN_CLOSE_WRITE)
	if err != nil {
		log.Fatal(err)
	}

	for {
		select {
		case ev := <- watcher.Event:
			m.AnalysisPipeline <- ev.Name
		case err := <- watcher.Error:
			log.Printf("inotify error: %s", err)
		}
	}
}

// MapFromJSON accepts an anonymous JSON object as a string and returns the
// resulting Map
func MapFromJSON(s string) map[string]interface{} {
	log.Printf("Performing mapping with string: %s", s)
	if s == "" {
		log.Println("Encountered invalid output from plugin.")
		return map[string]interface{}
	}

	var f interface{}
	json.Unmarshal([]byte(s), &f)
	m := f.(map[string]interface{})
	log.Printf("Mapping complete: %s", m)
	return m
}<|endoftext|>


<commit_before>package hstspreload

import (
	"crypto/tls"
	"crypto/x509"
	"net"
	"net/http"
	"strings"
	"time"

	"golang.org/x/net/publicsuffix"
)

const (
	// dialTimeout specifies the amount of time that TCP or TLS connections
	// can take to complete.
	dialTimeout = 10 * time.Second
)

// dialer is a global net.Dialer that's used whenever making TLS connections in
// order to enforce dialTimeout.
var dialer = net.Dialer{
	Timeout: dialTimeout,
}

var clientWithTimeout = http.Client{
	Timeout: dialTimeout,
}

// List of eTLDs for which:
// - `www` subdomains are commonly available over HTTP, but
// - but site owners have no way to serve valid HTTPS on the `www` subdomain.
//
// We whitelist such eTLDs to waive the `www` subdomain requirement.
var whitelistedWWWeTLDs = map[string]bool{
	"appspot.com": true,
}

// PreloadableDomain checks whether the domain passes HSTS preload
// requirements for Chromium. This includes:
//
// - Serving a single HSTS header that passes header requirements.
//
// - Using TLS settings that will not cause new problems for
// Chromium/Chrome users. (Example of a new problem: a missing intermediate certificate
// will turn an error page from overrideable to non-overridable on
// some mobile devices.)
//
// Iff a single HSTS header was received, `header` contains its value, else
// `header` is `nil`.
// To interpret `issues`, see the list of conventions in the
// documentation for Issues.
func PreloadableDomain(domain string) (header *string, issues Issues) {
	// Check domain format issues first, since we can report something
	// useful even if the other checks fail.
	issues = combineIssues(issues, checkDomainFormat(domain))
	if len(issues.Errors) > 0 {
		return header, issues
	}

	// We don't currently allow automatic submissions of subdomains.
	levelIssues := preloadableDomainLevel(domain)
	issues = combineIssues(issues, levelIssues)

	// Start with an initial probe, and don't do the follow-up checks if
	// we can't connect.
	resp, respIssues := getResponse(domain)
	issues = combineIssues(issues, respIssues)
	if len(respIssues.Errors) == 0 {
		issues = combineIssues(issues, checkChain(*resp.TLS))

		preloadableResponse := make(chan Issues)
		httpRedirectsGeneral := make(chan Issues)
		httpFirstRedirectHSTS := make(chan Issues)
		httpsRedirects := make(chan Issues)
		www := make(chan Issues)

		// PreloadableResponse
		go func() {
			var preloadableIssues Issues
			header, preloadableIssues = PreloadableResponse(resp)
			preloadableResponse <- preloadableIssues
		}()

		// checkHTTPRedirects
		go func() {
			general, firstRedirectHSTS := preloadableHTTPRedirects(domain)
			httpRedirectsGeneral <- general
			httpFirstRedirectHSTS <- firstRedirectHSTS
		}()

		// checkHTTPSRedirects
		go func() {
			httpsRedirects <- preloadableHTTPSRedirects(domain)
		}()

		// checkWWW
		go func() {
			eTLD, _ := publicsuffix.PublicSuffix(domain)

			// Skip the WWW check if the domain is not eTLD+1, or if the
			// eTLD is whitelisted.
			if len(levelIssues.Errors) != 0 || whitelistedWWWeTLDs[eTLD] {
				www <- Issues{}
			} else {
				www <- checkWWW(domain)
			}
		}()

		// Combine the issues in deterministic order.
		preloadableResponseIssues := <-preloadableResponse
		issues = combineIssues(issues, preloadableResponseIssues)
		issues = combineIssues(issues, <-httpRedirectsGeneral)
		// If there are issues with the HSTS header in the main
		// PreloadableResponse() check, it is redundant to report
		// them in the response after redirecting from HTTP.
		firstRedirectHSTS := <-httpFirstRedirectHSTS // always receive the value, to avoid leaking a goroutine
		if len(preloadableResponseIssues.Errors) == 0 {
			issues = combineIssues(issues, firstRedirectHSTS)
		}
		issues = combineIssues(issues, <-httpsRedirects)
		issues = combineIssues(issues, <-www)
	}

	return header, issues
}

// RemovableDomain checks whether the domain satisfies the requirements
// for being removed from the Chromium preload list:
//
// - Serving a single valid HSTS header.
//
// - The header must not contain the `preload` directive..
//
// Iff a single HSTS header was received, `header` contains its value, else
// `header` is `nil`.
// To interpret `issues`, see the list of conventions in the
// documentation for Issues.
func RemovableDomain(domain string) (header *string, issues Issues) {
	resp, respIssues := getResponse(domain)
	issues = combineIssues(issues, respIssues)
	if len(respIssues.Errors) == 0 {
		var removableIssues Issues
		header, removableIssues = RemovableResponse(resp)
		issues = combineIssues(issues, removableIssues)
	}

	return header, issues
}

func getResponse(domain string) (*http.Response, Issues) {
	issues := Issues{}

	// Try #1
	resp, err := getFirstResponse("https://" + domain)
	if err == nil {
		return resp, issues
	}

	// Try #2
	resp, err = getFirstResponse("https://" + domain)
	if err == nil {
		return resp, issues
	}

	// Check if ignoring cert issues works.
	transport := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
	resp, err = getFirstResponseWithTransport("https://"+domain, transport)
	if err == nil {
		return resp, issues.addErrorf(
			IssueCode("domain.tls.invalid_cert_chain"),
			"Invalid Certificate Chain",
			"https://%s uses an incomplete or "+
				"invalid certificate chain. Check out your site at "+
				"https://www.ssllabs.com/ssltest/",
			domain,
		)
	}

	return resp, issues.addErrorf(
		IssueCode("domain.tls.cannot_connect"),
		"Cannot connect using TLS",
		"We cannot connect to https://%s using TLS (%q).",
		domain,
		err,
	)
}

func checkDomainFormat(domain string) Issues {
	issues := Issues{}

	if strings.HasPrefix(domain, ".") {
		return issues.addErrorf(
			IssueCode("domain.format.begins_with_dot"),
			"Invalid domain name",
			"Please provide a domain that does not begin with `.`")
	}
	if strings.HasSuffix(domain, ".") {
		return issues.addErrorf(
			IssueCode("domain.format.ends_with_dot"),
			"Invalid domain name",
			"Please provide a domain that does not begin with `.`")
	}
	if strings.Index(domain, "..") != -1 {
		return issues.addErrorf(
			IssueCode("domain.format.contains_double_dot"),
			"Invalid domain name",
			"Please provide a domain that does not contain `..`")
	}
	if strings.Count(domain, ".") < 1 {
		return issues.addErrorf(
			IssueCode("domain.format.only_one_label"),
			"Invalid domain name",
			"Please provide a domain with least two labels "+
				"(e.g. `example.com` rather than `example` or `com`).")
	}

	domain = strings.ToLower(domain)
	for _, r := range domain {
		if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '-' || r == '.' {
			continue
		}

		return issues.addErrorf("domain.format.invalid_characters", "Invalid domain name", "Please provide a domain using valid characters (letters, numbers, dashes, dots).")
	}

	return issues
}

func preloadableDomainLevel(domain string) Issues {
	issues := Issues{}

	eTLD1, err := publicsuffix.EffectiveTLDPlusOne(domain)
	if err != nil {
		return issues.addErrorf("internal.domain.name.cannot_compute_etld1", "Internal Error", "Could not compute eTLD+1.")
	}

	if eTLD1 != domain {
		return issues.addErrorf(
			IssueCode("domain.is_subdomain"),
			"Subdomain",
			"`%s` is a subdomain. Please preload `%s` instead. "+
				"(Due to the size of the preload list and the behaviour of "+
				"cookies across subdomains, we only accept automated preload list "+
				"submissions of whole registered domains.)",
			domain,
			eTLD1,
		)
	}

	return issues
}

func checkChain(connState tls.ConnectionState) Issues {
	fullChain := connState.VerifiedChains[0]
	chain := fullChain[:len(fullChain)-1] // Ignore the root CA
	return checkSHA1(chain)
}

func checkSHA1(chain []*x509.Certificate) Issues {
	issues := Issues{}

	for _, cert := range chain {
		if cert.SignatureAlgorithm == x509.SHA1WithRSA || cert.SignatureAlgorithm == x509.ECDSAWithSHA1 {
			return issues.addErrorf(
				IssueCode("domain.tls.sha1"),
				"SHA-1 Certificate",
				"One or more of the certificates in your certificate chain "+
					"is signed using SHA-1. This needs to be replaced. "+
					"See https://security.googleblog.com/2015/12/an-update-on-sha-1-certificates-in.html. "+
					"(The first SHA-1 certificate found has a common-name of %q.)",
				cert.Subject.CommonName,
			)
		}
	}

	return issues
}

func checkWWW(host string) Issues {
	issues := Issues{}

	hasWWW := false
	if conn, err := net.DialTimeout("tcp", "www."+host+":443", dialTimeout); err == nil {
		hasWWW = true
		if err = conn.Close(); err != nil {
			return issues.addErrorf(
				"internal.domain.www.first_dial.no_close",
				"Internal error",
				"Error while closing a connection to %s: %s",
				"www."+host,
				err,
			)
		}
	}

	if hasWWW {
		wwwConn, err := tls.DialWithDialer(&dialer, "tcp", "www."+host+":443", nil)
		if err != nil {
			return issues.addErrorf(
				IssueCode("domain.www.no_tls"),
				"www subdomain does not support HTTPS",
				"Domain error: The www subdomain exists, but we couldn't connect to it using HTTPS (%q). "+
					"Since many people type this by habit, HSTS preloading would likely "+
					"cause issues for your site.",
				err,
			)
		}
		if err = wwwConn.Close(); err != nil {
			return issues.addErrorf(
				"internal.domain.www.second_dial.no_close",
				"Internal error",
				"Error while closing a connection to %s: %s",
				"www."+host,
				err,
			)
		}
	}

	return issues
}
<commit_msg>Fix typo in domain.format.ends_with_dot<commit_after>package hstspreload

import (
	"crypto/tls"
	"crypto/x509"
	"net"
	"net/http"
	"strings"
	"time"

	"golang.org/x/net/publicsuffix"
)

const (
	// dialTimeout specifies the amount of time that TCP or TLS connections
	// can take to complete.
	dialTimeout = 10 * time.Second
)

// dialer is a global net.Dialer that's used whenever making TLS connections in
// order to enforce dialTimeout.
var dialer = net.Dialer{
	Timeout: dialTimeout,
}

var clientWithTimeout = http.Client{
	Timeout: dialTimeout,
}

// List of eTLDs for which:
// - `www` subdomains are commonly available over HTTP, but
// - but site owners have no way to serve valid HTTPS on the `www` subdomain.
//
// We whitelist such eTLDs to waive the `www` subdomain requirement.
var whitelistedWWWeTLDs = map[string]bool{
	"appspot.com": true,
}

// PreloadableDomain checks whether the domain passes HSTS preload
// requirements for Chromium. This includes:
//
// - Serving a single HSTS header that passes header requirements.
//
// - Using TLS settings that will not cause new problems for
// Chromium/Chrome users. (Example of a new problem: a missing intermediate certificate
// will turn an error page from overrideable to non-overridable on
// some mobile devices.)
//
// Iff a single HSTS header was received, `header` contains its value, else
// `header` is `nil`.
// To interpret `issues`, see the list of conventions in the
// documentation for Issues.
func PreloadableDomain(domain string) (header *string, issues Issues) {
	// Check domain format issues first, since we can report something
	// useful even if the other checks fail.
	issues = combineIssues(issues, checkDomainFormat(domain))
	if len(issues.Errors) > 0 {
		return header, issues
	}

	// We don't currently allow automatic submissions of subdomains.
	levelIssues := preloadableDomainLevel(domain)
	issues = combineIssues(issues, levelIssues)

	// Start with an initial probe, and don't do the follow-up checks if
	// we can't connect.
	resp, respIssues := getResponse(domain)
	issues = combineIssues(issues, respIssues)
	if len(respIssues.Errors) == 0 {
		issues = combineIssues(issues, checkChain(*resp.TLS))

		preloadableResponse := make(chan Issues)
		httpRedirectsGeneral := make(chan Issues)
		httpFirstRedirectHSTS := make(chan Issues)
		httpsRedirects := make(chan Issues)
		www := make(chan Issues)

		// PreloadableResponse
		go func() {
			var preloadableIssues Issues
			header, preloadableIssues = PreloadableResponse(resp)
			preloadableResponse <- preloadableIssues
		}()

		// checkHTTPRedirects
		go func() {
			general, firstRedirectHSTS := preloadableHTTPRedirects(domain)
			httpRedirectsGeneral <- general
			httpFirstRedirectHSTS <- firstRedirectHSTS
		}()

		// checkHTTPSRedirects
		go func() {
			httpsRedirects <- preloadableHTTPSRedirects(domain)
		}()

		// checkWWW
		go func() {
			eTLD, _ := publicsuffix.PublicSuffix(domain)

			// Skip the WWW check if the domain is not eTLD+1, or if the
			// eTLD is whitelisted.
			if len(levelIssues.Errors) != 0 || whitelistedWWWeTLDs[eTLD] {
				www <- Issues{}
			} else {
				www <- checkWWW(domain)
			}
		}()

		// Combine the issues in deterministic order.
		preloadableResponseIssues := <-preloadableResponse
		issues = combineIssues(issues, preloadableResponseIssues)
		issues = combineIssues(issues, <-httpRedirectsGeneral)
		// If there are issues with the HSTS header in the main
		// PreloadableResponse() check, it is redundant to report
		// them in the response after redirecting from HTTP.
		firstRedirectHSTS := <-httpFirstRedirectHSTS // always receive the value, to avoid leaking a goroutine
		if len(preloadableResponseIssues.Errors) == 0 {
			issues = combineIssues(issues, firstRedirectHSTS)
		}
		issues = combineIssues(issues, <-httpsRedirects)
		issues = combineIssues(issues, <-www)
	}

	return header, issues
}

// RemovableDomain checks whether the domain satisfies the requirements
// for being removed from the Chromium preload list:
//
// - Serving a single valid HSTS header.
//
// - The header must not contain the `preload` directive..
//
// Iff a single HSTS header was received, `header` contains its value, else
// `header` is `nil`.
// To interpret `issues`, see the list of conventions in the
// documentation for Issues.
func RemovableDomain(domain string) (header *string, issues Issues) {
	resp, respIssues := getResponse(domain)
	issues = combineIssues(issues, respIssues)
	if len(respIssues.Errors) == 0 {
		var removableIssues Issues
		header, removableIssues = RemovableResponse(resp)
		issues = combineIssues(issues, removableIssues)
	}

	return header, issues
}

func getResponse(domain string) (*http.Response, Issues) {
	issues := Issues{}

	// Try #1
	resp, err := getFirstResponse("https://" + domain)
	if err == nil {
		return resp, issues
	}

	// Try #2
	resp, err = getFirstResponse("https://" + domain)
	if err == nil {
		return resp, issues
	}

	// Check if ignoring cert issues works.
	transport := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
	resp, err = getFirstResponseWithTransport("https://"+domain, transport)
	if err == nil {
		return resp, issues.addErrorf(
			IssueCode("domain.tls.invalid_cert_chain"),
			"Invalid Certificate Chain",
			"https://%s uses an incomplete or "+
				"invalid certificate chain. Check out your site at "+
				"https://www.ssllabs.com/ssltest/",
			domain,
		)
	}

	return resp, issues.addErrorf(
		IssueCode("domain.tls.cannot_connect"),
		"Cannot connect using TLS",
		"We cannot connect to https://%s using TLS (%q).",
		domain,
		err,
	)
}

func checkDomainFormat(domain string) Issues {
	issues := Issues{}

	if strings.HasPrefix(domain, ".") {
		return issues.addErrorf(
			IssueCode("domain.format.begins_with_dot"),
			"Invalid domain name",
			"Please provide a domain that does not begin with `.`")
	}
	if strings.HasSuffix(domain, ".") {
		return issues.addErrorf(
			IssueCode("domain.format.ends_with_dot"),
			"Invalid domain name",
			"Please provide a domain that does not end with `.`")
	}
	if strings.Index(domain, "..") != -1 {
		return issues.addErrorf(
			IssueCode("domain.format.contains_double_dot"),
			"Invalid domain name",
			"Please provide a domain that does not contain `..`")
	}
	if strings.Count(domain, ".") < 1 {
		return issues.addErrorf(
			IssueCode("domain.format.only_one_label"),
			"Invalid domain name",
			"Please provide a domain with least two labels "+
				"(e.g. `example.com` rather than `example` or `com`).")
	}

	domain = strings.ToLower(domain)
	for _, r := range domain {
		if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '-' || r == '.' {
			continue
		}

		return issues.addErrorf("domain.format.invalid_characters", "Invalid domain name", "Please provide a domain using valid characters (letters, numbers, dashes, dots).")
	}

	return issues
}

func preloadableDomainLevel(domain string) Issues {
	issues := Issues{}

	eTLD1, err := publicsuffix.EffectiveTLDPlusOne(domain)
	if err != nil {
		return issues.addErrorf("internal.domain.name.cannot_compute_etld1", "Internal Error", "Could not compute eTLD+1.")
	}

	if eTLD1 != domain {
		return issues.addErrorf(
			IssueCode("domain.is_subdomain"),
			"Subdomain",
			"`%s` is a subdomain. Please preload `%s` instead. "+
				"(Due to the size of the preload list and the behaviour of "+
				"cookies across subdomains, we only accept automated preload list "+
				"submissions of whole registered domains.)",
			domain,
			eTLD1,
		)
	}

	return issues
}

func checkChain(connState tls.ConnectionState) Issues {
	fullChain := connState.VerifiedChains[0]
	chain := fullChain[:len(fullChain)-1] // Ignore the root CA
	return checkSHA1(chain)
}

func checkSHA1(chain []*x509.Certificate) Issues {
	issues := Issues{}

	for _, cert := range chain {
		if cert.SignatureAlgorithm == x509.SHA1WithRSA || cert.SignatureAlgorithm == x509.ECDSAWithSHA1 {
			return issues.addErrorf(
				IssueCode("domain.tls.sha1"),
				"SHA-1 Certificate",
				"One or more of the certificates in your certificate chain "+
					"is signed using SHA-1. This needs to be replaced. "+
					"See https://security.googleblog.com/2015/12/an-update-on-sha-1-certificates-in.html. "+
					"(The first SHA-1 certificate found has a common-name of %q.)",
				cert.Subject.CommonName,
			)
		}
	}

	return issues
}

func checkWWW(host string) Issues {
	issues := Issues{}

	hasWWW := false
	if conn, err := net.DialTimeout("tcp", "www."+host+":443", dialTimeout); err == nil {
		hasWWW = true
		if err = conn.Close(); err != nil {
			return issues.addErrorf(
				"internal.domain.www.first_dial.no_close",
				"Internal error",
				"Error while closing a connection to %s: %s",
				"www."+host,
				err,
			)
		}
	}

	if hasWWW {
		wwwConn, err := tls.DialWithDialer(&dialer, "tcp", "www."+host+":443", nil)
		if err != nil {
			return issues.addErrorf(
				IssueCode("domain.www.no_tls"),
				"www subdomain does not support HTTPS",
				"Domain error: The www subdomain exists, but we couldn't connect to it using HTTPS (%q). "+
					"Since many people type this by habit, HSTS preloading would likely "+
					"cause issues for your site.",
				err,
			)
		}
		if err = wwwConn.Close(); err != nil {
			return issues.addErrorf(
				"internal.domain.www.second_dial.no_close",
				"Internal error",
				"Error while closing a connection to %s: %s",
				"www."+host,
				err,
			)
		}
	}

	return issues
}
<|endoftext|>


<commit_before>package main

import (
	"./lib"
	"flag"
	"fmt"
)

func main() {
	now := time.Now()
	optDest := flag.String("dest", "", "Please Specify ServiceMetric or Host (※default Host)")
	optApiKey := flag.String("api-key", "", "API Key must have read and write authority")
	optServiceName := flag.String("service-name", "", "target serviceName")
	optAccessKeyID := flag.String("access-key-id", "", "AWS Access Key ID")
	optSecretAccessKey := flag.String("secret-access-key", "", "AWS Secret Access Key")
	optCurrency := flag.String("currency", "USD", "Unit of currency")
	optTarget := flag.String("target", "", "Target AWS Service. if no specific Aws Service, get metrics list from cloudwatch and draw all available metrics")
	flag.Parse()

	if now.Minute() == 0 {
		mpawsbilling.WriteCache(*optAccessKeyID, *optSecretAccessKey, *optCurrency, *optTarget)
	}

	if *optDest == "ServiceMetric" {
		mpawsbilling.SendServiceMetric(*optApiKey, *optServiceName)
	} else if *optDest == "Host" {
		mpawsbilling.OutputData()
	}
}
<commit_msg>fix: delete unused package<commit_after>package main

import (
	"./lib"
	"flag"
	"time"
)

func main() {
	now := time.Now()
	optDest := flag.String("dest", "", "Please Specify ServiceMetric or Host (※default Host)")
	optApiKey := flag.String("api-key", "", "API Key must have read and write authority")
	optServiceName := flag.String("service-name", "", "target serviceName")
	optAccessKeyID := flag.String("access-key-id", "", "AWS Access Key ID")
	optSecretAccessKey := flag.String("secret-access-key", "", "AWS Secret Access Key")
	optCurrency := flag.String("currency", "USD", "Unit of currency")
	optTarget := flag.String("target", "", "Target AWS Service. if no specific Aws Service, get metrics list from cloudwatch and draw all available metrics")
	flag.Parse()

	if now.Minute() == 0 {
		mpawsbilling.WriteCache(*optAccessKeyID, *optSecretAccessKey, *optCurrency, *optTarget)
	}

	if *optDest == "ServiceMetric" {
		mpawsbilling.SendServiceMetric(*optApiKey, *optServiceName)
	} else if *optDest == "Host" {
		mpawsbilling.OutputData()
	}
}
<|endoftext|>


<commit_before>package test_helpers

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"syscall"
	"testing"

	"github.com/rfjakob/gocryptfs/internal/nametransform"
)

// TmpDir will be created inside this directory
const testParentDir = "/tmp/gocryptfs-test-parent"

// GocryptfsBinary is the assumed path to the gocryptfs build.
const GocryptfsBinary = "../../gocryptfs"

// TmpDir is a unique temporary directory. "go test" runs package tests in parallel. We create a
// unique TmpDir in init() so the tests do not interfere.
var TmpDir string

// DefaultPlainDir is TmpDir + "/default-plain"
var DefaultPlainDir string

// DefaultCipherDir is TmpDir + "/default-cipher"
var DefaultCipherDir string

func init() {
	os.MkdirAll(testParentDir, 0700)
	var err error
	TmpDir, err = ioutil.TempDir(testParentDir, "")
	if err != nil {
		panic(err)
	}
	DefaultPlainDir = TmpDir + "/default-plain"
	DefaultCipherDir = TmpDir + "/default-cipher"
}

// ResetTmpDir deletes TmpDir, create new dir tree:
//
// TmpDir
// |-- DefaultPlainDir
// *-- DefaultCipherDir
//     *-- gocryptfs.diriv
func ResetTmpDir(createDirIV bool) {
	// Try to unmount and delete everything
	entries, err := ioutil.ReadDir(TmpDir)
	if err == nil {
		for _, e := range entries {
			d := filepath.Join(TmpDir, e.Name())
			err = os.Remove(d)
			if err != nil {
				pe := err.(*os.PathError)
				if pe.Err == syscall.EBUSY {
					if testing.Verbose() {
						fmt.Printf("Remove failed: %v. Maybe still mounted?\n", pe)
					}
					err = UnmountErr(d)
					if err != nil {
						panic(err)
					}
				} else if pe.Err != syscall.ENOTEMPTY {
					panic("Unhandled error: " + pe.Err.Error())
				}
				err = os.RemoveAll(d)
				if err != nil {
					panic(err)
				}
			}
		}
	}
	err = os.Mkdir(DefaultPlainDir, 0700)
	if err != nil {
		panic(err)
	}
	err = os.Mkdir(DefaultCipherDir, 0700)
	if err != nil {
		panic(err)
	}
	if createDirIV {
		err = nametransform.WriteDirIV(DefaultCipherDir)
		if err != nil {
			panic(err)
		}
	}
}

// InitFS calls "gocryptfs -init" on a new directory in TmpDir, passing
// "extraArgs" in addition to useful defaults.
//
// The returned cipherdir has NO trailing slash.
func InitFS(t *testing.T, extraArgs ...string) string {
	dir, err := ioutil.TempDir(TmpDir, "")
	if err != nil {
		t.Fatal(err)
	}
	args := []string{"-q", "-init", "-extpass", "echo test", "-scryptn=10"}
	args = append(args, extraArgs...)
	args = append(args, dir)

	cmd := exec.Command(GocryptfsBinary, args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	err = cmd.Run()
	if err != nil {
		t.Fatalf("InitFS with args %v failed: %v", args, err)
	}

	return dir
}

// Mount CIPHERDIR "c" on PLAINDIR "p"
// Creates "p" if it does not exist.
func Mount(c string, p string, showOutput bool, extraArgs ...string) error {
	var args []string
	args = append(args, extraArgs...)
	args = append(args, "-q", "-wpanic")
	//args = append(args, "-fusedebug")
	//args = append(args, "-d")
	args = append(args, c)
	args = append(args, p)

	if _, err := os.Stat(p); err != nil {
		err = os.Mkdir(p, 0777)
		if err != nil {
			return err
		}
	}

	cmd := exec.Command(GocryptfsBinary, args...)
	if showOutput {
		cmd.Stderr = os.Stderr
		cmd.Stdout = os.Stdout
	}

	return cmd.Run()
}

// MountOrExit calls Mount() and exits on failure.
func MountOrExit(c string, p string, extraArgs ...string) {
	err := Mount(c, p, true, extraArgs...)
	if err != nil {
		fmt.Printf("mount failed: %v", err)
		os.Exit(1)
	}
}

// MountOrFatal calls Mount() and calls t.Fatal() on failure.
func MountOrFatal(t *testing.T, c string, p string, extraArgs ...string) {
	err := Mount(c, p, true, extraArgs...)
	if err != nil {
		t.Fatal(fmt.Errorf("mount failed: %v", err))
	}
}

// UnmountPanic tries to umount "dir" and panics on error.
func UnmountPanic(dir string) {
	err := UnmountErr(dir)
	if err != nil {
		fmt.Println(err)
		panic(err)
	}
}

// UnmountErr tries to unmount "dir" and returns the resulting error.
func UnmountErr(dir string) error {
	var cmd *exec.Cmd
	if runtime.GOOS == "darwin" {
		cmd = exec.Command("umount", dir)
	} else {
		cmd = exec.Command("fusermount", "-u", "-z", dir)
	}
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

// Md5fn returns an md5 string for file "filename"
func Md5fn(filename string) string {
	buf, err := ioutil.ReadFile(filename)
	if err != nil {
		fmt.Printf("ReadFile: %v\n", err)
		return ""
	}
	return Md5hex(buf)
}

// Md5hex returns an md5 string for "buf"
func Md5hex(buf []byte) string {
	rawHash := md5.Sum(buf)
	hash := hex.EncodeToString(rawHash[:])
	return hash
}

// VerifySize checks that the file size equals "want". This checks:
// 1) Size reported by Stat()
// 2) Number of bytes returned when reading the whole file
func VerifySize(t *testing.T, path string, want int) {
	buf, err := ioutil.ReadFile(path)
	if err != nil {
		t.Errorf("ReadFile failed: %v", err)
	} else if len(buf) != want {
		t.Errorf("wrong read size: got=%d want=%d", len(buf), want)
	}

	fi, err := os.Stat(path)
	if err != nil {
		t.Errorf("Stat failed: %v", err)
	} else if fi.Size() != int64(want) {
		t.Errorf("wrong stat file size, got=%d want=%d", fi.Size(), want)
	}
}

// TestMkdirRmdir creates and deletes a directory
func TestMkdirRmdir(t *testing.T, plainDir string) {
	dir := plainDir + "/dir1"
	err := os.Mkdir(dir, 0777)
	if err != nil {
		t.Fatal(err)
	}
	err = syscall.Rmdir(dir)
	if err != nil {
		t.Fatal(err)
	}

	// Removing a non-empty dir should fail with ENOTEMPTY
	if os.Mkdir(dir, 0777) != nil {
		t.Fatal(err)
	}
	f, err := os.Create(dir + "/file")
	if err != nil {
		t.Fatal(err)
	}
	f.Close()
	err = syscall.Rmdir(dir)
	errno := err.(syscall.Errno)
	if errno != syscall.ENOTEMPTY {
		t.Errorf("Should have gotten ENOTEMPTY, go %v", errno)
	}
	if syscall.Unlink(dir+"/file") != nil {
		t.Fatal(err)
	}
	if syscall.Rmdir(dir) != nil {
		t.Fatal(err)
	}

	// We should also be able to remove a directory we do not have permissions to
	// read or write
	err = os.Mkdir(dir, 0000)
	if err != nil {
		t.Fatal(err)
	}
	err = syscall.Rmdir(dir)
	if err != nil {
		// Make sure the directory can cleaned up by the next test run
		os.Chmod(dir, 0700)
		t.Fatal(err)
	}
}

// TestRename creates and renames a file
func TestRename(t *testing.T, plainDir string) {
	file1 := plainDir + "/rename1"
	file2 := plainDir + "/rename2"
	err := ioutil.WriteFile(file1, []byte("content"), 0777)
	if err != nil {
		t.Fatal(err)
	}
	err = syscall.Rename(file1, file2)
	if err != nil {
		t.Fatal(err)
	}
	syscall.Unlink(file2)
}

// VerifyExistence checks in 3 ways that "path" exists:
// stat, open, readdir
func VerifyExistence(path string) bool {
	// Check that file can be stated
	_, err := os.Stat(path)
	if err != nil {
		//t.Log(err)
		return false
	}
	// Check that file can be opened
	fd, err := os.Open(path)
	if err != nil {
		//t.Log(err)
		return false
	}
	fd.Close()
	// Check that file shows up in directory listing
	dir := filepath.Dir(path)
	name := filepath.Base(path)
	fi, err := ioutil.ReadDir(dir)
	if err != nil {
		//t.Log(err)
		return false
	}
	for _, i := range fi {
		if i.Name() == name {
			return true
		}
	}
	return false
}

// Du returns the disk usage of the file "fd" points to, in bytes.
// Same as "du --block-size=1".
func Du(t *testing.T, fd int) (nBytes int64) {
	var st syscall.Stat_t
	err := syscall.Fstat(fd, &st)
	if err != nil {
		t.Fatal(err)
	}
	// st.Blocks = number of 512-byte blocks
	return st.Blocks * 512
}
<commit_msg>tests: pass "-nosyslog"<commit_after>package test_helpers

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"syscall"
	"testing"

	"github.com/rfjakob/gocryptfs/internal/nametransform"
)

// TmpDir will be created inside this directory
const testParentDir = "/tmp/gocryptfs-test-parent"

// GocryptfsBinary is the assumed path to the gocryptfs build.
const GocryptfsBinary = "../../gocryptfs"

// TmpDir is a unique temporary directory. "go test" runs package tests in parallel. We create a
// unique TmpDir in init() so the tests do not interfere.
var TmpDir string

// DefaultPlainDir is TmpDir + "/default-plain"
var DefaultPlainDir string

// DefaultCipherDir is TmpDir + "/default-cipher"
var DefaultCipherDir string

func init() {
	os.MkdirAll(testParentDir, 0700)
	var err error
	TmpDir, err = ioutil.TempDir(testParentDir, "")
	if err != nil {
		panic(err)
	}
	DefaultPlainDir = TmpDir + "/default-plain"
	DefaultCipherDir = TmpDir + "/default-cipher"
}

// ResetTmpDir deletes TmpDir, create new dir tree:
//
// TmpDir
// |-- DefaultPlainDir
// *-- DefaultCipherDir
//     *-- gocryptfs.diriv
func ResetTmpDir(createDirIV bool) {
	// Try to unmount and delete everything
	entries, err := ioutil.ReadDir(TmpDir)
	if err == nil {
		for _, e := range entries {
			d := filepath.Join(TmpDir, e.Name())
			err = os.Remove(d)
			if err != nil {
				pe := err.(*os.PathError)
				if pe.Err == syscall.EBUSY {
					if testing.Verbose() {
						fmt.Printf("Remove failed: %v. Maybe still mounted?\n", pe)
					}
					err = UnmountErr(d)
					if err != nil {
						panic(err)
					}
				} else if pe.Err != syscall.ENOTEMPTY {
					panic("Unhandled error: " + pe.Err.Error())
				}
				err = os.RemoveAll(d)
				if err != nil {
					panic(err)
				}
			}
		}
	}
	err = os.Mkdir(DefaultPlainDir, 0700)
	if err != nil {
		panic(err)
	}
	err = os.Mkdir(DefaultCipherDir, 0700)
	if err != nil {
		panic(err)
	}
	if createDirIV {
		err = nametransform.WriteDirIV(DefaultCipherDir)
		if err != nil {
			panic(err)
		}
	}
}

// InitFS calls "gocryptfs -init" on a new directory in TmpDir, passing
// "extraArgs" in addition to useful defaults.
//
// The returned cipherdir has NO trailing slash.
func InitFS(t *testing.T, extraArgs ...string) string {
	dir, err := ioutil.TempDir(TmpDir, "")
	if err != nil {
		t.Fatal(err)
	}
	args := []string{"-q", "-init", "-extpass", "echo test", "-scryptn=10"}
	args = append(args, extraArgs...)
	args = append(args, dir)

	cmd := exec.Command(GocryptfsBinary, args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	err = cmd.Run()
	if err != nil {
		t.Fatalf("InitFS with args %v failed: %v", args, err)
	}

	return dir
}

// Mount CIPHERDIR "c" on PLAINDIR "p"
// Creates "p" if it does not exist.
func Mount(c string, p string, showOutput bool, extraArgs ...string) error {
	var args []string
	args = append(args, extraArgs...)
	args = append(args, "-q", "-wpanic", "-nosyslog")
	//args = append(args, "-fusedebug")
	//args = append(args, "-d")
	args = append(args, c)
	args = append(args, p)

	if _, err := os.Stat(p); err != nil {
		err = os.Mkdir(p, 0777)
		if err != nil {
			return err
		}
	}

	cmd := exec.Command(GocryptfsBinary, args...)
	if showOutput {
		cmd.Stderr = os.Stderr
		cmd.Stdout = os.Stdout
	}

	return cmd.Run()
}

// MountOrExit calls Mount() and exits on failure.
func MountOrExit(c string, p string, extraArgs ...string) {
	err := Mount(c, p, true, extraArgs...)
	if err != nil {
		fmt.Printf("mount failed: %v", err)
		os.Exit(1)
	}
}

// MountOrFatal calls Mount() and calls t.Fatal() on failure.
func MountOrFatal(t *testing.T, c string, p string, extraArgs ...string) {
	err := Mount(c, p, true, extraArgs...)
	if err != nil {
		t.Fatal(fmt.Errorf("mount failed: %v", err))
	}
}

// UnmountPanic tries to umount "dir" and panics on error.
func UnmountPanic(dir string) {
	err := UnmountErr(dir)
	if err != nil {
		fmt.Println(err)
		panic(err)
	}
}

// UnmountErr tries to unmount "dir" and returns the resulting error.
func UnmountErr(dir string) error {
	var cmd *exec.Cmd
	if runtime.GOOS == "darwin" {
		cmd = exec.Command("umount", dir)
	} else {
		cmd = exec.Command("fusermount", "-u", "-z", dir)
	}
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

// Md5fn returns an md5 string for file "filename"
func Md5fn(filename string) string {
	buf, err := ioutil.ReadFile(filename)
	if err != nil {
		fmt.Printf("ReadFile: %v\n", err)
		return ""
	}
	return Md5hex(buf)
}

// Md5hex returns an md5 string for "buf"
func Md5hex(buf []byte) string {
	rawHash := md5.Sum(buf)
	hash := hex.EncodeToString(rawHash[:])
	return hash
}

// VerifySize checks that the file size equals "want". This checks:
// 1) Size reported by Stat()
// 2) Number of bytes returned when reading the whole file
func VerifySize(t *testing.T, path string, want int) {
	buf, err := ioutil.ReadFile(path)
	if err != nil {
		t.Errorf("ReadFile failed: %v", err)
	} else if len(buf) != want {
		t.Errorf("wrong read size: got=%d want=%d", len(buf), want)
	}

	fi, err := os.Stat(path)
	if err != nil {
		t.Errorf("Stat failed: %v", err)
	} else if fi.Size() != int64(want) {
		t.Errorf("wrong stat file size, got=%d want=%d", fi.Size(), want)
	}
}

// TestMkdirRmdir creates and deletes a directory
func TestMkdirRmdir(t *testing.T, plainDir string) {
	dir := plainDir + "/dir1"
	err := os.Mkdir(dir, 0777)
	if err != nil {
		t.Fatal(err)
	}
	err = syscall.Rmdir(dir)
	if err != nil {
		t.Fatal(err)
	}

	// Removing a non-empty dir should fail with ENOTEMPTY
	if os.Mkdir(dir, 0777) != nil {
		t.Fatal(err)
	}
	f, err := os.Create(dir + "/file")
	if err != nil {
		t.Fatal(err)
	}
	f.Close()
	err = syscall.Rmdir(dir)
	errno := err.(syscall.Errno)
	if errno != syscall.ENOTEMPTY {
		t.Errorf("Should have gotten ENOTEMPTY, go %v", errno)
	}
	if syscall.Unlink(dir+"/file") != nil {
		t.Fatal(err)
	}
	if syscall.Rmdir(dir) != nil {
		t.Fatal(err)
	}

	// We should also be able to remove a directory we do not have permissions to
	// read or write
	err = os.Mkdir(dir, 0000)
	if err != nil {
		t.Fatal(err)
	}
	err = syscall.Rmdir(dir)
	if err != nil {
		// Make sure the directory can cleaned up by the next test run
		os.Chmod(dir, 0700)
		t.Fatal(err)
	}
}

// TestRename creates and renames a file
func TestRename(t *testing.T, plainDir string) {
	file1 := plainDir + "/rename1"
	file2 := plainDir + "/rename2"
	err := ioutil.WriteFile(file1, []byte("content"), 0777)
	if err != nil {
		t.Fatal(err)
	}
	err = syscall.Rename(file1, file2)
	if err != nil {
		t.Fatal(err)
	}
	syscall.Unlink(file2)
}

// VerifyExistence checks in 3 ways that "path" exists:
// stat, open, readdir
func VerifyExistence(path string) bool {
	// Check that file can be stated
	_, err := os.Stat(path)
	if err != nil {
		//t.Log(err)
		return false
	}
	// Check that file can be opened
	fd, err := os.Open(path)
	if err != nil {
		//t.Log(err)
		return false
	}
	fd.Close()
	// Check that file shows up in directory listing
	dir := filepath.Dir(path)
	name := filepath.Base(path)
	fi, err := ioutil.ReadDir(dir)
	if err != nil {
		//t.Log(err)
		return false
	}
	for _, i := range fi {
		if i.Name() == name {
			return true
		}
	}
	return false
}

// Du returns the disk usage of the file "fd" points to, in bytes.
// Same as "du --block-size=1".
func Du(t *testing.T, fd int) (nBytes int64) {
	var st syscall.Stat_t
	err := syscall.Fstat(fd, &st)
	if err != nil {
		t.Fatal(err)
	}
	// st.Blocks = number of 512-byte blocks
	return st.Blocks * 512
}
<|endoftext|>


<commit_before>package atlas

import (
	"github.com/stretchr/testify/assert"
	"testing"
)

func TestCheckType(t *testing.T) {
	d := Definition{Type: "foo"}

	valid := checkType(d)
	assert.EqualValues(t, false, valid, "should be false")

	d = Definition{Type: "dns"}
	valid = checkType(d)
	assert.EqualValues(t, true, valid, "should be true")
}

func TestCheckTypeAs(t *testing.T) {
	d := Definition{Type: "dns"}
	valid := checkTypeAs(d, "foo")
	assert.EqualValues(t, false, valid, "should be false")

	valid = checkTypeAs(d, "dns")
	assert.EqualValues(t, true, valid, "should be true")
}

func TestCheckAllTypesAs(t *testing.T) {
	dl := []Definition{
		{Type: "foo"},
		{Type: "ping"},
	}

	valid := checkAllTypesAs(dl, "ping")
	assert.EqualValues(t, false, valid, "should be false")

	dl = []Definition{
		{Type: "dns"},
		{Type: "ping"},
	}
	valid = checkAllTypesAs(dl, "ping")
	assert.EqualValues(t, false, valid, "should be false")

	dl = []Definition{
		{Type: "ping"},
		{Type: "ping"},
	}
	valid = checkAllTypesAs(dl, "ping")
	assert.EqualValues(t, true, valid, "should be true")
}

func TestDNS(t *testing.T) {
	d := []Definition{{Type: "foo"}}
	r := &MeasurementRequest{Definitions: d}

	_, err := DNS(r)
	assert.Error(t, err, "should be an error")
	assert.EqualValues(t, ErrInvalidMeasurementType, err, "should be equal")
}

func TestNTP(t *testing.T) {
	d := []Definition{{Type: "foo"}}
	r := &MeasurementRequest{Definitions: d}

	_, err := NTP(r)
	assert.Error(t, err, "should be an error")
	assert.EqualValues(t, ErrInvalidMeasurementType, err, "should be equal")
}

func TestPing(t *testing.T) {
	d := []Definition{{Type: "foo"}}
	r := &MeasurementRequest{Definitions: d}

	_, err := Ping(r)
	assert.Error(t, err, "should be an error")
	assert.EqualValues(t, ErrInvalidMeasurementType, err, "should be equal")
}

func TestSSLCert(t *testing.T) {
	d := []Definition{{Type: "foo"}}
	r := &MeasurementRequest{Definitions: d}

	_, err := SSLCert(r)
	assert.Error(t, err, "should be an error")
	assert.EqualValues(t, ErrInvalidMeasurementType, err, "should be equal")
}

func TestTraceroute(t *testing.T) {
	d := []Definition{{Type: "foo"}}
	r := &MeasurementRequest{Definitions: d}

	_, err := Traceroute(r)
	assert.Error(t, err, "should be an error")
	assert.EqualValues(t, ErrInvalidMeasurementType, err, "should be equal")
}
<commit_msg>First try at mocking the HTTP bits.<commit_after>package atlas

import (
	"github.com/stretchr/testify/assert"
	"github.com/jarcoal/httpmock"
	"testing"
	"net/http"
	"encoding/json"
	"github.com/sendgrid/rest"
	"bytes"
	"log"
)

func TestCheckType(t *testing.T) {
	d := Definition{Type: "foo"}

	valid := checkType(d)
	assert.EqualValues(t, false, valid, "should be false")

	d = Definition{Type: "dns"}
	valid = checkType(d)
	assert.EqualValues(t, true, valid, "should be true")
}

func TestCheckTypeAs(t *testing.T) {
	d := Definition{Type: "dns"}
	valid := checkTypeAs(d, "foo")
	assert.EqualValues(t, false, valid, "should be false")

	valid = checkTypeAs(d, "dns")
	assert.EqualValues(t, true, valid, "should be true")
}

func TestCheckAllTypesAs(t *testing.T) {
	dl := []Definition{
		{Type: "foo"},
		{Type: "ping"},
	}

	valid := checkAllTypesAs(dl, "ping")
	assert.EqualValues(t, false, valid, "should be false")

	dl = []Definition{
		{Type: "dns"},
		{Type: "ping"},
	}
	valid = checkAllTypesAs(dl, "ping")
	assert.EqualValues(t, false, valid, "should be false")

	dl = []Definition{
		{Type: "ping"},
		{Type: "ping"},
	}
	valid = checkAllTypesAs(dl, "ping")
	assert.EqualValues(t, true, valid, "should be true")
}

func TestDNS(t *testing.T) {
	httpmock.Activate()
	defer httpmock.DeactivateAndReset()

	// mock to add a new measurement
	httpmock.RegisterResponder("POST", apiEndpoint+"/measurements/dns/",
		func(req *http.Request) (*http.Response, error) {
			var reqData MeasurementRequest
			var ap APIError
			var body bytes.Buffer
			var badType rest.Response
			var myerr error

			//respData := new(MeasurementResp)

			if err := json.NewDecoder(req.Body).Decode(&reqData); err != nil {
				return httpmock.NewStringResponse(400, ""), nil
			}

			log.Printf("test.req=%#v", reqData)
			if reqData.Definitions[0].Type != "dns" {

				ap.Error.Status = 500
				ap.Error.Code = 501
				ap.Error.Title = "Bad Type"
				ap.Error.Detail = "Type is not dns"
				badType.StatusCode = 500
				myerr = ErrInvalidMeasurementType
				if err := json.NewEncoder(&body).Encode(ap); err != nil {
					resp, _ := httpmock.NewJsonResponse(500, "argh")
					return resp, myerr
				}
			} else {
				ap.Error.Status = 200
				ap.Error.Code = 200
				ap.Error.Title = "Good Type"
				ap.Error.Detail = "Type is dns"
				badType.StatusCode = 200
				myerr = nil
			}

			if err := json.NewEncoder(&body).Encode(ap); err != nil {
				resp, _ := httpmock.NewJsonResponse(500, "argh")
				return resp, myerr
			}

			resp, err := httpmock.NewJsonResponse(200, body.String())
			if err != nil {
				return httpmock.NewStringResponse(500, ""), myerr
			}
			return resp, myerr
		},
	)

	d := []Definition{{Type: "foo"}}
	r := &MeasurementRequest{Definitions: d}

	rp, err := DNS(r)
	assert.NoError(t, err, "should be no error")
	assert.EqualValues(t, "", rp, "should be equal")
	assert.EqualValues(t, ErrInvalidMeasurementType, err, "should be equal")
}
/*
func TestNTP(t *testing.T) {
	d := []Definition{{Type: "foo"}}
	r := &MeasurementRequest{Definitions: d}

	_, err := NTP(r)
	assert.Error(t, err, "should be an error")
	assert.EqualValues(t, ErrInvalidMeasurementType, err, "should be equal")
}

func TestPing(t *testing.T) {
	d := []Definition{{Type: "foo"}}
	r := &MeasurementRequest{Definitions: d}

	_, err := Ping(r)
	assert.Error(t, err, "should be an error")
	assert.EqualValues(t, ErrInvalidMeasurementType, err, "should be equal")
}

func TestSSLCert(t *testing.T) {
	d := []Definition{{Type: "foo"}}
	r := &MeasurementRequest{Definitions: d}

	_, err := SSLCert(r)
	assert.Error(t, err, "should be an error")
	assert.EqualValues(t, ErrInvalidMeasurementType, err, "should be equal")
}

func TestTraceroute(t *testing.T) {
	d := []Definition{{Type: "foo"}}
	r := &MeasurementRequest{Definitions: d}

	_, err := Traceroute(r)
	assert.Error(t, err, "should be an error")
	assert.EqualValues(t, ErrInvalidMeasurementType, err, "should be equal")
}
*/<|endoftext|>