<commit_before>package listeners // import "github.com/docker/docker/daemon/listeners"

import (
	"crypto/tls"
	"fmt"
	"net"
	"os"
	"strconv"

	"github.com/coreos/go-systemd/v22/activation"
	"github.com/docker/docker/pkg/homedir"
	"github.com/docker/go-connections/sockets"
	"github.com/sirupsen/logrus"
)

// Init creates new listeners for the server.
// TODO: Clean up the fact that socketGroup and tlsConfig aren't always used.
func Init(proto, addr, socketGroup string, tlsConfig *tls.Config) ([]net.Listener, error) {
	ls := []net.Listener{}

	switch proto {
	case "fd":
		fds, err := listenFD(addr, tlsConfig)
		if err != nil {
			return nil, err
		}
		ls = append(ls, fds...)
	case "tcp":
		l, err := sockets.NewTCPSocket(addr, tlsConfig)
		if err != nil {
			return nil, err
		}
		ls = append(ls, l)
	case "unix":
		gid, err := lookupGID(socketGroup)
		if err != nil {
			if socketGroup != "" {
				if socketGroup != defaultSocketGroup {
					return nil, err
				}
				logrus.Warnf("could not change group %s to %s: %v", addr, defaultSocketGroup, err)
			}
			gid = os.Getgid()
		}
		l, err := sockets.NewUnixSocket(addr, gid)
		if err != nil {
			return nil, fmt.Errorf("can't create unix socket %s: %v", addr, err)
		}
		if _, err := homedir.StickRuntimeDirContents([]string{addr}); err != nil {
			// StickRuntimeDirContents returns nil error if XDG_RUNTIME_DIR is just unset
			logrus.WithError(err).Warnf("cannot set sticky bit on socket %s under XDG_RUNTIME_DIR", addr)
		}
		ls = append(ls, l)
	default:
		return nil, fmt.Errorf("invalid protocol format: %q", proto)
	}

	return ls, nil
}

// listenFD returns the specified socket activated files as a slice of
// net.Listeners or all of the activated files if "*" is given.
func listenFD(addr string, tlsConfig *tls.Config) ([]net.Listener, error) {
	var (
		err       error
		listeners []net.Listener
	)
	// socket activation
	if tlsConfig != nil {
		listeners, err = activation.TLSListeners(tlsConfig)
	} else {
		listeners, err = activation.Listeners()
	}
	if err != nil {
		return nil, err
	}

	if len(listeners) == 0 {
		return nil, fmt.Errorf("no sockets found via socket activation: make sure the service was started by systemd")
	}

	// default to all fds just like unix:// and tcp://
	if addr == "" || addr == "*" {
		return listeners, nil
	}

	fdNum, err := strconv.Atoi(addr)
	if err != nil {
		return nil, fmt.Errorf("failed to parse systemd fd address: should be a number: %v", addr)
	}
	fdOffset := fdNum - 3
	if len(listeners) < fdOffset+1 {
		return nil, fmt.Errorf("too few socket activated files passed in by systemd")
	}
	if listeners[fdOffset] == nil {
		return nil, fmt.Errorf("failed to listen on systemd activated file: fd %d", fdOffset+3)
	}
	for i, ls := range listeners {
		if i == fdOffset || ls == nil {
			continue
		}
		if err := ls.Close(); err != nil {
			return nil, fmt.Errorf("failed to close systemd activated file: fd %d: %v", fdOffset+3, err)
		}
	}
	return []net.Listener{listeners[fdOffset]}, nil
}
<commit_msg>daemon/listeners: use pkg/errors<commit_after>package listeners // import "github.com/docker/docker/daemon/listeners"

import (
	"crypto/tls"
	"net"
	"os"
	"strconv"

	"github.com/coreos/go-systemd/v22/activation"
	"github.com/docker/docker/pkg/homedir"
	"github.com/docker/go-connections/sockets"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

// Init creates new listeners for the server.
// TODO: Clean up the fact that socketGroup and tlsConfig aren't always used.
func Init(proto, addr, socketGroup string, tlsConfig *tls.Config) ([]net.Listener, error) {
	ls := []net.Listener{}

	switch proto {
	case "fd":
		fds, err := listenFD(addr, tlsConfig)
		if err != nil {
			return nil, err
		}
		ls = append(ls, fds...)
	case "tcp":
		l, err := sockets.NewTCPSocket(addr, tlsConfig)
		if err != nil {
			return nil, err
		}
		ls = append(ls, l)
	case "unix":
		gid, err := lookupGID(socketGroup)
		if err != nil {
			if socketGroup != "" {
				if socketGroup != defaultSocketGroup {
					return nil, err
				}
				logrus.Warnf("could not change group %s to %s: %v", addr, defaultSocketGroup, err)
			}
			gid = os.Getgid()
		}
		l, err := sockets.NewUnixSocket(addr, gid)
		if err != nil {
			return nil, errors.Wrapf(err, "can't create unix socket %s", addr)
		}
		if _, err := homedir.StickRuntimeDirContents([]string{addr}); err != nil {
			// StickRuntimeDirContents returns nil error if XDG_RUNTIME_DIR is just unset
			logrus.WithError(err).Warnf("cannot set sticky bit on socket %s under XDG_RUNTIME_DIR", addr)
		}
		ls = append(ls, l)
	default:
		return nil, errors.Errorf("invalid protocol format: %q", proto)
	}

	return ls, nil
}

// listenFD returns the specified socket activated files as a slice of
// net.Listeners or all of the activated files if "*" is given.
func listenFD(addr string, tlsConfig *tls.Config) ([]net.Listener, error) {
	var (
		err       error
		listeners []net.Listener
	)
	// socket activation
	if tlsConfig != nil {
		listeners, err = activation.TLSListeners(tlsConfig)
	} else {
		listeners, err = activation.Listeners()
	}
	if err != nil {
		return nil, err
	}

	if len(listeners) == 0 {
		return nil, errors.New("no sockets found via socket activation: make sure the service was started by systemd")
	}

	// default to all fds just like unix:// and tcp://
	if addr == "" || addr == "*" {
		return listeners, nil
	}

	fdNum, err := strconv.Atoi(addr)
	if err != nil {
		return nil, errors.Errorf("failed to parse systemd fd address: should be a number: %v", addr)
	}
	fdOffset := fdNum - 3
	if len(listeners) < fdOffset+1 {
		return nil, errors.New("too few socket activated files passed in by systemd")
	}
	if listeners[fdOffset] == nil {
		return nil, errors.Errorf("failed to listen on systemd activated file: fd %d", fdOffset+3)
	}
	for i, ls := range listeners {
		if i == fdOffset || ls == nil {
			continue
		}
		if err := ls.Close(); err != nil {
			return nil, errors.Wrapf(err, "failed to close systemd activated file: fd %d", fdOffset+3)
		}
	}
	return []net.Listener{listeners[fdOffset]}, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
)

const (
	inpath = "./true.in"
	outpath = "./true.out"
)

func main() {
	var content []byte
	var err error

	if content, err = ioutil.ReadFile(inpath); err != nil {
		os.Exit(1)
	}

	if err = ioutil.WriteFile(outpath, content, 0777); err != nil {
		os.Exit(1)
	}

	cmd := exec.Command(outpath)
	if err = cmd.Run(); err != nil {
		fmt.Println("Error:", err)
		os.Exit(1)
	}
}
<commit_msg>Mutate the executable a bit at a time.<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
)

const (
	inpath  = "./true.in"
	outpath = "./true.out"
)

func main() {
	content, err := ioutil.ReadFile(inpath)
	if err != nil {
		os.Exit(1)
	}

	for byt := range content {
		for bit := uint32(0); bit < 8; bit++ {
			content[byt] ^= (1 << bit)

			if err = ioutil.WriteFile(outpath, content, 0777); err != nil {
				os.Exit(1)
			}

			cmd := exec.Command(outpath)
			content[byt] ^= (1 << bit)
			if err = cmd.Run(); err != nil {
				fmt.Printf("byte = %d, bit = %d\n", byt, bit)
			}
		}
	}
}
<|endoftext|>


<commit_before>package synq

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/request"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
)

// These Amazon Web Services credentials are provided to the AWS SDK, which is
// used to upload content in multiple parts. There is no IAM user with these
// credentials; they are supplied because the AWS SDK requires some credentials
// to attempt to start uploading. This package replaces the AWS SDK's request
// signing method with its own method.
const multipartUploadAwsAccessKeyId = "AAAAAAAAAAAAAAAAAAAA"
const multipartUploadAwsSecretAccessKey = "ssssssssssssssssssssssssssssssssssssssss"

// TODO(mastensg): Determine region from bucket, or /v1/video/uploader
const multipartUploadS3BucketRegion = "us-east-1"

// UploaderSignatureUrlFormat is a printf format string which is used when
// signing multipart upload requests.
// TODO(mastensg): Determine this format (or at least prefix) at runtime, from
// the Synq HTTP API.
const UploaderSignatureUrlFormat = "https://uploader.synq.fm/uploader/signature/%s?token=%s"

// UploaderSignatureRequest is the request that is sent when using the
// embeddable web uploader's request signing service.
type UploaderSignatureRequest struct {
	Headers string `json:"headers"`
}

// UploaderSignatureResponse is the response that is received when using the
// embeddable web uploader's request signing service.
type UploaderSignatureResponse struct {
	Signature string `json:"signature"`
}

// UploaderSignature uses the backend of the embeddable web uploader to sign
// multipart upload requests.
func UploaderSignature(url_fmt, video_id, token, headers string) ([]byte, error) {
	url := fmt.Sprintf(url_fmt, video_id, token)

	// construct request body
	reqStruct := UploaderSignatureRequest{Headers: headers}
	reqBody, err := json.Marshal(reqStruct)
	if err != nil {
		return nil, err
	}

	// perform request
	resp, err := http.Post(url, "application/json", bytes.NewReader(reqBody))
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	// read response
	respBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	// parse response
	respStruct := UploaderSignatureResponse{}
	err = json.Unmarshal(respBody, &respStruct)
	if err != nil {
		return nil, err
	}

	return []byte(respStruct.Signature), nil
}

// tokenOfUploaderURL parses an uploader URL string, and returns its token
// parameter.
//
// Example:
//         const u = "https://uploader.synq.fm/uploader/" +
//         "00000000000000000000000000000000" +
//         "?token=11111111111111111111111111111111"
//
//         token, err := tokenOfUploaderURL(u)
//         if err != nil {
//                 log.Fatal(err)
//         }
//         fmt.Println(token) // prints 11111111111111111111111111111111
func tokenOfUploaderURL(uploaderURL string) (string, error) {
	u, err := url.Parse(uploaderURL)
	if err != nil {
		return "", err
	}

	values, err := url.ParseQuery(u.RawQuery)
	if err != nil {
		return "", err
	}

	token := values.Get("token")
	if token == "" {
		return "", errors.New("Found no token parameter in URL.")
	}

	return token, nil
}

// makeMultipartUploadSigner returns a function that can be added to an s3
// client's list of handlers. The function will take over signing of requests
// from aws-sdk-go.
//
// The signer function uses SYNQ's embeddable web uploader's remote procedure
// to sign requests.
//
// Example:
//         // AWS session.
//         sess := session.Must(session.NewSession())
//
//         // S3 service client.
//         svc := s3.New(sess)
//
//         signer := makeSigner()
//
//         // Register handler as the last handler of the signing phase.
//         svc.Handlers.Sign.PushBack(signer)
//
//         // S3 requests are now signed by signer().
func makeMultipartUploadSigner(acl, awsAccessKeyId, bucket, contentType, key, token, video_id string) func(r *request.Request) {
	signer := func(r *request.Request) {
		hr := r.HTTPRequest

		// rewrite the X-Amz-Date header into the format that
		// https://uploader.synq.fm/uploader/signature expects
		{
			x_amz_date_in := hr.Header.Get("X-Amz-Date")
			if x_amz_date_in == "" {
				return // TODO(mastensg): how to report errors from handlers?
			}
			x_amz_date_t, err := time.Parse("20060102T150405Z", x_amz_date_in)
			if err != nil {
				return // TODO(mastensg): how to report errors from handlers?
			}
			x_amz_date := x_amz_date_t.Format("Mon, 2 Jan 2006 15:04:05 MST")
			delete(hr.Header, "X-Amz-Date") // TODO(mastensg): enough to just set and not delete?
			hr.Header.Set("X-Amz-Date", x_amz_date)
		}

		x_amz_date := hr.Header.Get("X-Amz-Date")

		// extract url query parameters and http headers in the formats
		// that https://uploader.synq.fm/uploader/signature expects
		query := hr.URL.Path
		headers := ""
		if hr.URL.RawQuery == "uploads=" {
			// Initiate multi-part upload
			query += "?uploads"

			// TODO(mastensg): parameterize bucket name, content-type, acl
			headers = fmt.Sprintf("%s\n\n%s\n\n%s\nx-amz-date:%s\n/synqfm%s",
				hr.Method,
				"video/mp4",
				"x-amz-acl:public-read",
				x_amz_date,
				query,
			)
		} else if hr.Method == "PUT" {
			// Upload one part
			query += "?" + hr.URL.RawQuery

			// TODO(mastensg): parameterize bucket name
			headers = fmt.Sprintf("%s\n\n%s\n\nx-amz-date:%s\n/synqfm%s",
				hr.Method,
				"",
				x_amz_date,
				query,
			)
		} else if hr.Method == "POST" {
			// Finish multi-part upload
			query += "?" + hr.URL.RawQuery

			// TODO(mastensg): parameterize bucket name, content-type(?)
			headers = fmt.Sprintf("%s\n\n%s\n\nx-amz-date:%s\n/synqfm%s",
				hr.Method,
				"application/xml; charset=UTF-8",
				x_amz_date,
				query,
			)

			// TODO(mastensg): the content-type header set by
			// aws-sdk-go is not exactly the one expected by
			// uploader/signature, maybe
			hr.Header.Set("Content-Type", "application/xml; charset=UTF-8")
		} else {
			// Unknown request type
			return // TODO(mastensg): how to report errors from handlers?
		}

		signature, err := UploaderSignature(UploaderSignatureUrlFormat, video_id, token, headers)
		if err != nil {
			return // TODO(mastensg): how to report errors from handlers?
		}

		// rewrite authorization header(s)
		delete(hr.Header, "X-Amz-Content-Sha256") //TODO(mastensg): can this be left in?
		delete(hr.Header, "Authorization")
		authorization := fmt.Sprintf("AWS %s:%s", awsAccessKeyId, signature)
		hr.Header.Set("Authorization", authorization)
	}

	return signer
}

// multipartUpload uploads a file as the video's original_file.
// This procedure will use Amazon S3's Multipart Upload API:
// http://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html
//
// This is the internal function to make uploads, which is called by the public
// MultipartUpload. This function uses s3manager from aws-sdk-go to upload.
func multipartUpload(body io.Reader, acl, awsAccessKeyId, bucket, contentType, key, uploaderURL, video_id string) (*s3manager.UploadOutput, error) {
	token, err := tokenOfUploaderURL(uploaderURL)
	if err != nil {
		return nil, err
	}

	// credentials
	provider := credentials.StaticProvider{}
	provider.Value.AccessKeyID = multipartUploadAwsAccessKeyId
	provider.Value.SecretAccessKey = multipartUploadAwsSecretAccessKey
	credentials := credentials.NewCredentials(&provider)

	// session
	sess, err := session.NewSession(&aws.Config{
		Credentials: credentials,
		Region:      aws.String(multipartUploadS3BucketRegion),
	})
	if err != nil {
		return nil, err
	}

	svc := s3.New(sess)

	// sign handler
	signer := makeMultipartUploadSigner(acl, awsAccessKeyId, bucket, contentType, key, token, video_id)
	svc.Handlers.Sign.PushBack(signer)

	// s3manager uploader
	uploader := s3manager.NewUploaderWithClient(svc)

	// upload parameters
	uploadInput := &s3manager.UploadInput{
		ACL:         &acl,
		Body:        body,
		Bucket:      &bucket,
		ContentType: &contentType,
		Key:         &key,
	}

	return uploader.Upload(uploadInput)
}
<commit_msg>synq/multipart_upload: change comments. export MultipartUploadSigner<commit_after>package synq

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/request"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
)

// These Amazon Web Services credentials are provided to the AWS SDK, which is
// used to upload content in multiple parts. There is no IAM user with these
// credentials; they are supplied because the AWS SDK requires some credentials
// to attempt to start uploading. This package replaces the AWS SDK's request
// signing method with its own method.
const multipartUploadAwsAccessKeyId = "AAAAAAAAAAAAAAAAAAAA"
const multipartUploadAwsSecretAccessKey = "ssssssssssssssssssssssssssssssssssssssss"

// TODO(mastensg): Determine region from bucket, or /v1/video/uploader
const multipartUploadS3BucketRegion = "us-east-1"

// UploaderSignatureUrlFormat is a printf format string which is used when
// signing multipart upload requests.
// TODO(mastensg): Determine this format (or at least prefix) at runtime, from
// the Synq HTTP API.
const UploaderSignatureUrlFormat = "https://uploader.synq.fm/uploader/signature/%s?token=%s"

// UploaderSignatureRequest is the request that is sent when using the
// embeddable web uploader's request signing service.
type UploaderSignatureRequest struct {
	Headers string `json:"headers"`
}

// UploaderSignatureResponse is the response that is received when using the
// embeddable web uploader's request signing service.
type UploaderSignatureResponse struct {
	Signature string `json:"signature"`
}

// UploaderSignature uses the backend of the embeddable web uploader to sign
// multipart upload requests.
func UploaderSignature(url_fmt, video_id, token, headers string) ([]byte, error) {
	url := fmt.Sprintf(url_fmt, video_id, token)

	// construct request body
	reqStruct := UploaderSignatureRequest{Headers: headers}
	reqBody, err := json.Marshal(reqStruct)
	if err != nil {
		return nil, err
	}

	// perform request
	resp, err := http.Post(url, "application/json", bytes.NewReader(reqBody))
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	// read response
	respBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	// parse response
	respStruct := UploaderSignatureResponse{}
	err = json.Unmarshal(respBody, &respStruct)
	if err != nil {
		return nil, err
	}

	return []byte(respStruct.Signature), nil
}

// tokenOfUploaderURL parses an uploader URL string, and returns its token
// parameter.
//
// Example:
//         const u = "https://uploader.synq.fm/uploader/" +
//         "00000000000000000000000000000000" +
//         "?token=11111111111111111111111111111111"
//
//         token, err := tokenOfUploaderURL(u)
//         if err != nil {
//                 log.Fatal(err)
//         }
//         fmt.Println(token) // prints 11111111111111111111111111111111
func tokenOfUploaderURL(uploaderURL string) (string, error) {
	u, err := url.Parse(uploaderURL)
	if err != nil {
		return "", err
	}

	values, err := url.ParseQuery(u.RawQuery)
	if err != nil {
		return "", err
	}

	token := values.Get("token")
	if token == "" {
		return "", errors.New("Found no token parameter in URL.")
	}

	return token, nil
}

// MultipartUploadSigner returns a function that can be added to an s3 client's
// list of handlers. The function will take over signing of requests from
// aws-sdk-go.
//
// The signer function uses SYNQ's embeddable web uploader's remote procedure
// to sign requests.
//
// This function is used by the internal multipartUpload function.
//
// Example:
//         // AWS session.
//         sess := session.Must(session.NewSession())
//
//         // S3 service client.
//         svc := s3.New(sess)
//
//         // Signer function. Determine the arguments somehow.
//         signer := MultipartUploadSigner(acl, awsAccessKeyId, bucket, contentType, key, token, video_id)
//
//         // Register handler as the last handler of the signing phase.
//         svc.Handlers.Sign.PushBack(signer)
//
//         // S3 requests are now signed by signer().
func MultipartUploadSigner(acl, awsAccessKeyId, bucket, contentType, key, token, video_id string) func(r *request.Request) {
	signer := func(r *request.Request) {
		hr := r.HTTPRequest

		// rewrite the X-Amz-Date header into the format that
		// https://uploader.synq.fm/uploader/signature expects
		{
			x_amz_date_in := hr.Header.Get("X-Amz-Date")
			if x_amz_date_in == "" {
				return // TODO(mastensg): how to report errors from handlers?
			}
			x_amz_date_t, err := time.Parse("20060102T150405Z", x_amz_date_in)
			if err != nil {
				return // TODO(mastensg): how to report errors from handlers?
			}
			x_amz_date := x_amz_date_t.Format("Mon, 2 Jan 2006 15:04:05 MST")
			delete(hr.Header, "X-Amz-Date") // TODO(mastensg): enough to just set and not delete?
			hr.Header.Set("X-Amz-Date", x_amz_date)
		}

		x_amz_date := hr.Header.Get("X-Amz-Date")

		// construct "headers" string to send to
		// https://uploader.synq.fm/uploader/signature
		headers := ""
		if hr.URL.RawQuery == "uploads=" {
			// Initiate multi-part upload

			// TODO(mastensg): parameterize bucket name, content-type, acl
			headers = fmt.Sprintf("%s\n\n%s\n\n%s\nx-amz-date:%s\n/synqfm%s",
				hr.Method,
				"video/mp4",
				"x-amz-acl:public-read",
				x_amz_date,
				hr.URL.Path+"?uploads",
			)
		} else if hr.Method == "PUT" {
			// Upload one part

			// TODO(mastensg): parameterize bucket name
			headers = fmt.Sprintf("%s\n\n%s\n\nx-amz-date:%s\n/synqfm%s",
				hr.Method,
				"",
				x_amz_date,
				hr.URL.Path+"?" + hr.URL.RawQuery,
			)
		} else if hr.Method == "POST" {
			// Finish multi-part upload

			// TODO(mastensg): parameterize bucket name, content-type(?)
			headers = fmt.Sprintf("%s\n\n%s\n\nx-amz-date:%s\n/synqfm%s",
				hr.Method,
				"application/xml; charset=UTF-8",
				x_amz_date,
				hr.URL.Path+"?"+hr.URL.RawQuery,
			)

			// TODO(mastensg): the content-type header set by
			// aws-sdk-go is not exactly the one expected by
			// uploader/signature, maybe
			hr.Header.Set("Content-Type", "application/xml; charset=UTF-8")
		} else {
			// Unknown request type
			return // TODO(mastensg): how to report errors from handlers?
		}

		signature, err := UploaderSignature(UploaderSignatureUrlFormat, video_id, token, headers)
		if err != nil {
			return // TODO(mastensg): how to report errors from handlers?
		}

		// rewrite authorization header(s)
		delete(hr.Header, "X-Amz-Content-Sha256") // TODO(mastensg): can this be left in?
		delete(hr.Header, "Authorization")
		authorization := fmt.Sprintf("AWS %s:%s", awsAccessKeyId, signature)
		hr.Header.Set("Authorization", authorization)
	}

	return signer
}

// multipartUpload uploads a file as the video's original_file.
// This procedure will use Amazon S3's Multipart Upload API:
// http://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html
//
// This is the internal function to make uploads, which is called by the public
// MultipartUpload. This function uses s3manager from aws-sdk-go to manage the
// process of uploading in multiple parts. In particular, this will start
// several goroutines that will upload parts concurrently.
func multipartUpload(body io.Reader, acl, awsAccessKeyId, bucket, contentType, key, uploaderURL, video_id string) (*s3manager.UploadOutput, error) {
	token, err := tokenOfUploaderURL(uploaderURL)
	if err != nil {
		return nil, err
	}

	// credentials
	provider := credentials.StaticProvider{}
	provider.Value.AccessKeyID = multipartUploadAwsAccessKeyId
	provider.Value.SecretAccessKey = multipartUploadAwsSecretAccessKey
	credentials := credentials.NewCredentials(&provider)

	// session
	sess, err := session.NewSession(&aws.Config{
		Credentials: credentials,
		Region:      aws.String(multipartUploadS3BucketRegion),
	})
	if err != nil {
		return nil, err
	}

	svc := s3.New(sess)

	// sign handler
	signer := makeMultipartUploadSigner(acl, awsAccessKeyId, bucket, contentType, key, token, video_id)
	svc.Handlers.Sign.PushBack(signer)

	// s3manager uploader
	uploader := s3manager.NewUploaderWithClient(svc)

	// upload parameters
	uploadInput := &s3manager.UploadInput{
		ACL:         &acl,
		Body:        body,
		Bucket:      &bucket,
		ContentType: &contentType,
		Key:         &key,
	}

	return uploader.Upload(uploadInput)
}
<|endoftext|>


<commit_before>package storage

import (
	"bytes"
	"github.com/viant/endly"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/storage"
	"github.com/viant/toolbox/url"
	"io"
	"io/ioutil"
	"strings"
)

func canExpand(content []byte) bool {
	if len(content) == 0 {
		return false
	}
	limit := 100
	if limit >= len(content) {
		limit = len(content) - 1
	}
	return toolbox.IsPrintText(string(content[:limit]))
}

//NewExpandedContentHandler return a new reader that can substitute content with state map, replacement data provided in replacement map.
func NewExpandedContentHandler(context *endly.Context, replaceMap map[string]string, expand bool) func(reader io.ReadCloser) (io.ReadCloser, error) {
	return func(reader io.ReadCloser) (io.ReadCloser, error) {
		var replacted = false
		defer reader.Close()
		content, err := ioutil.ReadAll(reader)
		if err != nil {
			return nil, err
		}
		var result = string(content)
		if expand && canExpand(content) {
			result = context.Expand(result)
			if err != nil {
				return nil, err
			}
			replacted = len(result) != len(content)
		}
		for k, v := range replaceMap {
			if !replacted && strings.Contains(result, k) {
				replacted = true
			}
			result = strings.Replace(result, k, v, len(result))
		}
		if replacted {
			return ioutil.NopCloser(strings.NewReader(toolbox.AsString(result))), nil
		}
		return ioutil.NopCloser(bytes.NewReader(content)), nil
	}
}

//UseMemoryService sets flag on context to always use memory service (testing only)
func UseMemoryService(context *endly.Context) storage.Service {
	state := context.State()
	state.Put(useMemoryService, true)
	return storage.NewMemoryService()
}

//GetStorageService return toolbox storage service
func GetStorageService(context *endly.Context, resource *url.Resource) (storage.Service, error) {
	var state = context.State()
	if state.Has(useMemoryService) {
		return storage.NewMemoryService(), nil
	}
	return storage.NewServiceForURL(resource.URL, resource.Credentials)
}

//IsShellCompressable returns true if resource can be compress via shell command.
func IsShellCompressable(protScheme string) bool {
	return protScheme == "scp" || protScheme == "file"
}

//Copy transfers data for provided transfer definition.
func Copy(context *endly.Context, transfers ...*Transfer) (interface{}, error) {
	if transfers == nil {
		return nil, nil
	}
	transferService, err := context.Service(ServiceID)
	if err != nil {
		return nil, err
	}
	response := transferService.Run(context, &CopyRequest{Transfers: transfers})
	if response.Err != nil {
		return nil, response.Err
	}
	return nil, nil
}

func joinIfNeeded(parent *url.Resource, URI string) (result *url.Resource) {
	defer func() {
		if parent != nil {
			result.Credentials = parent.Credentials
		}
	}()

	if strings.Contains(URI, ":/") {
		result = url.NewResource(URI)
	} else if parent != nil {
		result = url.NewResource(toolbox.URLPathJoin(parent.URL, URI))

	} else {
		result = url.NewResource(URI)
	}
	return result
}
<commit_msg>added limit on expandable asset size<commit_after>package storage

import (
	"bytes"
	"github.com/viant/endly"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/storage"
	"github.com/viant/toolbox/url"
	"io"
	"io/ioutil"
	"strings"
)

//MaxContentSize represent max allowed expandable content size
var MaxContentSize = 1024 * 64

func canExpand(content []byte) bool {
	if len(content) == 0 {
		return false
	}
	limit := 100
	if limit >= len(content) {
		limit = len(content) - 1
	}
	return toolbox.IsPrintText(string(content[:limit]))
}

//NewExpandedContentHandler return a new reader that can substitute content with state map, replacement data provided in replacement map.
func NewExpandedContentHandler(context *endly.Context, replaceMap map[string]string, expand bool) func(reader io.ReadCloser) (io.ReadCloser, error) {
	return func(reader io.ReadCloser) (io.ReadCloser, error) {
		var replaced = false
		defer reader.Close()
		content, err := ioutil.ReadAll(reader)
		if err != nil {
			return nil, err
		}
		if len(content) > MaxContentSize {
			return ioutil.NopCloser(bytes.NewReader(content)), nil
		}

		var result = string(content)
		if expand && canExpand(content) {
			result = context.Expand(result)
			if err != nil {
				return nil, err
			}
			replaced = len(result) != len(content)
		}

		for k, v := range replaceMap {
			if !replaced && strings.Contains(result, k) {
				replaced = true
			}
			result = strings.Replace(result, k, v, len(result))
		}
		if replaced {
			return ioutil.NopCloser(strings.NewReader(toolbox.AsString(result))), nil
		}
		return ioutil.NopCloser(bytes.NewReader(content)), nil
	}
}

//UseMemoryService sets flag on context to always use memory service (testing only)
func UseMemoryService(context *endly.Context) storage.Service {
	state := context.State()
	state.Put(useMemoryService, true)
	return storage.NewMemoryService()
}

//GetStorageService return toolbox storage service
func GetStorageService(context *endly.Context, resource *url.Resource) (storage.Service, error) {
	var state = context.State()
	if state.Has(useMemoryService) {
		return storage.NewMemoryService(), nil
	}
	return storage.NewServiceForURL(resource.URL, resource.Credentials)
}

//IsShellCompressable returns true if resource can be compress via shell command.
func IsShellCompressable(protScheme string) bool {
	return protScheme == "scp" || protScheme == "file"
}

//Copy transfers data for provided transfer definition.
func Copy(context *endly.Context, transfers ...*Transfer) (interface{}, error) {
	if transfers == nil {
		return nil, nil
	}
	transferService, err := context.Service(ServiceID)
	if err != nil {
		return nil, err
	}
	response := transferService.Run(context, &CopyRequest{Transfers: transfers})
	if response.Err != nil {
		return nil, response.Err
	}
	return nil, nil
}

func joinIfNeeded(parent *url.Resource, URI string) (result *url.Resource) {
	defer func() {
		if parent != nil {
			result.Credentials = parent.Credentials
		}
	}()

	if strings.Contains(URI, ":/") {
		result = url.NewResource(URI)
	} else if parent != nil {
		result = url.NewResource(toolbox.URLPathJoin(parent.URL, URI))

	} else {
		result = url.NewResource(URI)
	}
	return result
}
<|endoftext|>


<commit_before>package pop

import (
	"fmt"
	"reflect"
	"sync"
	"time"

	"github.com/markbates/inflect"
	"github.com/pkg/errors"
	"github.com/satori/go.uuid"
)

var tableMap = map[string]string{}
var tableMapMu = sync.RWMutex{}

// MapTableName allows for the customize table mapping
// between a name and the database. For example the value
// `User{}` will automatically map to "users".
// MapTableName would allow this to change.
//
//	m := &pop.Model{Value: User{}}
//	m.TableName() // "users"
//
//	pop.MapTableName("User", "people")
//	m = &pop.Model{Value: User{}}
//	m.TableName() // "people"
func MapTableName(name string, tableName string) {
	defer tableMapMu.Unlock()
	tableMapMu.Lock()
	tableMap[name] = tableName
}

// Value is the contents of a `Model`.
type Value interface{}

// Model is used throughout Pop to wrap the end user interface
// that is passed in to many functions.
type Model struct {
	Value
	tableName string
	As        string
}

// ID returns the ID of the Model. All models must have an `ID` field this is
// of type `int`,`int64` or of type `uuid.UUID`.
func (m *Model) ID() interface{} {
	fbn, err := m.fieldByName("ID")
	if err != nil {
		return 0
	}
	if m.PrimaryKeyType() == "UUID" {
		return fbn.Interface().(uuid.UUID).String()
	}
	return fbn.Interface()
}

// PrimaryKeyType gives the primary key type of the `Model`.
func (m *Model) PrimaryKeyType() string {
	fbn, err := m.fieldByName("ID")
	if err != nil {
		return "int"
	}
	return fbn.Type().Name()
}

// TableName returns the corresponding name of the underlying database table
// for a given `Model`. See also `MapTableName` to change the default name of
// the table.
func (m *Model) TableName() string {
	if m.tableName != "" {
		return m.tableName
	}

	t := reflect.TypeOf(m.Value)
	name := m.typeName(t)

	defer tableMapMu.Unlock()
	tableMapMu.Lock()

	if tableMap[name] == "" {
		m.tableName = inflect.Tableize(name)
		tableMap[name] = m.tableName
	}
	return tableMap[name]
}

func (m *Model) typeName(t reflect.Type) string {
	if t.Kind() == reflect.Ptr {
		t = t.Elem()
	}
	switch t.Kind() {
	case reflect.String:
		return m.Value.(string)
	case reflect.Slice, reflect.Array:
		el := t.Elem()
		if el.Kind() == reflect.Ptr {
			el = el.Elem()
		}
		return el.Name()
	default:
		return t.Name()
	}
}

func (m *Model) fieldByName(s string) (reflect.Value, error) {
	el := reflect.ValueOf(m.Value).Elem()
	fbn := el.FieldByName(s)
	if !fbn.IsValid() {
		return fbn, errors.Errorf("Model does not have a field named %s", s)
	}
	return fbn, nil
}

func (m *Model) associationName() string {
	tn := inflect.Singularize(m.TableName())
	return fmt.Sprintf("%s_id", tn)
}

func (m *Model) setID(i interface{}) {
	fbn, err := m.fieldByName("ID")
	if err == nil {
		v := reflect.ValueOf(i)
		switch fbn.Kind() {
		case reflect.Int, reflect.Int64:
			fbn.SetInt(v.Int())
		default:
			fbn.Set(reflect.ValueOf(i))
		}
	}
}

func (m *Model) touchCreatedAt() {
	fbn, err := m.fieldByName("CreatedAt")
	if err == nil {
		var thetime time.Time
		if UTCTime {
			fbn.Set(reflect.ValueOf(time.Now().UTC()))
		} else {
			fbn.Set(reflect.ValueOf(time.Now()))
		}
	}
}

func (m *Model) touchUpdatedAt() {
	fbn, err := m.fieldByName("UpdatedAt")
	if err == nil {
		var thetime time.Time
		if UTCTime {
			fbn.Set(reflect.ValueOf(time.Now().UTC()))
		} else {
			fbn.Set(reflect.ValueOf(time.Now()))
		}
	}
}

func (m *Model) whereID() string {
	id := m.ID()
	var value string
	switch id.(type) {
	case int, int64:
		value = fmt.Sprintf("%s.id = %d", m.TableName(), id)
	default:
		value = fmt.Sprintf("%s.id ='%s'", m.TableName(), id)
	}
	return value
}
<commit_msg>removed unused<commit_after>package pop

import (
	"fmt"
	"reflect"
	"sync"
	"time"

	"github.com/markbates/inflect"
	"github.com/pkg/errors"
	"github.com/satori/go.uuid"
)

var tableMap = map[string]string{}
var tableMapMu = sync.RWMutex{}

// MapTableName allows for the customize table mapping
// between a name and the database. For example the value
// `User{}` will automatically map to "users".
// MapTableName would allow this to change.
//
//	m := &pop.Model{Value: User{}}
//	m.TableName() // "users"
//
//	pop.MapTableName("User", "people")
//	m = &pop.Model{Value: User{}}
//	m.TableName() // "people"
func MapTableName(name string, tableName string) {
	defer tableMapMu.Unlock()
	tableMapMu.Lock()
	tableMap[name] = tableName
}

// Value is the contents of a `Model`.
type Value interface{}

// Model is used throughout Pop to wrap the end user interface
// that is passed in to many functions.
type Model struct {
	Value
	tableName string
	As        string
}

// ID returns the ID of the Model. All models must have an `ID` field this is
// of type `int`,`int64` or of type `uuid.UUID`.
func (m *Model) ID() interface{} {
	fbn, err := m.fieldByName("ID")
	if err != nil {
		return 0
	}
	if m.PrimaryKeyType() == "UUID" {
		return fbn.Interface().(uuid.UUID).String()
	}
	return fbn.Interface()
}

// PrimaryKeyType gives the primary key type of the `Model`.
func (m *Model) PrimaryKeyType() string {
	fbn, err := m.fieldByName("ID")
	if err != nil {
		return "int"
	}
	return fbn.Type().Name()
}

// TableName returns the corresponding name of the underlying database table
// for a given `Model`. See also `MapTableName` to change the default name of
// the table.
func (m *Model) TableName() string {
	if m.tableName != "" {
		return m.tableName
	}

	t := reflect.TypeOf(m.Value)
	name := m.typeName(t)

	defer tableMapMu.Unlock()
	tableMapMu.Lock()

	if tableMap[name] == "" {
		m.tableName = inflect.Tableize(name)
		tableMap[name] = m.tableName
	}
	return tableMap[name]
}

func (m *Model) typeName(t reflect.Type) string {
	if t.Kind() == reflect.Ptr {
		t = t.Elem()
	}
	switch t.Kind() {
	case reflect.String:
		return m.Value.(string)
	case reflect.Slice, reflect.Array:
		el := t.Elem()
		if el.Kind() == reflect.Ptr {
			el = el.Elem()
		}
		return el.Name()
	default:
		return t.Name()
	}
}

func (m *Model) fieldByName(s string) (reflect.Value, error) {
	el := reflect.ValueOf(m.Value).Elem()
	fbn := el.FieldByName(s)
	if !fbn.IsValid() {
		return fbn, errors.Errorf("Model does not have a field named %s", s)
	}
	return fbn, nil
}

func (m *Model) associationName() string {
	tn := inflect.Singularize(m.TableName())
	return fmt.Sprintf("%s_id", tn)
}

func (m *Model) setID(i interface{}) {
	fbn, err := m.fieldByName("ID")
	if err == nil {
		v := reflect.ValueOf(i)
		switch fbn.Kind() {
		case reflect.Int, reflect.Int64:
			fbn.SetInt(v.Int())
		default:
			fbn.Set(reflect.ValueOf(i))
		}
	}
}

func (m *Model) touchCreatedAt() {
	fbn, err := m.fieldByName("CreatedAt")
	if err == nil {
		if UTCTime {
			fbn.Set(reflect.ValueOf(time.Now().UTC()))
		} else {
			fbn.Set(reflect.ValueOf(time.Now()))
		}
	}
}

func (m *Model) touchUpdatedAt() {
	fbn, err := m.fieldByName("UpdatedAt")
	if err == nil {
		if UTCTime {
			fbn.Set(reflect.ValueOf(time.Now().UTC()))
		} else {
			fbn.Set(reflect.ValueOf(time.Now()))
		}
	}
}

func (m *Model) whereID() string {
	id := m.ID()
	var value string
	switch id.(type) {
	case int, int64:
		value = fmt.Sprintf("%s.id = %d", m.TableName(), id)
	default:
		value = fmt.Sprintf("%s.id ='%s'", m.TableName(), id)
	}
	return value
}
<|endoftext|>


<commit_before>// Package panos interacts with Palo Alto and Panorama devices using the XML API.
package panos

import (
	"encoding/xml"
	"fmt"
	"github.com/scottdware/go-rested"
	"strings"
)

// PaloAlto is a container for our session state.
type PaloAlto struct {
	Host            string
	Key             string
	URI             string
	Platform        string
	Model           string
	Serial          string
	SoftwareVersion string
	DeviceType      string
	Panorama        bool
}

// DeviceGroups lists all of the device-group's in Panorama.
type DeviceGroups struct {
	XMLName      xml.Name      `xml:"response"`
	Status       string        `xml:"status,attr"`
	Code         string        `xml:"code,attr"`
	DeviceGroups []DeviceGroup `xml:"result>device-group>entry"`
}

// DeviceGroup contains information about each individual device-group.
type DeviceGroup struct {
	Name    string   `xml:"name,attr"`
	Devices []Serial `xml:"devices>entry"`
}

// Serial contains the serial number of each device in the device-group.
type Serial struct {
	Serial string `xml:"name,attr"`
}

// Tags contains information about all tags on the system.
type Tags struct {
	Tags []Tag
}

// Tag contains information about each individual tag.
type Tag struct {
	Name     string
	Color    string
	Comments string
}

// xmlTags is used for parsing all tags on the system.
type xmlTags struct {
	XMLName xml.Name `xml:"response"`
	Status  string   `xml:"status,attr"`
	Code    string   `xml:"code,attr"`
	Tags    []xmlTag `xml:"result>tag>entry"`
}

// xmlTag is used for parsing each individual tag.
type xmlTag struct {
	Name     string `xml:"name,attr"`
	Color    string `xml:"color,omitempty"`
	Comments string `xml:"comments,omitempty"`
}

// authKey holds our API key.
type authKey struct {
	XMLName xml.Name `xml:"response"`
	Status  string   `xml:"status,attr"`
	Code    string   `xml:"code,attr"`
	Key     string   `xml:"result>key"`
}

// systemInfo holds basic system information.
type systemInfo struct {
	XMLName         xml.Name `xml:"response"`
	Status          string   `xml:"status,attr"`
	Code            string   `xml:"code,attr"`
	Platform        string   `xml:"result>system>platform-family"`
	Model           string   `xml:"result>system>model"`
	Serial          string   `xml:"result>system>serial"`
	SoftwareVersion string   `xml:"result>system>sw-version"`
}

// panoramaStatus gets the connection status to Panorama.
type panoramaStatus struct {
	XMLName xml.Name `xml:"response"`
	Status  string   `xml:"status,attr"`
	Code    string   `xml:"code,attr"`
	Data    string   `xml:"result"`
}

// requestError contains information about any error we get from a request.
type requestError struct {
	XMLName xml.Name `xml:"response"`
	Status  string   `xml:"status,attr"`
	Code    string   `xml:"code,attr"`
	Message string   `xml:"result>msg,omitempty"`
}

var (
	headers = map[string]string{
		"Content-Type": "application/xml",
	}
	tagColors = map[string]string{
		"color1":  "Red",
		"color2":  "Green",
		"color3":  "Blue",
		"color4":  "Yellow",
		"color5":  "Copper",
		"color6":  "Orange",
		"color7":  "Purple",
		"color8":  "Gray",
		"color9":  "Light Green",
		"color10": "Cyan",
		"color11": "Light Gray",
		"color12": "Blue Gray",
		"color13": "Lime",
		"color14": "Black",
		"color15": "Gold",
		"color16": "Brown",
	}
	errorCodes = map[string]string{
		"400": "Bad request - Returned when a required parameter is missing, an illegal parameter value is used",
		"403": "Forbidden - Returned for authentication or authorization errors including invalid key, insufficient admin access rights",
		"1":   "Unknown command - The specific config or operational command is not recognized",
		"2":   "Internal error - Check with technical support when seeing these errors",
		"3":   "Internal error - Check with technical support when seeing these errors",
		"4":   "Internal error - Check with technical support when seeing these errors",
		"5":   "Internal error - Check with technical support when seeing these errors",
		"6":   "Bad Xpath - The xpath specified in one or more attributes of the command is invalid. Check the API browser for proper xpath values",
		"7":   "Object not present - Object specified by the xpath is not present. For example, entry[@name=’value’] where no object with name ‘value’ is present",
		"8":   "Object not unique - For commands that operate on a single object, the specified object is not unique",
		"9":   "Internal error - Check with technical support when seeing these errors",
		"10":  "Reference count not zero - Object cannot be deleted as there are other objects that refer to it. For example, address object still in use in policy",
		"11":  "Internal error - Check with technical support when seeing these errors",
		"12":  "Invalid object - Xpath or element values provided are not complete",
		"13":  "Operation failed - A descriptive error message is returned in the response",
		"14":  "Operation not possible - Operation is not possible. For example, moving a rule up one position when it is already at the top",
		"15":  "Operation denied - For example, Admin not allowed to delete own account, Running a command that is not allowed on a passive device",
		"16":  "Unauthorized - The API role does not have access rights to run this query",
		"17":  "Invalid command - Invalid command or parameters",
		"18":  "Malformed command - The XML is malformed",
		"19":  "Success - Command completed successfully",
		"20":  "Success - Command completed successfully",
		"21":  "Internal error - Check with technical support when seeing these errors",
		"22":  "Session timed out - The session for this query timed out",
	}
)

// NewSession sets up our connection to the Palo Alto firewall system.
func NewSession(host, user, passwd string) (*PaloAlto, error) {
	var key authKey
	var info systemInfo
	var pan panoramaStatus
	status := false
	deviceType := "panos"
	r := rested.NewRequest()

	resp := r.Send("get", fmt.Sprintf("https://%s/api/?type=keygen&user=%s&password=%s", host, user, passwd), nil, nil, nil)
	if resp.Error != nil {
		return nil, resp.Error
	}

	err := xml.Unmarshal(resp.Body, &key)
	if err != nil {
		return nil, err
	}

	if key.Status != "success" {
		return nil, fmt.Errorf("error code %s: %s (keygen)", key.Code, errorCodes[key.Code])
	}

	uri := fmt.Sprintf("https://%s/api/?", host)
	getInfo := r.Send("get", fmt.Sprintf("%s&key=%s&type=op&cmd=<show><system><info></info></system></show>", uri, key.Key), nil, nil, nil)

	if getInfo.Error != nil {
		return nil, getInfo.Error
	}

	err = xml.Unmarshal(getInfo.Body, &info)
	if err != nil {
		return nil, err
	}

	if info.Status != "success" {
		return nil, fmt.Errorf("error code %s: %s (show system info)", info.Code, errorCodes[info.Code])
	}

	panStatus := r.Send("get", fmt.Sprintf("%s&key=%s&type=op&cmd=<show><panorama-status></panorama-status></show>", uri, key.Key), nil, nil, nil)
	if panStatus.Error != nil {
		return nil, panStatus.Error
	}

	err = xml.Unmarshal(panStatus.Body, &pan)
	if err != nil {
		return nil, err
	}

	if info.Platform == "m" {
		deviceType = "panorama"
	}

	if strings.Contains(pan.Data, ": yes") {
		status = true
	}

	return &PaloAlto{
		Host:            host,
		Key:             key.Key,
		URI:             fmt.Sprintf("https://%s/api/?", host),
		Platform:        info.Platform,
		Model:           info.Model,
		Serial:          info.Serial,
		SoftwareVersion: info.SoftwareVersion,
		DeviceType:      deviceType,
		Panorama:        status,
	}, nil
}

// DeviceGroups returns information about all of the device-groups in Panorama.
func (p *PaloAlto) DeviceGroups() (*DeviceGroups, error) {
	var devices DeviceGroups
	var xpath string
	// xpath := "/config/devices/entry/vsys/entry/address"
	r := rested.NewRequest()

	if p.DeviceType == "panorama" {
		// xpath = "/config/devices/entry/device-group/entry/address"
		xpath = "/config/devices/entry//device-group"
	}

	query := map[string]string{
		"type":   "config",
		"action": "get",
		"xpath":  xpath,
		"key":    p.Key,
	}
	devData := r.Send("get", p.URI, nil, headers, query)

	if err := xml.Unmarshal(devData.Body, &devices); err != nil {
		return nil, err
	}

	if devices.Status != "success" {
		return nil, fmt.Errorf("error code %s: %s", devices.Code, errorCodes[devices.Code])
	}

	return &devices, nil
}

// Tags returns information about all tags on the system.
func (p *PaloAlto) Tags() (*Tags, error) {
	var parsedTags xmlTags
	var tags Tags
	xpath := "/config/devices/entry//tag"
	// xpath := "/config/devices/entry/vsys/entry/tag"
	r := rested.NewRequest()

	if p.DeviceType == "panos" && p.Panorama == true {
		xpath = "/config/panorama//tag"
	}

	if p.DeviceType == "panorama" {
		// xpath = "/config/devices/entry/device-group/entry/tag"
		xpath = "/config/devices/entry//tag"
	}

	query := map[string]string{
		"type":   "config",
		"action": "get",
		"xpath":  xpath,
		"key":    p.Key,
	}
	tData := r.Send("get", p.URI, nil, headers, query)

	if err := xml.Unmarshal(tData.Body, &parsedTags); err != nil {
		return nil, err
	}

	if parsedTags.Status != "success" {
		return nil, fmt.Errorf("error code %s: %s", parsedTags.Code, errorCodes[parsedTags.Code])
	}

	for _, t := range parsedTags.Tags {
		tname := t.Name
		tcolor := tagColors[t.Color]
		tcomments := t.Comments

		tags.Tags = append(tags.Tags, Tag{Name: tname, Color: tcolor, Comments: tcomments})
	}

	return &tags, nil

}

// Commit issues a commit on the device. When issuing a commit against a Panorama device, the configuration
// will only be committed to Panorama, and not specific device groups.
func (p *PaloAlto) Commit() error {
	var reqError requestError
	cmd := "<commit></commit>"
	r := rested.NewRequest()

	query := map[string]string{
		"type": "commit",
		"cmd":  cmd,
		"key":  p.Key,
	}

	resp := r.Send("get", p.URI, nil, nil, query)
	if resp.Error != nil {
		return resp.Error
	}

	if err := xml.Unmarshal(resp.Body, &reqError); err != nil {
		return err
	}

	if reqError.Status != "success" {
		return fmt.Errorf("error code %s: %s", reqError.Code, errorCodes[reqError.Code])
	}

	return nil
}
<commit_msg>Added CommitAll function<commit_after>// Package panos interacts with Palo Alto and Panorama devices using the XML API.
package panos

import (
	"encoding/xml"
	"fmt"
	"github.com/scottdware/go-rested"
	"strings"
)

// PaloAlto is a container for our session state.
type PaloAlto struct {
	Host            string
	Key             string
	URI             string
	Platform        string
	Model           string
	Serial          string
	SoftwareVersion string
	DeviceType      string
	Panorama        bool
}

// DeviceGroups lists all of the device-group's in Panorama.
type DeviceGroups struct {
	XMLName      xml.Name      `xml:"response"`
	Status       string        `xml:"status,attr"`
	Code         string        `xml:"code,attr"`
	DeviceGroups []DeviceGroup `xml:"result>device-group>entry"`
}

// DeviceGroup contains information about each individual device-group.
type DeviceGroup struct {
	Name    string   `xml:"name,attr"`
	Devices []Serial `xml:"devices>entry"`
}

// Serial contains the serial number of each device in the device-group.
type Serial struct {
	Serial string `xml:"name,attr"`
}

// Tags contains information about all tags on the system.
type Tags struct {
	Tags []Tag
}

// Tag contains information about each individual tag.
type Tag struct {
	Name     string
	Color    string
	Comments string
}

// xmlTags is used for parsing all tags on the system.
type xmlTags struct {
	XMLName xml.Name `xml:"response"`
	Status  string   `xml:"status,attr"`
	Code    string   `xml:"code,attr"`
	Tags    []xmlTag `xml:"result>tag>entry"`
}

// xmlTag is used for parsing each individual tag.
type xmlTag struct {
	Name     string `xml:"name,attr"`
	Color    string `xml:"color,omitempty"`
	Comments string `xml:"comments,omitempty"`
}

// authKey holds our API key.
type authKey struct {
	XMLName xml.Name `xml:"response"`
	Status  string   `xml:"status,attr"`
	Code    string   `xml:"code,attr"`
	Key     string   `xml:"result>key"`
}

// systemInfo holds basic system information.
type systemInfo struct {
	XMLName         xml.Name `xml:"response"`
	Status          string   `xml:"status,attr"`
	Code            string   `xml:"code,attr"`
	Platform        string   `xml:"result>system>platform-family"`
	Model           string   `xml:"result>system>model"`
	Serial          string   `xml:"result>system>serial"`
	SoftwareVersion string   `xml:"result>system>sw-version"`
}

// panoramaStatus gets the connection status to Panorama.
type panoramaStatus struct {
	XMLName xml.Name `xml:"response"`
	Status  string   `xml:"status,attr"`
	Code    string   `xml:"code,attr"`
	Data    string   `xml:"result"`
}

// requestError contains information about any error we get from a request.
type requestError struct {
	XMLName xml.Name `xml:"response"`
	Status  string   `xml:"status,attr"`
	Code    string   `xml:"code,attr"`
	Message string   `xml:"result>msg,omitempty"`
}

var (
	headers = map[string]string{
		"Content-Type": "application/xml",
	}
	tagColors = map[string]string{
		"color1":  "Red",
		"color2":  "Green",
		"color3":  "Blue",
		"color4":  "Yellow",
		"color5":  "Copper",
		"color6":  "Orange",
		"color7":  "Purple",
		"color8":  "Gray",
		"color9":  "Light Green",
		"color10": "Cyan",
		"color11": "Light Gray",
		"color12": "Blue Gray",
		"color13": "Lime",
		"color14": "Black",
		"color15": "Gold",
		"color16": "Brown",
	}
	errorCodes = map[string]string{
		"400": "Bad request - Returned when a required parameter is missing, an illegal parameter value is used",
		"403": "Forbidden - Returned for authentication or authorization errors including invalid key, insufficient admin access rights",
		"1":   "Unknown command - The specific config or operational command is not recognized",
		"2":   "Internal error - Check with technical support when seeing these errors",
		"3":   "Internal error - Check with technical support when seeing these errors",
		"4":   "Internal error - Check with technical support when seeing these errors",
		"5":   "Internal error - Check with technical support when seeing these errors",
		"6":   "Bad Xpath - The xpath specified in one or more attributes of the command is invalid. Check the API browser for proper xpath values",
		"7":   "Object not present - Object specified by the xpath is not present. For example, entry[@name=’value’] where no object with name ‘value’ is present",
		"8":   "Object not unique - For commands that operate on a single object, the specified object is not unique",
		"9":   "Internal error - Check with technical support when seeing these errors",
		"10":  "Reference count not zero - Object cannot be deleted as there are other objects that refer to it. For example, address object still in use in policy",
		"11":  "Internal error - Check with technical support when seeing these errors",
		"12":  "Invalid object - Xpath or element values provided are not complete",
		"13":  "Operation failed - A descriptive error message is returned in the response",
		"14":  "Operation not possible - Operation is not possible. For example, moving a rule up one position when it is already at the top",
		"15":  "Operation denied - For example, Admin not allowed to delete own account, Running a command that is not allowed on a passive device",
		"16":  "Unauthorized - The API role does not have access rights to run this query",
		"17":  "Invalid command - Invalid command or parameters",
		"18":  "Malformed command - The XML is malformed",
		"19":  "Success - Command completed successfully",
		"20":  "Success - Command completed successfully",
		"21":  "Internal error - Check with technical support when seeing these errors",
		"22":  "Session timed out - The session for this query timed out",
	}
)

// NewSession sets up our connection to the Palo Alto firewall system.
func NewSession(host, user, passwd string) (*PaloAlto, error) {
	var key authKey
	var info systemInfo
	var pan panoramaStatus
	status := false
	deviceType := "panos"
	r := rested.NewRequest()

	resp := r.Send("get", fmt.Sprintf("https://%s/api/?type=keygen&user=%s&password=%s", host, user, passwd), nil, nil, nil)
	if resp.Error != nil {
		return nil, resp.Error
	}

	err := xml.Unmarshal(resp.Body, &key)
	if err != nil {
		return nil, err
	}

	if key.Status != "success" {
		return nil, fmt.Errorf("error code %s: %s (keygen)", key.Code, errorCodes[key.Code])
	}

	uri := fmt.Sprintf("https://%s/api/?", host)
	getInfo := r.Send("get", fmt.Sprintf("%s&key=%s&type=op&cmd=<show><system><info></info></system></show>", uri, key.Key), nil, nil, nil)

	if getInfo.Error != nil {
		return nil, getInfo.Error
	}

	err = xml.Unmarshal(getInfo.Body, &info)
	if err != nil {
		return nil, err
	}

	if info.Status != "success" {
		return nil, fmt.Errorf("error code %s: %s (show system info)", info.Code, errorCodes[info.Code])
	}

	panStatus := r.Send("get", fmt.Sprintf("%s&key=%s&type=op&cmd=<show><panorama-status></panorama-status></show>", uri, key.Key), nil, nil, nil)
	if panStatus.Error != nil {
		return nil, panStatus.Error
	}

	err = xml.Unmarshal(panStatus.Body, &pan)
	if err != nil {
		return nil, err
	}

	if info.Platform == "m" {
		deviceType = "panorama"
	}

	if strings.Contains(pan.Data, ": yes") {
		status = true
	}

	return &PaloAlto{
		Host:            host,
		Key:             key.Key,
		URI:             fmt.Sprintf("https://%s/api/?", host),
		Platform:        info.Platform,
		Model:           info.Model,
		Serial:          info.Serial,
		SoftwareVersion: info.SoftwareVersion,
		DeviceType:      deviceType,
		Panorama:        status,
	}, nil
}

// DeviceGroups returns information about all of the device-groups in Panorama.
func (p *PaloAlto) DeviceGroups() (*DeviceGroups, error) {
	var devices DeviceGroups
	var xpath string
	// xpath := "/config/devices/entry/vsys/entry/address"
	r := rested.NewRequest()

	if p.DeviceType == "panorama" {
		// xpath = "/config/devices/entry/device-group/entry/address"
		xpath = "/config/devices/entry//device-group"
	}

	query := map[string]string{
		"type":   "config",
		"action": "get",
		"xpath":  xpath,
		"key":    p.Key,
	}
	devData := r.Send("get", p.URI, nil, headers, query)

	if err := xml.Unmarshal(devData.Body, &devices); err != nil {
		return nil, err
	}

	if devices.Status != "success" {
		return nil, fmt.Errorf("error code %s: %s", devices.Code, errorCodes[devices.Code])
	}

	return &devices, nil
}

// Tags returns information about all tags on the system.
func (p *PaloAlto) Tags() (*Tags, error) {
	var parsedTags xmlTags
	var tags Tags
	xpath := "/config/devices/entry//tag"
	// xpath := "/config/devices/entry/vsys/entry/tag"
	r := rested.NewRequest()

	if p.DeviceType == "panos" && p.Panorama == true {
		xpath = "/config/panorama//tag"
	}

	if p.DeviceType == "panorama" {
		// xpath = "/config/devices/entry/device-group/entry/tag"
		xpath = "/config/devices/entry//tag"
	}

	query := map[string]string{
		"type":   "config",
		"action": "get",
		"xpath":  xpath,
		"key":    p.Key,
	}
	tData := r.Send("get", p.URI, nil, headers, query)

	if err := xml.Unmarshal(tData.Body, &parsedTags); err != nil {
		return nil, err
	}

	if parsedTags.Status != "success" {
		return nil, fmt.Errorf("error code %s: %s", parsedTags.Code, errorCodes[parsedTags.Code])
	}

	for _, t := range parsedTags.Tags {
		tname := t.Name
		tcolor := tagColors[t.Color]
		tcomments := t.Comments

		tags.Tags = append(tags.Tags, Tag{Name: tname, Color: tcolor, Comments: tcomments})
	}

	return &tags, nil

}

// Commit issues a commit on the device. When issuing a commit against a Panorama device,
// the configuration will only be committed to Panorama, and not an individual device-group.
func (p *PaloAlto) Commit() error {
	var reqError requestError
	r := rested.NewRequest()

	query := map[string]string{
		"type": "commit",
		"cmd":  "<commit></commit>",
		"key":  p.Key,
	}

	resp := r.Send("get", p.URI, nil, nil, query)
	if resp.Error != nil {
		return resp.Error
	}

	if err := xml.Unmarshal(resp.Body, &reqError); err != nil {
		return err
	}

	if reqError.Status != "success" {
		return fmt.Errorf("error code %s: %s", reqError.Code, errorCodes[reqError.Code])
	}

	return nil
}

// CommitAll issues a commit to a Panorama device, with the given 'devicegroup.' You can (optionally) specify
// individual devices within that device group by adding each serial number as an additional parameter.
func (p *PaloAlto) CommitAll(devicegroup string, devices ...string) error {
	var reqError requestError
	var cmd string

	r := rested.NewRequest()

	if p.DeviceType == "panorama" && len(devices) <= 0 {
		cmd = fmt.Sprintf("<commit-all><shared-policy><device-group><name>%s</name></device-group></shared-policy></commit-all>", devicegroup)
	}

	if p.DeviceType == "panorama" && len(devices) > 0 {
		cmd = fmt.Sprintf("<commit-all><shared-policy><device-group><name>%s</name><devices>", devicegroup)

		for _, d := range devices {
			cmd += fmt.Sprintf("<entry name=\"%s\"/>", d)
		}

		cmd += "</devices></device-group></shared-policy></commit-all>"
	}

	query := map[string]string{
		"type":   "commit",
		"action": "all",
		"cmd":    cmd,
		"key":    p.Key,
	}

	resp := r.Send("get", p.URI, nil, nil, query)
	if resp.Error != nil {
		return resp.Error
	}

	if err := xml.Unmarshal(resp.Body, &reqError); err != nil {
		return err
	}

	if reqError.Status != "success" {
		return fmt.Errorf("error code %s: %s", reqError.Code, errorCodes[reqError.Code])
	}

	return nil
}
<|endoftext|>


<commit_before>/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
    "fmt"
    "github.com/sbinet/go-clang"
)

func getSymbolFromCursor(cursor *clang.Cursor) *Symbol {
    f, line, col, _ := cursor.Location().GetFileLocation()
    fName := f.Name()
    return &Symbol{cursor.Spelling(), cursor.USR(), fName, int(line), int(col)}
}

func Parse(file string, db *SymbolsDB) {
    // files already inserted in the DB from this parsing
    insertedFiles := map[string]bool{ file: true }

    // insert symbols
    idx := clang.NewIndex(0, 0)
    defer idx.Dispose()

    args := []string{}

    tu := idx.Parse(file, args, nil, clang.TU_DetailedPreprocessingRecord)
    defer tu.Dispose()

    visitNode := func (cursor, parent clang.Cursor) clang.ChildVisitResult {
        if cursor.IsNull() {
            return clang.CVR_Continue
        }

        f, line, col, _ := cursor.Location().GetFileLocation()
        fName := f.Name()

        if fName == "" {
            // ignore system code
            return clang.CVR_Continue
        }

        /* TODO: erase! this is not required */
        fmt.Printf("%s: %s (%s)\n",
            cursor.Kind().Spelling(), cursor.Spelling(), cursor.USR())
        fmt.Println(fName, ":", line, col)
        /*******************/

        if !insertedFiles[fName] {
            db.NeedToProcessFile(fName)
            insertedFiles[fName] = true
        }

        switch cursor.Kind() {
        case clang.CK_FunctionDecl:
            dec := getSymbolFromCursor(&cursor)
            defCursor := cursor.DefinitionCursor()
            if !defCursor.IsNull() {
                def := getSymbolFromCursor(&defCursor)
                fmt.Println("declaration ", dec)
                fmt.Println("definition ", def)
                db.InsertFuncSymb(dec, def)
            } else {
                db.InsertSymbol(dec)
            }
        case clang.CK_VarDecl:
            dec := getSymbolFromCursor(&cursor)
            db.InsertSymbol(dec)
        case clang.CK_ParmDecl:
            if cursor.Spelling() != "" {
                dec := getSymbolFromCursor(&cursor)
                db.InsertParamDecl(dec)
            }
        case clang.CK_CallExpr:
            call := getSymbolFromCursor(&cursor)
            decCursor := cursor.Referenced()
            dec := getSymbolFromCursor(&decCursor)
            db.InsertFuncCall(call, dec)
        case clang.CK_DeclRefExpr:
            use := getSymbolFromCursor(&cursor)
            decCursor := cursor.Referenced()
            dec := getSymbolFromCursor(&decCursor)
            db.InsertSymbolUse(use, dec)
        }

        /* TODO: eventually we need to continue on some cases for faster run */
        return clang.CVR_Recurse
    }

    tu.ToCursor().Visit(visitNode)
}
<commit_msg>Removed debugging printing accidentally introduced in last commit<commit_after>/*
 * Copyright 2015 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
    "fmt"
    "github.com/sbinet/go-clang"
)

func getSymbolFromCursor(cursor *clang.Cursor) *Symbol {
    f, line, col, _ := cursor.Location().GetFileLocation()
    fName := f.Name()
    return &Symbol{cursor.Spelling(), cursor.USR(), fName, int(line), int(col)}
}

func Parse(file string, db *SymbolsDB) {
    // files already inserted in the DB from this parsing
    insertedFiles := map[string]bool{ file: true }

    // insert symbols
    idx := clang.NewIndex(0, 0)
    defer idx.Dispose()

    args := []string{}

    tu := idx.Parse(file, args, nil, clang.TU_DetailedPreprocessingRecord)
    defer tu.Dispose()

    visitNode := func (cursor, parent clang.Cursor) clang.ChildVisitResult {
        if cursor.IsNull() {
            return clang.CVR_Continue
        }

        f, line, col, _ := cursor.Location().GetFileLocation()
        fName := f.Name()

        if fName == "" {
            // ignore system code
            return clang.CVR_Continue
        }

        /* TODO: erase! this is not required */
        fmt.Printf("%s: %s (%s)\n",
            cursor.Kind().Spelling(), cursor.Spelling(), cursor.USR())
        fmt.Println(fName, ":", line, col)
        /*******************/

        if !insertedFiles[fName] {
            db.NeedToProcessFile(fName)
            insertedFiles[fName] = true
        }

        switch cursor.Kind() {
        case clang.CK_FunctionDecl:
            dec := getSymbolFromCursor(&cursor)
            defCursor := cursor.DefinitionCursor()
            if !defCursor.IsNull() {
                def := getSymbolFromCursor(&defCursor)
                db.InsertFuncSymb(dec, def)
            } else {
                db.InsertSymbol(dec)
            }
        case clang.CK_VarDecl:
            dec := getSymbolFromCursor(&cursor)
            db.InsertSymbol(dec)
        case clang.CK_ParmDecl:
            if cursor.Spelling() != "" {
                dec := getSymbolFromCursor(&cursor)
                db.InsertParamDecl(dec)
            }
        case clang.CK_CallExpr:
            call := getSymbolFromCursor(&cursor)
            decCursor := cursor.Referenced()
            dec := getSymbolFromCursor(&decCursor)
            db.InsertFuncCall(call, dec)
        case clang.CK_DeclRefExpr:
            use := getSymbolFromCursor(&cursor)
            decCursor := cursor.Referenced()
            dec := getSymbolFromCursor(&decCursor)
            db.InsertSymbolUse(use, dec)
        }

        /* TODO: eventually we need to continue on some cases for faster run */
        return clang.CVR_Recurse
    }

    tu.ToCursor().Visit(visitNode)
}
<|endoftext|>


<commit_before>package set

type Set struct {
	elements map[interface{}]struct{}
}

// New creates and returns a new, empty set.
func New(xs ...interface{}) *Set {
	n := make(map[interface{}]struct{})
	for _, x := range xs {
		n[x] = struct{}{}
	}
	return &Set{n}
}

// Copy returns a distinct copy of `s`.
func (s *Set) Copy() *Set {
	x := New()
	for k, _ := range s.elements {
		x.elements[k] = struct{}{}
	}

	return x
}

// Add adds the element `x` to the set `s` and returns true if `x` was added.
func (s *Set) Add(x interface{}) bool {
	if _, exists := s.elements[x]; !exists {
		s.elements[x] = struct{}{}
		return true
	}
	return false
}

// Delete removes the element `x` from the set `s` and returns true if `x` was found.
func (s *Set) Delete(x interface{}) bool {
	if _, exists := s.elements[x]; exists {
		delete(s.elements, x)
		return true
	}
	return false
}

// Member returns true if `x` is contained in `s`.
func (s *Set) Member(x interface{}) bool {
	_, exists := s.elements[x]
	return exists
}

// Merge adds the elements in `x` to `s`.
func (s *Set) Merge(x *Set) {
	for k, _ := range x.elements {
		s.elements[k] = struct{}{}
	}
}

// Discard removes the elements of `x` from `s`.
func (s *Set) Discard(x *Set) {
	for k, _ := range x.elements {
		delete(s.elements, k)
	}
}

// Difference returns a new set containing the difference between `s` and `x`.
func (s *Set) Difference(x *Set) *Set {
	n := s.Copy()
	for k, _ := range x.elements {
		delete(n.elements, k)
	}

	return n
}

// Union returns a new set composed of the union of `s` and `x`.
func (s *Set) Union(x *Set) *Set {
	n := New()
	for k, _ := range s.elements {
		n.elements[k] = struct{}{}
	}

	for k, _ := range x.elements {
		n.elements[k] = struct{}{}
	}

	return n
}

// Intersection returns a new set composed of the intersection between `s` and `x`.
func (s *Set) Intersection(x *Set) *Set {
	n := New()
	for k1, _ := range x.elements {
		for k2, _ := range s.elements {
			if k1 == k2 {
				n.elements[k1] = struct{}{}
			}
		}
	}

	return n
}

// Superset returns true if `s` is a superset of `x`.
func (s *Set) Superset(x *Set) bool {
	return x.Subset(s)
}

// Subset returns true of `s` is a subset of `x`.
func (s *Set) Subset(x *Set) bool {
	for k, _ := range s.elements {
		if _, exists := x.elements[k]; !exists {
			return false
		}
	}

	return true
}

// Len returns the cardinality of the set.
func (s *Set) Len() int {
	return len(s.elements)
}

// Empty returns true if set is empty
func (s *Set) Empty() bool {
	return len(s.elements) == 0
}

// Freeze returns a slice representing the underlying set data.
func (s *Set) Freeze() []interface{} {
	x := make([]interface{}, len(s.elements))
	i := 0
	for k, _ := range s.elements {
		x[i] = k
		i++
	}

	return x
}
<commit_msg>Fixed doc typo for New.<commit_after>package set

type Set struct {
	elements map[interface{}]struct{}
}

// New creates and returns a new set containing `xs`.
func New(xs ...interface{}) *Set {
	n := make(map[interface{}]struct{})
	for _, x := range xs {
		n[x] = struct{}{}
	}
	return &Set{n}
}

// Copy returns a distinct copy of `s`.
func (s *Set) Copy() *Set {
	x := New()
	for k, _ := range s.elements {
		x.elements[k] = struct{}{}
	}

	return x
}

// Add adds the element `x` to the set `s` and returns true if `x` was added.
func (s *Set) Add(x interface{}) bool {
	if _, exists := s.elements[x]; !exists {
		s.elements[x] = struct{}{}
		return true
	}
	return false
}

// Delete removes the element `x` from the set `s` and returns true if `x` was found.
func (s *Set) Delete(x interface{}) bool {
	if _, exists := s.elements[x]; exists {
		delete(s.elements, x)
		return true
	}
	return false
}

// Member returns true if `x` is contained in `s`.
func (s *Set) Member(x interface{}) bool {
	_, exists := s.elements[x]
	return exists
}

// Merge adds the elements in `x` to `s`.
func (s *Set) Merge(x *Set) {
	for k, _ := range x.elements {
		s.elements[k] = struct{}{}
	}
}

// Discard removes the elements of `x` from `s`.
func (s *Set) Discard(x *Set) {
	for k, _ := range x.elements {
		delete(s.elements, k)
	}
}

// Difference returns a new set containing the difference between `s` and `x`.
func (s *Set) Difference(x *Set) *Set {
	n := s.Copy()
	for k, _ := range x.elements {
		delete(n.elements, k)
	}

	return n
}

// Union returns a new set composed of the union of `s` and `x`.
func (s *Set) Union(x *Set) *Set {
	n := New()
	for k, _ := range s.elements {
		n.elements[k] = struct{}{}
	}

	for k, _ := range x.elements {
		n.elements[k] = struct{}{}
	}

	return n
}

// Intersection returns a new set composed of the intersection between `s` and `x`.
func (s *Set) Intersection(x *Set) *Set {
	n := New()
	for k1, _ := range x.elements {
		for k2, _ := range s.elements {
			if k1 == k2 {
				n.elements[k1] = struct{}{}
			}
		}
	}

	return n
}

// Superset returns true if `s` is a superset of `x`.
func (s *Set) Superset(x *Set) bool {
	return x.Subset(s)
}

// Subset returns true of `s` is a subset of `x`.
func (s *Set) Subset(x *Set) bool {
	for k, _ := range s.elements {
		if _, exists := x.elements[k]; !exists {
			return false
		}
	}

	return true
}

// Len returns the cardinality of the set.
func (s *Set) Len() int {
	return len(s.elements)
}

// Empty returns true if set is empty
func (s *Set) Empty() bool {
	return len(s.elements) == 0
}

// Freeze returns a slice representing the underlying set data.
func (s *Set) Freeze() []interface{} {
	x := make([]interface{}, len(s.elements))
	i := 0
	for k, _ := range s.elements {
		x[i] = k
		i++
	}

	return x
}
<|endoftext|>


<commit_before>// errchk $G $D/$F.go

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

func main() {
	L: ;  // ';' terminates empty statement => L does not apply to for loop
	for i := 0; i < 10; i++ {
		println(i);
		break L;  // ERROR "L"
	}

	L1: { // L1 labels block => L1 does not apply to for loop
		for i := 0; i < 10; i++ {
			println(i);
			break L1;  // ERROR "L1"
		}
	}
}
<commit_msg>fix build<commit_after>// errchk $G $D/$F.go

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

func main() {
	L: ;  // ';' terminates empty statement => L does not apply to for loop
	for i := 0; i < 10; i++ {
		println(i);
		break L  // L does not apply to for loop
	}
}
<|endoftext|>


<commit_before>/**
 * (C) Copyright 2014, Deft Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package dlshared

import (
	"fmt"
	"time"
	"labix.org/v2/mgo"
)

// Create a new Mongo component from a configuration path. The path passed must be in the following format.
//
// mongodb: {
//     configDb: {
//         mongoUrl: "mongodb://localhost:27017/test",
//         mode: 0,
//         dialTimeoutInMs: 3000,
//         socketTimeoutInMs: 3000,
//         syncTimeoutInMs: 3000,
//         cursorTimeoutInMs: 30000,
//     }
// }
//
// The configPath for this component would be "mongodb.configDb". The path can be any arbitrary set of nested
// json documents (json path). If the path is incorrect, the Start() method will panic when called by the kernel.
//
// All of the params above must be present or
// the Start method will panic. If the componentId or configPath param is nil or empty,
// this method will panic.
func NewMongoFromConfigPath(componentId, configPath string) *Mongo {

	if len(componentId) == 0 {
		panic("When calling NewMongoFromConfigPath you must pass in a non-empty componentId param")
	}

	if len(configPath) == 0 {
		panic("When calling NewMongoFromConfigPath you must pass in a non-empty configPath param")
	}

	return &Mongo{ componentId : componentId, configPath : configPath }
}

// Create a new Mongo component. This method will panic if either of the params are nil or len == 0.
func NewMongo(componentId, mongoUrl string, mode, dialTimeoutInMs, socketTimeoutInMs, syncTimeoutInMs, cursorTimeoutInMs int) *Mongo {

	if len(componentId) == 0 {
		panic("When calling NewMongo you must pass in a non-empty component id")
	}

	if len(mongoUrl) == 0 {
		panic("When calling NewMongo you must pass in a non-empty Mongo url")
	}

	return &Mongo{
		componentId : componentId,
		mongoUrl: mongoUrl,
		mode : mode,
		dialTimeoutInMs : dialTimeoutInMs,
		socketTimeoutInMs : socketTimeoutInMs,
		syncTimeoutInMs : syncTimeoutInMs,
		cursorTimeoutInMs : cursorTimeoutInMs,
	}
}

type Mongo struct {
	kernel *Kernel
	Logger

	configPath string

	componentId string
	mongoUrl string
	mode int
	dialTimeoutInMs int
	socketTimeoutInMs int
	syncTimeoutInMs int
	cursorTimeoutInMs int
	session *mgo.Session
}

// Returns the collection from the session.
func (self *Mongo) Collection(dbName, collectionName string) *mgo.Collection { return self.Db(dbName).C(collectionName) }

// Returns the database from the session.
func (self *Mongo) Db(name string) *mgo.Database { return self.session.DB(name) }

// Returns the session struct.
func (self *Mongo) Session() *mgo.Session { return self.session }

// Returns a clone of the session struct.
func (self *Mongo) SessionClone() *mgo.Session { return self.session.Clone() }

// Returns a copy of the session struct.
func (self *Mongo) SessionCopy() *mgo.Session { return self.session.Clone() }

func (self *Mongo) Start(kernel *Kernel) error {
	self.kernel = kernel
	self.Logger = kernel.Logger

	var err error

	// This is a configuration based creation. Load the config data first.
	if len(self.configPath) > 0 {
		self.mongoUrl = self.kernel.Configuration.String(fmt.Sprintf("%s.%s", self.configPath, "mongoUrl"), "")
		self.mode = self.kernel.Configuration.Int(fmt.Sprintf("%s.%s", self.configPath, "mode"), -1)
		self.dialTimeoutInMs = self.kernel.Configuration.Int(fmt.Sprintf("%s.%s", self.configPath, "dialTimeoutInMs"), -1)
		self.socketTimeoutInMs = self.kernel.Configuration.Int(fmt.Sprintf("%s.%s", self.configPath, "socketTimeoutInMs"), -1)
		self.syncTimeoutInMs = self.kernel.Configuration.Int(fmt.Sprintf("%s.%s", self.configPath, "syncTimeoutInMs"), -1)
		self.cursorTimeoutInMs = self.kernel.Configuration.Int(fmt.Sprintf("%s.%s", self.configPath, "cursorTimeoutInMs"), -1)
	}

	// Validate the params
	if len(self.mongoUrl) == 0 {
		panic(fmt.Sprintf("In Mongo - mongoUrl is not set - componentId: %s", self.componentId))
	}

	if self.dialTimeoutInMs < 0 {
		panic(fmt.Sprintf("In Mongo - dialTimeoutInMs is invalid - value: %d - componentId: %s", self.dialTimeoutInMs, self.componentId))
	}

	if self.socketTimeoutInMs < 0 {
		panic(fmt.Sprintf("In Mongo - socketTimeoutInMs is invalid - value: %d - componentId: %s", self.socketTimeoutInMs, self.componentId))
	}

	if self.syncTimeoutInMs < 0 {
		panic(fmt.Sprintf("In Mongo - syncTimeoutInMs is invalid - value: %d - componentId: %s", self.syncTimeoutInMs, self.componentId))
	}

	if self.cursorTimeoutInMs < 0 {
		panic(fmt.Sprintf("In Mongo - cursorTimeoutInMs is invalid - value: %d - componentId: %s", self.cursorTimeoutInMs, self.componentId))
	}

	if self.mode < 0  || self.mode > 2 {
		panic(fmt.Sprintf("In Mongo - mode is invalid - value: %d - componentId: %s", self.mode, self.componentId))
	}

	// Create the session.
	if self.session, err = mgo.DialWithTimeout(self.mongoUrl, time.Duration(self.dialTimeoutInMs) * time.Millisecond); err != nil {
		return NewStackError("Unable to init Mongo session - component: %s - mongodbUrl: %s", self.componentId, self.mongoUrl)
	}

	// This is annoying, but mgo defines these constants as the restricted "mode" type.
	switch self.mode {
		case 0: self.session.SetMode(mgo.Eventual, true)
		case 1: self.session.SetMode(mgo.Monotonic, true)
		case 2: self.session.SetMode(mgo.Strong, true)
	}

	self.session.SetSocketTimeout(time.Duration(self.socketTimeoutInMs) * time.Millisecond)
	self.session.SetSyncTimeout(time.Duration(self.syncTimeoutInMs) * time.Millisecond)

	return nil
}

// Stop the component. This will close the base session.
func (self *Mongo) Stop(kernel *Kernel) error {

	if self.session != nil {
		self.session.Close()
	}

	return nil
}

<commit_msg>added connection type and changed mode to use strings<commit_after>/**
 * (C) Copyright 2014, Deft Labs
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package dlshared

import (
	"fmt"
	"time"
	"labix.org/v2/mgo"
)

type MongoConnectionType string

const (
	MongosConnectionType = MongoConnectionType("mongos")
	StandaloneConnectionType = MongoConnectionType("standalone")
	ReplicaSetConnectionType = MongoConnectionType("replicaSet")
)

// Create a new Mongo component from a configuration path. The path passed must be in the following format.
//
// mongodb: {
//     configDb: {
//         mongoUrl: "mongodb://localhost:27017/test",
//         mode: "strong",
//         dialTimeoutInMs: 3000,
//         socketTimeoutInMs: 3000,
//         syncTimeoutInMs: 3000,
//         cursorTimeoutInMs: 30000,
//         type: "standalone",
//     }
// }
//
// The configPath for this component would be "mongodb.configDb". The path can be any arbitrary set of nested
// json documents (json path). If the path is incorrect, the Start() method will panic when called by the kernel.
//
// All of the params above must be present or
// the Start method will panic. If the componentId or configPath param is nil or empty,
// this method will panic.
func NewMongoFromConfigPath(componentId, configPath string) *Mongo {

	if len(componentId) == 0 {
		panic("When calling NewMongoFromConfigPath you must pass in a non-empty componentId param")
	}

	if len(configPath) == 0 {
		panic("When calling NewMongoFromConfigPath you must pass in a non-empty configPath param")
	}

	return &Mongo{ componentId : componentId, configPath : configPath }
}

// Create a new Mongo component. This method will panic if either of the params are nil or len == 0.
func NewMongo(	componentId,
				mongoUrl string,
				connectionType MongoConnectionType,
				mode string,
				dialTimeoutInMs,
				socketTimeoutInMs,
				syncTimeoutInMs,
				cursorTimeoutInMs int) *Mongo {

	if len(componentId) == 0 {
		panic("When calling NewMongo you must pass in a non-empty component id")
	}

	if len(mongoUrl) == 0 {
		panic("When calling NewMongo you must pass in a non-empty Mongo url")
	}

	if len(connectionType) == 0 {
		panic("When calling NewMongo you must pass in a non-empty connection type (standalone | mongos | replicaSet)")
	}

	return &Mongo{
		componentId: componentId,
		mongoUrl: mongoUrl,
		connectionType: MongoConnectionType(connectionType),
		mode: mode,
		dialTimeoutInMs: dialTimeoutInMs,
		socketTimeoutInMs: socketTimeoutInMs,
		syncTimeoutInMs: syncTimeoutInMs,
		cursorTimeoutInMs: cursorTimeoutInMs,
		DefaultSafe: defaultSafe(MongoConnectionType(connectionType)),
	}
}

type Mongo struct {
	kernel *Kernel
	Logger

	configPath string

	componentId string
	mongoUrl string
	mode string
	dialTimeoutInMs int
	socketTimeoutInMs int
	syncTimeoutInMs int
	cursorTimeoutInMs int
	session *mgo.Session
	connectionType MongoConnectionType

	DefaultSafe *mgo.Safe
}

// Returns the collection from the session.
func (self *Mongo) Collection(dbName, collectionName string) *mgo.Collection { return self.Db(dbName).C(collectionName) }

// Returns the database from the session.
func (self *Mongo) Db(name string) *mgo.Database { return self.session.DB(name) }

// Returns the session struct.
func (self *Mongo) Session() *mgo.Session { return self.session }

// Returns a clone of the session struct.
func (self *Mongo) SessionClone() *mgo.Session { return self.session.Clone() }

// Returns a copy of the session struct.
func (self *Mongo) SessionCopy() *mgo.Session { return self.session.Clone() }

// Returns a default safest mode. If this is a mongos or replica set, WMode: "majority" - if this is
// a standalone instance, w: 1
func defaultSafe(connectionType MongoConnectionType) *mgo.Safe {
	switch connectionType {
		case MongosConnectionType: return &mgo.Safe{ WMode: "majority" }
		case ReplicaSetConnectionType: return &mgo.Safe{ WMode: "majority" }
		case StandaloneConnectionType: return &mgo.Safe{ W: 1 }
		default: panic("Unknown connection type: " + connectionType)
	}

	return nil
}

func (self *Mongo) Start(kernel *Kernel) error {
	self.kernel = kernel
	self.Logger = kernel.Logger

	var err error

	// This is a configuration based creation. Load the config data first.
	if len(self.configPath) > 0 {
		self.mongoUrl = self.kernel.Configuration.String(fmt.Sprintf("%s.%s", self.configPath, "mongoUrl"), "")
		self.connectionType = MongoConnectionType(self.kernel.Configuration.String(fmt.Sprintf("%s.%s", self.configPath, "type"), ""))
		self.mode = self.kernel.Configuration.String(fmt.Sprintf("%s.%s", self.configPath, "mode"), "")
		self.dialTimeoutInMs = self.kernel.Configuration.Int(fmt.Sprintf("%s.%s", self.configPath, "dialTimeoutInMs"), -1)
		self.socketTimeoutInMs = self.kernel.Configuration.Int(fmt.Sprintf("%s.%s", self.configPath, "socketTimeoutInMs"), -1)
		self.syncTimeoutInMs = self.kernel.Configuration.Int(fmt.Sprintf("%s.%s", self.configPath, "syncTimeoutInMs"), -1)
		self.cursorTimeoutInMs = self.kernel.Configuration.Int(fmt.Sprintf("%s.%s", self.configPath, "cursorTimeoutInMs"), -1)
	}

	// Validate the params
	if len(self.mongoUrl) == 0 {
		panic(fmt.Sprintf("In Mongo - mongoUrl is not set - componentId: %s", self.componentId))
	}

	if len(self.connectionType) == 0 {
		panic(fmt.Sprintf("In Mongo - type is not set - componentId: %s", self.componentId))
	}

	if self.dialTimeoutInMs < 0 {
		panic(fmt.Sprintf("In Mongo - dialTimeoutInMs is invalid - value: %d - componentId: %s", self.dialTimeoutInMs, self.componentId))
	}

	if self.socketTimeoutInMs < 0 {
		panic(fmt.Sprintf("In Mongo - socketTimeoutInMs is invalid - value: %d - componentId: %s", self.socketTimeoutInMs, self.componentId))
	}

	if self.syncTimeoutInMs < 0 {
		panic(fmt.Sprintf("In Mongo - syncTimeoutInMs is invalid - value: %d - componentId: %s", self.syncTimeoutInMs, self.componentId))
	}

	if self.cursorTimeoutInMs < 0 {
		panic(fmt.Sprintf("In Mongo - cursorTimeoutInMs is invalid - value: %d - componentId: %s", self.cursorTimeoutInMs, self.componentId))
	}

	if len(self.mode) == 0 {
		panic(fmt.Sprintf("In Mongo - mode is invalid - value: %s - componentId: %s", self.mode, self.componentId))
	}

	if self.mode != "strong" && self.mode != "eventual" && self.mode != "montonic" {
		panic(fmt.Sprintf("In Mongo - mode is invalid - value: %s - componentId: %s", self.mode, self.componentId))
	}

	// Create the session.
	if self.session, err = mgo.DialWithTimeout(self.mongoUrl, time.Duration(self.dialTimeoutInMs) * time.Millisecond); err != nil {
		return NewStackError("Unable to init Mongo session - component: %s - mongodbUrl: %s", self.componentId, self.mongoUrl)
	}

	// This is annoying, but mgo defines these constants as the restricted "mode" type.
	switch self.mode {
		case "eventual": self.session.SetMode(mgo.Eventual, true)
		case "monotonic": self.session.SetMode(mgo.Monotonic, true)
		case "strong": self.session.SetMode(mgo.Strong, true)
	}

	self.DefaultSafe = defaultSafe(self.connectionType)

	self.session.SetSocketTimeout(time.Duration(self.socketTimeoutInMs) * time.Millisecond)
	self.session.SetSyncTimeout(time.Duration(self.syncTimeoutInMs) * time.Millisecond)
	self.session.SetSafe(nil)

	return nil
}

// Stop the component. This will close the base session.
func (self *Mongo) Stop(kernel *Kernel) error {

	if self.session != nil {
		self.session.Close()
	}

	return nil
}

<|endoftext|>