<commit_before>// (c) Copyright 2015 JONNALAGADDA Srinivas
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package flow

import (
	"fmt"
	"sync"
	"time"
)

// Document represents a task in a workflow, whose life cycle it
// tracks.
//
// Documents are central to the workflow engine and its operations.
// Each document represents a task whose life cycle it tracks.  In the
// process, it accumulates various details, and tracks the times of
// its modifications.  The life cycle typically involves several state
// transitions, whose details are also tracked.
//
// Most applications should embed `Document` in their document
// structures rather than use this directly.
type Document struct {
	id     uint64 // globally-unique
	dtype  DocType
	title  string
	text   string // primary content
	author *User  // creator of the document
	ctime  time.Time

	mutex    sync.Mutex
	state    *DocState   // current state
	events   []*DocEvent // state transitions so far, tracked in order
	revision uint16
}

// NewDocument creates and initialises a document.
//
// The document created through this method has a life cycle that is
// associated with it through a particular workflow.
func NewDocument(id uint64, dtype DocType, title string, author *User, instate *DocState) (*Document, error) {
	if id == 0 || string(dtype) == "" || title == "" || author == nil {
		return nil, fmt.Errorf("invalid initialisation data -- id: %d, dtype: %s, title: %s, author: %s", id, dtype, title, author.Name())
	}

	d := &Document{id: id, dtype: dtype, title: title, author: author}
	d.ctime = time.Now().UTC()
	d.state = instate
	d.events = make([]*DocEvent, 1)
	d.revision = 1
	return d, nil
}

// TODO(js): OpenDocument()

// ID answers this document's globally-unique ID.
func (d *Document) ID() uint64 {
	return d.id
}

// Type answers this document's type.
func (d *Document) Type() DocType {
	return d.dtype
}

// Title answers this document's title.
func (d *Document) Title() string {
	return d.title
}

// SetText sets the primary content of this document.
func (d *Document) SetText(t string) error {
	if d.revision > 1 {
		return fmt.Errorf("document revision > 1 : %d", d.revision)
	}
	if d.text != "" {
		return fmt.Errorf("document text already set")
	}
	if t == "" {
		return fmt.Errorf("empty content given")
	}

	d.mutex.Lock()
	defer d.mutex.Unlock()

	d.text = t
	return nil
}

// Text answers this document's primary content.
func (d *Document) Text() string {
	return d.text
}

// Author answers the user who created this document.
func (d *Document) Author() *User {
	return d.author
}

// Ctime answers the creation time of this document.
func (d *Document) Ctime() time.Time {
	return d.ctime
}

// State answer this document's current state.
func (d *Document) State() *DocState {
	return d.state
}

// Mtime answers the time when this document was most-recently
// modified.  It answers the creation time, if it has not been updated
// since.
func (d *Document) Mtime() time.Time {
	l := len(d.events)
	if l == 0 {
		return d.ctime
	}

	return d.events[l-1].mtime
}

// Revision answers the current revision number of this document.
func (d *Document) Revision() uint16 {
	return d.revision
}

// Events answers a copy of the sequence of events that has
// transformed this document so far.
func (d *Document) Events() []*DocEvent {
	// Synchronised because events are dynamic.
	d.mutex.Lock()
	defer d.mutex.Unlock()

	es := make([]*DocEvent, len(d.events))
	copy(es, d.events)
	return es
}

// applyEvent transitions this document into a new state as per the
// applied event.
//
// We perform as many validations as possible when constructing the
// event, so that we spend a minimum amount of time in this
// synchronised method.
func (d *Document) applyEvent(e *DocEvent) error {
	if e.doc.id != d.id {
		return fmt.Errorf("mismatched document IDs -- current: %d, event's: %d", d.id, e.doc.id)
	}

	d.mutex.Lock()
	defer d.mutex.Unlock()

	e.mtime = time.Now().UTC()
	d.revision++
	e.revision = d.revision
	d.state = e.state
	d.events = append(d.events, e)
	return nil
}
<commit_msg>Add a check to ensure that an event is applied only once<commit_after>// (c) Copyright 2015 JONNALAGADDA Srinivas
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package flow

import (
	"fmt"
	"sync"
	"time"
)

// Document represents a task in a workflow, whose life cycle it
// tracks.
//
// Documents are central to the workflow engine and its operations.
// Each document represents a task whose life cycle it tracks.  In the
// process, it accumulates various details, and tracks the times of
// its modifications.  The life cycle typically involves several state
// transitions, whose details are also tracked.
//
// Most applications should embed `Document` in their document
// structures rather than use this directly.
type Document struct {
	id     uint64 // globally-unique
	dtype  DocType
	title  string
	text   string // primary content
	author *User  // creator of the document
	ctime  time.Time

	mutex    sync.Mutex
	state    *DocState   // current state
	events   []*DocEvent // state transitions so far, tracked in order
	revision uint16
}

// NewDocument creates and initialises a document.
//
// The document created through this method has a life cycle that is
// associated with it through a particular workflow.
func NewDocument(id uint64, dtype DocType, title string, author *User, instate *DocState) (*Document, error) {
	if id == 0 || string(dtype) == "" || title == "" || author == nil {
		return nil, fmt.Errorf("invalid initialisation data -- id: %d, dtype: %s, title: %s, author: %s", id, dtype, title, author.Name())
	}

	d := &Document{id: id, dtype: dtype, title: title, author: author}
	d.ctime = time.Now().UTC()
	d.state = instate
	d.events = make([]*DocEvent, 1)
	d.revision = 1
	return d, nil
}

// TODO(js): OpenDocument()

// ID answers this document's globally-unique ID.
func (d *Document) ID() uint64 {
	return d.id
}

// Type answers this document's type.
func (d *Document) Type() DocType {
	return d.dtype
}

// Title answers this document's title.
func (d *Document) Title() string {
	return d.title
}

// SetText sets the primary content of this document.
func (d *Document) SetText(t string) error {
	if d.revision > 1 {
		return fmt.Errorf("document revision > 1 : %d", d.revision)
	}
	if d.text != "" {
		return fmt.Errorf("document text already set")
	}
	if t == "" {
		return fmt.Errorf("empty content given")
	}

	d.mutex.Lock()
	defer d.mutex.Unlock()

	d.text = t
	return nil
}

// Text answers this document's primary content.
func (d *Document) Text() string {
	return d.text
}

// Author answers the user who created this document.
func (d *Document) Author() *User {
	return d.author
}

// Ctime answers the creation time of this document.
func (d *Document) Ctime() time.Time {
	return d.ctime
}

// State answer this document's current state.
func (d *Document) State() *DocState {
	return d.state
}

// Mtime answers the time when this document was most-recently
// modified.  It answers the creation time, if it has not been updated
// since.
func (d *Document) Mtime() time.Time {
	l := len(d.events)
	if l == 0 {
		return d.ctime
	}

	return d.events[l-1].mtime
}

// Revision answers the current revision number of this document.
func (d *Document) Revision() uint16 {
	return d.revision
}

// Events answers a copy of the sequence of events that has
// transformed this document so far.
func (d *Document) Events() []*DocEvent {
	// Synchronised because events are dynamic.
	d.mutex.Lock()
	defer d.mutex.Unlock()

	es := make([]*DocEvent, len(d.events))
	copy(es, d.events)
	return es
}

// applyEvent transitions this document into a new state as per the
// applied event.
//
// We perform as many validations as possible when constructing the
// event, so that we spend a minimum amount of time in this
// synchronised method.
func (d *Document) applyEvent(e *DocEvent) error {
	if e.doc.id != d.id {
		return fmt.Errorf("mismatched document IDs -- current: %d, event's: %d", d.id, e.doc.id)
	}

	d.mutex.Lock()
	defer d.mutex.Unlock()

	if !e.mtime.IsZero() {
		return fmt.Errorf("event already applied")
	}

	e.mtime = time.Now().UTC()
	d.revision++
	e.revision = d.revision
	d.state = e.state
	d.events = append(d.events, e)
	return nil
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"os"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccAWSSnapshotCreateVolumePermission_Basic(t *testing.T) {
	snapshot_id := ""
	account_id := os.Getenv("AWS_ACCOUNT_ID")

	resource.Test(t, resource.TestCase{
		PreCheck: func() {
			testAccPreCheck(t)
			if os.Getenv("AWS_ACCOUNT_ID") == "" {
				t.Fatal("AWS_ACCOUNT_ID must be set")
			}
		},
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			// Scaffold everything
			resource.TestStep{
				Config: testAccAWSSnapshotCreateVolumePermissionConfig(account_id, true),
				Check: resource.ComposeTestCheckFunc(
					testCheckResourceGetAttr("aws_ami_copy.test", "block_device_mappings.0.ebs.snapshot_id", &snapshot_id),
					testAccAWSSnapshotCreateVolumePermissionExists(account_id, &snapshot_id),
				),
			},
			// Drop just create volume permission to test destruction
			resource.TestStep{
				Config: testAccAWSSnapshotCreateVolumePermissionConfig(account_id, false),
				Check: resource.ComposeTestCheckFunc(
					testAccAWSSnapshotCreateVolumePermissionDestroyed(account_id, &snapshot_id),
				),
			},
		},
	})
}

func testAccAWSSnapshotCreateVolumePermissionExists(account_id string, snapshot_id *string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		conn := testAccProvider.Meta().(*AWSClient).ec2conn
		if has, err := hasCreateVolumePermission(conn, *snapshot_id, account_id); err != nil {
			return err
		} else if !has {
			return fmt.Errorf("create volume permission does not exist for '%s' on '%s'", account_id, *snapshot_id)
		}
		return nil
	}
}

func testAccAWSSnapshotCreateVolumePermissionDestroyed(account_id string, snapshot_id *string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		conn := testAccProvider.Meta().(*AWSClient).ec2conn
		if has, err := hasCreateVolumePermission(conn, *snapshot_id, account_id); err != nil {
			return err
		} else if has {
			return fmt.Errorf("create volume permission still exists for '%s' on '%s'", account_id, *snapshot_id)
		}
		return nil
	}
}

func testAccAWSSnapshotCreateVolumePermissionConfig(account_id string, includeCreateVolumePermission bool) string {
	base := `
resource "aws_ami_copy" "test" {
  name = "create-volume-permission-test"
  description = "Create Volume Permission Test Copy"
  source_ami_id = "ami-7172b611"
  source_ami_region = "us-west-2"
}
`

	if !includeCreateVolumePermission {
		return base
	}

	return base + fmt.Sprintf(`
resource "aws_snapshot_create_volume_permission" "self-test" {
  snapshot_id   = "${lookup(lookup(element(aws_ami_copy.test.block_device_mappings, 0), "ebs"), "snapshot_id")}"
  account_id = "%s"
}
`, account_id)
}
<commit_msg>refactor the test to use caller_identity data source, and new ebs_snapshot resource<commit_after>package aws

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccAWSSnapshotCreateVolumePermission_Basic(t *testing.T) {
	var snapshotId, accountId string

	resource.Test(t, resource.TestCase{
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			// Scaffold everything
			resource.TestStep{
				Config: testAccAWSSnapshotCreateVolumePermissionConfig(true),
				Check: resource.ComposeTestCheckFunc(
					testCheckResourceGetAttr("aws_ebs_snapshot.example_snapshot", "id", &snapshotId),
					testCheckResourceGetAttr("data.aws_caller_identity.current", "account_id", &accountId),
					testAccAWSSnapshotCreateVolumePermissionExists(&accountId, &snapshotId),
				),
			},
			// Drop just create volume permission to test destruction
			resource.TestStep{
				Config: testAccAWSSnapshotCreateVolumePermissionConfig(false),
				Check: resource.ComposeTestCheckFunc(
					testAccAWSSnapshotCreateVolumePermissionDestroyed(&accountId, &snapshotId),
				),
			},
		},
	})
}

func testAccAWSSnapshotCreateVolumePermissionExists(accountId, snapshotId *string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		conn := testAccProvider.Meta().(*AWSClient).ec2conn
		if has, err := hasCreateVolumePermission(conn, *snapshotId, *accountId); err != nil {
			return err
		} else if !has {
			return fmt.Errorf("create volume permission does not exist for '%s' on '%s'", *accountId, *snapshotId)
		}
		return nil
	}
}

func testAccAWSSnapshotCreateVolumePermissionDestroyed(accountId, snapshotId *string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		conn := testAccProvider.Meta().(*AWSClient).ec2conn
		if has, err := hasCreateVolumePermission(conn, *snapshotId, *accountId); err != nil {
			return err
		} else if has {
			return fmt.Errorf("create volume permission still exists for '%s' on '%s'", *accountId, *snapshotId)
		}
		return nil
	}
}

func testAccAWSSnapshotCreateVolumePermissionConfig(includeCreateVolumePermission bool) string {
	base := `
data "aws_caller_identity" "current" {}

resource "aws_ebs_volume" "example" {
  availability_zone = "us-west-2a"
  size              = 40

  tags {
    Name = "ebs_snap_perm"
  }
}

resource "aws_ebs_snapshot" "example_snapshot" {
  volume_id = "${aws_ebs_volume.example.id}"
}
`

	if !includeCreateVolumePermission {
		return base
	}

	return base + fmt.Sprintf(`
resource "aws_snapshot_create_volume_permission" "self-test" {
  snapshot_id = "${aws_ebs_snapshot.example_snapshot.id}"
  account_id  = "${data.aws_caller_identity.current.account_id}"
}
`)
}
<|endoftext|>


<commit_before>// Contains utility functions for helping combine .jar files.
package java

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"zip"

	"gopkg.in/op/go-logging.v1"
)

var log = logging.MustGetLogger("zip_writer")

// AddZipFile copies the contents of a zip file into an existing zip writer.
func AddZipFile(w *zip.Writer, filepath string, include, exclude []string, stripPrefix string, strict bool, renameDirs map[string]string) error {
	r, err := zip.OpenReader(filepath)
	if err != nil {
		return err
	}
	defer r.Close()

	// Reopen file to get a directly readable version without decompression.
	r2, err := os.Open(filepath)
	if err != nil {
		return err
	}
	defer r2.Close()

outer:
	for _, f := range r.File {
		log.Debug("Found file %s (from %s)", f.Name, filepath)
		// This directory is very awkward. We need to merge the contents by concatenating them,
		// we can't replace them or leave them out.
		if strings.HasPrefix(f.Name, "META-INF/services/") ||
			strings.HasPrefix(f.Name, "META-INF/spring") ||
			f.Name == "META-INF/please_sourcemap" {
			if err := concatenateFile(w, f); err != nil {
				return err
			}
			continue
		}
		if !shouldInclude(f.Name, include, exclude) {
			continue outer
		}
		hasTrailingSlash := strings.HasSuffix(f.Name, "/")
		isDir := hasTrailingSlash || f.FileInfo().IsDir()
		if isDir && !hasTrailingSlash {
			f.Name = f.Name + "/"
		}
		if existing, present := getExistingFile(w, f.Name); present {
			// Allow duplicates of directories. Seemingly the best way to identify them is that
			// they end in a trailing slash.
			if isDir {
				continue
			}
			// Allow skipping existing files that are exactly the same as the added ones.
			// It's unnecessarily awkward to insist on not ever doubling up on a dependency.
			if existing.CRC32 == f.CRC32 {
				log.Info("Skipping %s / %s: already added (from %s)", filepath, f.Name, existing.ZipFile)
				continue
			}
			if strict {
				log.Error("Duplicate file %s (from %s, already added from %s); crc %d / %d", f.Name, filepath, existing.ZipFile, f.CRC32, existing.CRC32)
				return fmt.Errorf("File %s already added to destination zip file (from %s)", f.Name, existing.ZipFile)
			}
			continue
		}
		for before, after := range renameDirs {
			if strings.HasPrefix(f.Name, before) {
				f.Name = path.Join(after, strings.TrimPrefix(f.Name, before))
				if isDir {
					f.Name = f.Name + "/"
				}
				break
			}
		}
		f.Name = strings.TrimPrefix(f.Name, stripPrefix)
		// Java tools don't seem to like writing a data descriptor for stored items.
		// Unsure if this is a limitation of the format or a problem of those tools.
		f.Flags = 0
		addExistingFile(w, f.Name, filepath, f.CompressedSize64, f.UncompressedSize64, f.CRC32)

		start, err := f.DataOffset()
		if err != nil {
			return err
		}
		if _, err := r2.Seek(start, 0); err != nil {
			return err
		}
		// Make these deterministic.
		f.FileHeader.ModifiedDate = 0
		f.FileHeader.ModifiedTime = 0
		if err := addFile(w, &f.FileHeader, r2, f.CRC32); err != nil {
			return err
		}
	}
	return nil
}

func shouldInclude(name string, include, exclude []string) bool {
	for _, excl := range exclude {
		if matched, _ := filepath.Match(excl, name); matched {
			log.Debug("Skipping %s (excluded by %s)", name, excl)
			return false
		} else if matched, _ := filepath.Match(excl, filepath.Base(name)); matched {
			log.Debug("Skipping %s (excluded by %s)", name, excl)
			return false
		}
	}
	if len(include) == 0 {
		return true
	}
	for _, incl := range include {
		if matched, _ := filepath.Match(incl, name); matched || strings.HasPrefix(name, incl) {
			return true
		}
	}
	log.Debug("Skipping %s (didn't match any includes)", name)
	return false
}

// AddInitPyFiles adds an __init__.py file to every directory in the zip file that doesn't already have one.
func AddInitPyFiles(w *zip.Writer) error {
	m := files[w]
	s := make([]string, 0, len(m))
	for p := range m {
		s = append(s, p)
	}
	sort.Strings(s)
	for _, p := range s {
		d := filepath.Dir(p)
		if filepath.Base(d) == "__pycache__" {
			continue // Don't need to add an __init__.py here.
		}
		initPyPath := path.Join(d, "__init__.py")
		if _, present := m[initPyPath]; !present {
			// If we already have a pyc / pyo we don't need the __init__.py as well.
			if _, present := m[initPyPath+"c"]; present {
				continue
			} else if _, present := m[initPyPath+"o"]; present {
				continue
			}
			// Don't write one at the root, it's not necessary.
			if initPyPath != "__init__.py" {
				log.Debug("Adding %s", initPyPath)
				m[initPyPath] = fileRecord{}
				if err := WriteFile(w, initPyPath, []byte{}); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

// AddManifest adds a manifest to the given zip writer with a Main-Class entry (and a couple of others)
func AddManifest(w *zip.Writer, mainClass string) error {
	manifest := fmt.Sprintf("Manifest-Version: 1.0\nMain-Class: %s\n", mainClass)
	return WriteFile(w, "META-INF/MANIFEST.MF", []byte(manifest))
}

// Records some information about a file that we use to check if they're exact duplicates.
type fileRecord struct {
	ZipFile            string
	CompressedSize64   uint64
	UncompressedSize64 uint64
	CRC32              uint32
}

var files = map[*zip.Writer]map[string]fileRecord{}
var concatenatedFiles = map[string]string{}

func getExistingFile(w *zip.Writer, name string) (fileRecord, bool) {
	if m := files[w]; m != nil {
		record, present := m[name]
		return record, present
	}
	return fileRecord{}, false
}

// HasExistingFile returns true if the writer has already written the given file.
func HasExistingFile(w *zip.Writer, name string) bool {
	_, b := getExistingFile(w, name)
	return b
}

func addExistingFile(w *zip.Writer, name, file string, c, u uint64, crc uint32) {
	m := files[w]
	if m == nil {
		m = map[string]fileRecord{}
		files[w] = m
	}
	m[name] = fileRecord{file, c, u, crc}
}

// Add a file to the zip which is concatenated with any existing content with the same name.
// Writing is deferred since we obviously can't append to it later.
func concatenateFile(w *zip.Writer, f *zip.File) error {
	r, err := f.Open()
	if err != nil {
		return err
	}
	defer r.Close()
	buf := new(bytes.Buffer)
	if _, err := io.Copy(buf, r); err != nil {
		return err
	}
	contents := buf.String()
	if !strings.HasSuffix(contents, "\n") {
		contents += "\n"
	}
	concatenatedFiles[f.Name] += contents
	return nil
}

// HandleConcatenatedFiles appends concatenated files to the archive's directory for writing.
func HandleConcatenatedFiles(w *zip.Writer) error {
	// Must do it in a deterministic order
	files := make([]string, 0, len(concatenatedFiles))
	for name := range concatenatedFiles {
		files = append(files, name)
	}
	sort.Strings(files)
	for _, name := range files {
		if err := WriteFile(w, name, []byte(concatenatedFiles[name])); err != nil {
			return err
		}
	}
	return nil
}

// Writes a file to the new writer.
func addFile(w *zip.Writer, fh *zip.FileHeader, r io.Reader, crc uint32) error {
	fh.Flags = 0 // we're not writing a data descriptor after the file
	comp := func(w io.Writer) (io.WriteCloser, error) { return nopCloser{w}, nil }
	fw, err := w.CreateHeaderWithCompressor(fh, comp, fixedCrc32{value: crc})
	if err == nil {
		_, err = io.CopyN(fw, r, int64(fh.CompressedSize64))
	}
	return err
}

// WriteFile writes a complete file to the writer.
func WriteFile(w *zip.Writer, filename string, data []byte) error {
	if fw, err := w.Create(filename); err != nil {
		return err
	} else if _, err := fw.Write(data); err != nil {
		return err
	}
	addExistingFile(w, filename, filename, 0, 0, 0)
	return nil
}

// WriteDir writes a directory entry to the writer.
func WriteDir(w *zip.Writer, filename string) error {
	filename += "/" // Must have trailing slash to tell it it's a directory.
	fh := zip.FileHeader{
		Name:   filename,
		Method: zip.Store,
	}
	if _, err := w.CreateHeader(&fh); err != nil {
		return err
	}
	addExistingFile(w, filename, filename, 0, 0, 0)
	return nil
}

type nopCloser struct {
	io.Writer
}

func (w nopCloser) Close() error {
	return nil
}

// fixedCrc32 implements a Hash32 interface that just writes out a predetermined value.
// this is really cheating of course but serves our purposes here.
type fixedCrc32 struct {
	value uint32
}

func (crc fixedCrc32) Write(p []byte) (n int, err error) {
	return len(p), nil
}

func (crc fixedCrc32) Sum(b []byte) []byte {
	buf := make([]byte, 4)
	binary.LittleEndian.PutUint32(buf, crc.value)
	return b
}

func (crc fixedCrc32) Sum32() uint32 {
	return crc.value
}

func (crc fixedCrc32) Reset() {
}

func (crc fixedCrc32) Size() int {
	return 32
}

func (crc fixedCrc32) BlockSize() int {
	return 32
}
<commit_msg>Make zip_writer a bit more permissive in some rare cases<commit_after>// Contains utility functions for helping combine .jar files.
package java

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"zip"

	"gopkg.in/op/go-logging.v1"
)

var log = logging.MustGetLogger("zip_writer")

// AddZipFile copies the contents of a zip file into an existing zip writer.
func AddZipFile(w *zip.Writer, filepath string, include, exclude []string, stripPrefix string, strict bool, renameDirs map[string]string) error {
	r, err := zip.OpenReader(filepath)
	if err != nil {
		return err
	}
	defer r.Close()

	// Reopen file to get a directly readable version without decompression.
	r2, err := os.Open(filepath)
	if err != nil {
		return err
	}
	defer r2.Close()

outer:
	for _, f := range r.File {
		log.Debug("Found file %s (from %s)", f.Name, filepath)
		// This directory is very awkward. We need to merge the contents by concatenating them,
		// we can't replace them or leave them out.
		if strings.HasPrefix(f.Name, "META-INF/services/") ||
			strings.HasPrefix(f.Name, "META-INF/spring") ||
			f.Name == "META-INF/please_sourcemap" {
			if err := concatenateFile(w, f); err != nil {
				return err
			}
			continue
		}
		if !shouldInclude(f.Name, include, exclude) {
			continue outer
		}
		hasTrailingSlash := strings.HasSuffix(f.Name, "/")
		isDir := hasTrailingSlash || f.FileInfo().IsDir()
		if isDir && !hasTrailingSlash {
			f.Name = f.Name + "/"
		}
		if existing, present := getExistingFile(w, f.Name); present {
			// Allow duplicates of directories. Seemingly the best way to identify them is that
			// they end in a trailing slash.
			if isDir {
				continue
			}
			// Allow skipping existing files that are exactly the same as the added ones.
			// It's unnecessarily awkward to insist on not ever doubling up on a dependency.
			// TODO(pebers): Bit of a hack ignoring it when CRC is 0, would be better to add
			//               the correct CRC when added through WriteFile.
			if existing.CRC32 == f.CRC32 || existing.CRC32 == 0 {
				log.Info("Skipping %s / %s: already added (from %s)", filepath, f.Name, existing.ZipFile)
				continue
			}
			if strict {
				log.Error("Duplicate file %s (from %s, already added from %s); crc %d / %d", f.Name, filepath, existing.ZipFile, f.CRC32, existing.CRC32)
				return fmt.Errorf("File %s already added to destination zip file (from %s)", f.Name, existing.ZipFile)
			}
			continue
		}
		for before, after := range renameDirs {
			if strings.HasPrefix(f.Name, before) {
				f.Name = path.Join(after, strings.TrimPrefix(f.Name, before))
				if isDir {
					f.Name = f.Name + "/"
				}
				break
			}
		}
		f.Name = strings.TrimPrefix(f.Name, stripPrefix)
		// Java tools don't seem to like writing a data descriptor for stored items.
		// Unsure if this is a limitation of the format or a problem of those tools.
		f.Flags = 0
		addExistingFile(w, f.Name, filepath, f.CompressedSize64, f.UncompressedSize64, f.CRC32)

		start, err := f.DataOffset()
		if err != nil {
			return err
		}
		if _, err := r2.Seek(start, 0); err != nil {
			return err
		}
		// Make these deterministic.
		f.FileHeader.ModifiedDate = 0
		f.FileHeader.ModifiedTime = 0
		if err := addFile(w, &f.FileHeader, r2, f.CRC32); err != nil {
			return err
		}
	}
	return nil
}

func shouldInclude(name string, include, exclude []string) bool {
	for _, excl := range exclude {
		if matched, _ := filepath.Match(excl, name); matched {
			log.Debug("Skipping %s (excluded by %s)", name, excl)
			return false
		} else if matched, _ := filepath.Match(excl, filepath.Base(name)); matched {
			log.Debug("Skipping %s (excluded by %s)", name, excl)
			return false
		}
	}
	if len(include) == 0 {
		return true
	}
	for _, incl := range include {
		if matched, _ := filepath.Match(incl, name); matched || strings.HasPrefix(name, incl) {
			return true
		}
	}
	log.Debug("Skipping %s (didn't match any includes)", name)
	return false
}

// AddInitPyFiles adds an __init__.py file to every directory in the zip file that doesn't already have one.
func AddInitPyFiles(w *zip.Writer) error {
	m := files[w]
	s := make([]string, 0, len(m))
	for p := range m {
		s = append(s, p)
	}
	sort.Strings(s)
	for _, p := range s {
		d := filepath.Dir(p)
		if filepath.Base(d) == "__pycache__" {
			continue // Don't need to add an __init__.py here.
		}
		initPyPath := path.Join(d, "__init__.py")
		if _, present := m[initPyPath]; !present {
			// If we already have a pyc / pyo we don't need the __init__.py as well.
			if _, present := m[initPyPath+"c"]; present {
				continue
			} else if _, present := m[initPyPath+"o"]; present {
				continue
			}
			// Don't write one at the root, it's not necessary.
			if initPyPath != "__init__.py" {
				log.Debug("Adding %s", initPyPath)
				m[initPyPath] = fileRecord{}
				if err := WriteFile(w, initPyPath, []byte{}); err != nil {
					return err
				}
			}
		}
	}
	return nil
}

// AddManifest adds a manifest to the given zip writer with a Main-Class entry (and a couple of others)
func AddManifest(w *zip.Writer, mainClass string) error {
	manifest := fmt.Sprintf("Manifest-Version: 1.0\nMain-Class: %s\n", mainClass)
	return WriteFile(w, "META-INF/MANIFEST.MF", []byte(manifest))
}

// Records some information about a file that we use to check if they're exact duplicates.
type fileRecord struct {
	ZipFile            string
	CompressedSize64   uint64
	UncompressedSize64 uint64
	CRC32              uint32
}

var files = map[*zip.Writer]map[string]fileRecord{}
var concatenatedFiles = map[string]string{}

func getExistingFile(w *zip.Writer, name string) (fileRecord, bool) {
	if m := files[w]; m != nil {
		record, present := m[name]
		return record, present
	}
	return fileRecord{}, false
}

// HasExistingFile returns true if the writer has already written the given file.
func HasExistingFile(w *zip.Writer, name string) bool {
	_, b := getExistingFile(w, name)
	return b
}

func addExistingFile(w *zip.Writer, name, file string, c, u uint64, crc uint32) {
	m := files[w]
	if m == nil {
		m = map[string]fileRecord{}
		files[w] = m
	}
	m[name] = fileRecord{file, c, u, crc}
}

// Add a file to the zip which is concatenated with any existing content with the same name.
// Writing is deferred since we obviously can't append to it later.
func concatenateFile(w *zip.Writer, f *zip.File) error {
	r, err := f.Open()
	if err != nil {
		return err
	}
	defer r.Close()
	buf := new(bytes.Buffer)
	if _, err := io.Copy(buf, r); err != nil {
		return err
	}
	contents := buf.String()
	if !strings.HasSuffix(contents, "\n") {
		contents += "\n"
	}
	concatenatedFiles[f.Name] += contents
	return nil
}

// HandleConcatenatedFiles appends concatenated files to the archive's directory for writing.
func HandleConcatenatedFiles(w *zip.Writer) error {
	// Must do it in a deterministic order
	files := make([]string, 0, len(concatenatedFiles))
	for name := range concatenatedFiles {
		files = append(files, name)
	}
	sort.Strings(files)
	for _, name := range files {
		if err := WriteFile(w, name, []byte(concatenatedFiles[name])); err != nil {
			return err
		}
	}
	return nil
}

// Writes a file to the new writer.
func addFile(w *zip.Writer, fh *zip.FileHeader, r io.Reader, crc uint32) error {
	fh.Flags = 0 // we're not writing a data descriptor after the file
	comp := func(w io.Writer) (io.WriteCloser, error) { return nopCloser{w}, nil }
	fw, err := w.CreateHeaderWithCompressor(fh, comp, fixedCrc32{value: crc})
	if err == nil {
		_, err = io.CopyN(fw, r, int64(fh.CompressedSize64))
	}
	return err
}

// WriteFile writes a complete file to the writer.
func WriteFile(w *zip.Writer, filename string, data []byte) error {
	if fw, err := w.Create(filename); err != nil {
		return err
	} else if _, err := fw.Write(data); err != nil {
		return err
	}
	addExistingFile(w, filename, filename, 0, 0, 0)
	return nil
}

// WriteDir writes a directory entry to the writer.
func WriteDir(w *zip.Writer, filename string) error {
	filename += "/" // Must have trailing slash to tell it it's a directory.
	fh := zip.FileHeader{
		Name:   filename,
		Method: zip.Store,
	}
	if _, err := w.CreateHeader(&fh); err != nil {
		return err
	}
	addExistingFile(w, filename, filename, 0, 0, 0)
	return nil
}

type nopCloser struct {
	io.Writer
}

func (w nopCloser) Close() error {
	return nil
}

// fixedCrc32 implements a Hash32 interface that just writes out a predetermined value.
// this is really cheating of course but serves our purposes here.
type fixedCrc32 struct {
	value uint32
}

func (crc fixedCrc32) Write(p []byte) (n int, err error) {
	return len(p), nil
}

func (crc fixedCrc32) Sum(b []byte) []byte {
	buf := make([]byte, 4)
	binary.LittleEndian.PutUint32(buf, crc.value)
	return b
}

func (crc fixedCrc32) Sum32() uint32 {
	return crc.value
}

func (crc fixedCrc32) Reset() {
}

func (crc fixedCrc32) Size() int {
	return 32
}

func (crc fixedCrc32) BlockSize() int {
	return 32
}
<|endoftext|>


<commit_before>package ssl

import (
	"crypto/tls"
	"crypto/x509"
	"errors"
	"io/ioutil"
	nethttp "net/http"
	"strings"

	"github.com/go-martini/martini"
	"github.com/outbrain/golib/log"
	"github.com/outbrain/orchestrator/go/config"
)

// Determine if a string element is in a string array
func HasString(elem string, arr []string) bool {
	for _, s := range arr {
		if s == elem {
			return true
		}
	}
	return false
}

// NewTLSConfig returns an initialized TLS configuration suitable for client
// authentication. If caFile is non-empty, it will be loaded.
func NewTLSConfig(caFile string, mutualTLS bool) (*tls.Config, error) {
	var c tls.Config

	// TLS 1.0 at a minimum (for mysql)
	c.MinVersion = tls.VersionTLS10
	c.PreferServerCipherSuites = true

	if mutualTLS {
		log.Info("MutualTLS requested, client certificates will be verified")
		c.ClientAuth = tls.VerifyClientCertIfGiven
	}
	if caFile != "" {
		data, err := ioutil.ReadFile(caFile)
		if err != nil {
			return &c, err
		}
		c.ClientCAs = x509.NewCertPool()
		if !c.ClientCAs.AppendCertsFromPEM(data) {
			return &c, errors.New("No certificates parsed")
		}
		log.Info("Read in CA file:", caFile)
	}
	c.BuildNameToCertificate()
	return &c, nil
}

// Verify that the OU of the presented client certificate matches the list
// of Valid OUs
func Verify(r *nethttp.Request, validOUs []string) error {
	if strings.Contains(r.URL.String(), config.Config.StatusEndpoint) && !config.Config.StatusOUVerify {
		return nil
	}
	if r.TLS == nil {
		return errors.New("No TLS")
	}
	for _, chain := range r.TLS.VerifiedChains {
		s := chain[0].Subject.OrganizationalUnit
		log.Debug("All OUs:", strings.Join(s, " "))
		for _, ou := range s {
			log.Debug("Client presented OU:", ou)
			if HasString(ou, validOUs) {
				log.Debug("Found valid OU:", ou)
				return nil
			}
		}
	}
	log.Error("No valid OUs found")
	return errors.New("Invalid OU")
}

// TODO: make this testable?
func VerifyOUs(validOUs []string) martini.Handler {
	return func(res nethttp.ResponseWriter, req *nethttp.Request, c martini.Context) {
		log.Debug("Verifying client OU")
		if err := Verify(req, validOUs); err != nil {
			nethttp.Error(res, err.Error(), nethttp.StatusUnauthorized)
		}
	}
}

// AppendKeyPair loads the given TLS key pair and appends it to
// tlsConfig.Certificates.
func AppendKeyPair(tlsConfig *tls.Config, certFile string, keyFile string) error {
	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
	if err != nil {
		return err
	}
	tlsConfig.Certificates = append(tlsConfig.Certificates, cert)
	return nil
}

// ListenAndServeTLS acts identically to http.ListenAndServeTLS, except that it
// expects TLS configuration.
// TODO: refactor so this is testable?
func ListenAndServeTLS(addr string, handler nethttp.Handler, tlsConfig *tls.Config) error {
	if addr == "" {
		// On unix Listen calls getaddrinfo to parse the port, so named ports are fine as long
		// as they exist in /etc/services
		addr = ":https"
	}
	l, err := tls.Listen("tcp", addr, tlsConfig)
	if err != nil {
		return err
	}
	return nethttp.Serve(l, handler)
}
<commit_msg>Bumping minimum TLS to TLS 1.2 Removing insecure ciphers<commit_after>package ssl

import (
	"crypto/tls"
	"crypto/x509"
	"errors"
	"io/ioutil"
	nethttp "net/http"
	"strings"

	"github.com/go-martini/martini"
	"github.com/outbrain/golib/log"
	"github.com/outbrain/orchestrator/go/config"
)

var cipherSuites = []uint16{
	tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
	tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
	tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
	tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
	tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
	tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
	tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
	tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
	tls.TLS_RSA_WITH_AES_128_CBC_SHA,
	tls.TLS_RSA_WITH_AES_256_CBC_SHA,
}

// Determine if a string element is in a string array
func HasString(elem string, arr []string) bool {
	for _, s := range arr {
		if s == elem {
			return true
		}
	}
	return false
}

// NewTLSConfig returns an initialized TLS configuration suitable for client
// authentication. If caFile is non-empty, it will be loaded.
func NewTLSConfig(caFile string, mutualTLS bool) (*tls.Config, error) {
	var c tls.Config

	// TLS 1.0 at a minimum (for mysql)
	c.MinVersion = tls.VersionTLS12
	// Remove insecure ciphers from the list
	c.CipherSuites = cipherSuites
	c.PreferServerCipherSuites = true

	if mutualTLS {
		log.Info("MutualTLS requested, client certificates will be verified")
		c.ClientAuth = tls.VerifyClientCertIfGiven
	}
	if caFile != "" {
		data, err := ioutil.ReadFile(caFile)
		if err != nil {
			return &c, err
		}
		c.ClientCAs = x509.NewCertPool()
		if !c.ClientCAs.AppendCertsFromPEM(data) {
			return &c, errors.New("No certificates parsed")
		}
		log.Info("Read in CA file:", caFile)
	}
	c.BuildNameToCertificate()
	return &c, nil
}

// Verify that the OU of the presented client certificate matches the list
// of Valid OUs
func Verify(r *nethttp.Request, validOUs []string) error {
	if strings.Contains(r.URL.String(), config.Config.StatusEndpoint) && !config.Config.StatusOUVerify {
		return nil
	}
	if r.TLS == nil {
		return errors.New("No TLS")
	}
	for _, chain := range r.TLS.VerifiedChains {
		s := chain[0].Subject.OrganizationalUnit
		log.Debug("All OUs:", strings.Join(s, " "))
		for _, ou := range s {
			log.Debug("Client presented OU:", ou)
			if HasString(ou, validOUs) {
				log.Debug("Found valid OU:", ou)
				return nil
			}
		}
	}
	log.Error("No valid OUs found")
	return errors.New("Invalid OU")
}

// TODO: make this testable?
func VerifyOUs(validOUs []string) martini.Handler {
	return func(res nethttp.ResponseWriter, req *nethttp.Request, c martini.Context) {
		log.Debug("Verifying client OU")
		if err := Verify(req, validOUs); err != nil {
			nethttp.Error(res, err.Error(), nethttp.StatusUnauthorized)
		}
	}
}

// AppendKeyPair loads the given TLS key pair and appends it to
// tlsConfig.Certificates.
func AppendKeyPair(tlsConfig *tls.Config, certFile string, keyFile string) error {
	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
	if err != nil {
		return err
	}
	tlsConfig.Certificates = append(tlsConfig.Certificates, cert)
	return nil
}

// ListenAndServeTLS acts identically to http.ListenAndServeTLS, except that it
// expects TLS configuration.
// TODO: refactor so this is testable?
func ListenAndServeTLS(addr string, handler nethttp.Handler, tlsConfig *tls.Config) error {
	if addr == "" {
		// On unix Listen calls getaddrinfo to parse the port, so named ports are fine as long
		// as they exist in /etc/services
		addr = ":https"
	}
	l, err := tls.Listen("tcp", addr, tlsConfig)
	if err != nil {
		return err
	}
	return nethttp.Serve(l, handler)
}
<|endoftext|>


<commit_before>package pwr

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"sync/atomic"

	"github.com/go-errors/errors"
	"github.com/itchio/wharf/counter"
	"github.com/itchio/wharf/pools"
	"github.com/itchio/wharf/pools/nullpool"
	"github.com/itchio/wharf/state"
)

// MaxWoundSize is how large AggregateWounds will let an aggregat
// wound get before passing it along to its consumer. The idea is, when
// we're verifying a large file, we can start healing it before it's done
// verifying. In practice, this would require sharing file descriptors,
// and so it isn't done yet.
const MaxWoundSize int64 = 4 * 1024 * 1024 // 4MB

// ValidatorContext holds both input and output parameters to the validation
// process (checking that a container corresponds to its signature: that all
// directories exist, symlinks exist and point to the right destinations, files
// exist and have the right content)
type ValidatorContext struct {
	WoundsPath string
	HealPath   string
	NumWorkers int

	Consumer *state.Consumer

	// FailFast makes Validate return Wounds as errors and stop checking
	FailFast bool

	// Result

	// internal
	Wounds         chan *Wound
	WoundsConsumer WoundsConsumer
}

// Validate checks the directory at target using the container info and hashes
// contained in signature. FailFast mode returns an error on the first corruption
// seen, other modes write wounds to a file or for a wounds consumer, like a healer.
func (vctx *ValidatorContext) Validate(target string, signature *SignatureInfo) error {
	if vctx.Consumer == nil {
		vctx.Consumer = &state.Consumer{}
	}

	numWorkers := vctx.NumWorkers
	if numWorkers == 0 {
		numWorkers = runtime.NumCPU() + 1
	}

	vctx.Wounds = make(chan *Wound, 1024)
	workerErrs := make(chan error, numWorkers)
	consumerErrs := make(chan error, 1)
	cancelled := make(chan struct{})

	var woundsStateConsumer *state.Consumer
	var healerProgress float64
	var bytesDone int64

	updateProgress := func() {
		currentBytesDone := atomic.LoadInt64(&bytesDone)
		scanProgress := float64(currentBytesDone) / float64(signature.Container.Size)

		if woundsStateConsumer == nil {
			vctx.Consumer.Progress(scanProgress)
		} else {
			vctx.Consumer.Progress(healerProgress)
		}
	}

	onProgress := func(delta int64) {
		atomic.AddInt64(&bytesDone, delta)
		updateProgress()
	}

	if vctx.FailFast {
		if vctx.WoundsPath != "" {
			return fmt.Errorf("ValidatorContext: FailFast is not compatible with WoundsPath")
		}
		if vctx.HealPath != "" {
			return fmt.Errorf("ValidatorContext: FailFast is not compatible with HealPath")
		}

		vctx.WoundsConsumer = &WoundsGuardian{}
	} else if vctx.WoundsPath != "" {
		vctx.WoundsConsumer = &WoundsWriter{
			WoundsPath: vctx.WoundsPath,
		}
	} else if vctx.HealPath != "" {
		// healers can deal with "everything missing"
		err := os.MkdirAll(target, 0755)
		if err != nil {
			return err
		}

		healer, err := NewHealer(vctx.HealPath, target)
		if err != nil {
			return err
		}

		woundsStateConsumer = &state.Consumer{
			OnProgress: func(progress float64) {
				healerProgress = progress
				updateProgress()
			},
			OnProgressLabel: func(label string) {
				vctx.Consumer.ProgressLabel(label)
			},
		}
		healer.SetConsumer(woundsStateConsumer)

		vctx.WoundsConsumer = healer
	} else {
		vctx.WoundsConsumer = &WoundsPrinter{
			Consumer: vctx.Consumer,
		}
	}

	go func() {
		consumerErrs <- vctx.WoundsConsumer.Do(signature.Container, vctx.Wounds)

		// throw away wounds until closed
		for range vctx.Wounds {
			// muffin
		}
	}()

	// validate dirs and symlinks first
	for dirIndex, dir := range signature.Container.Dirs {
		path := filepath.Join(target, filepath.FromSlash(dir.Path))
		stats, err := os.Lstat(path)
		if err != nil {
			if os.IsNotExist(err) {
				vctx.Wounds <- &Wound{
					Kind:  WoundKind_DIR,
					Index: int64(dirIndex),
				}
				continue
			} else {
				return err
			}
		}

		if !stats.IsDir() {
			vctx.Wounds <- &Wound{
				Kind:  WoundKind_DIR,
				Index: int64(dirIndex),
			}
			continue
		}
	}

	for symlinkIndex, symlink := range signature.Container.Symlinks {
		path := filepath.Join(target, filepath.FromSlash(symlink.Path))
		dest, err := os.Readlink(path)
		if err != nil {
			if os.IsNotExist(err) {
				vctx.Wounds <- &Wound{
					Kind:  WoundKind_SYMLINK,
					Index: int64(symlinkIndex),
				}
				continue
			} else {
				return err
			}
		}

		if dest != filepath.FromSlash(symlink.Dest) {
			vctx.Wounds <- &Wound{
				Kind:  WoundKind_SYMLINK,
				Index: int64(symlinkIndex),
			}
			continue
		}
	}

	fileIndices := make(chan int64)

	for i := 0; i < numWorkers; i++ {
		go vctx.validate(target, signature, fileIndices, workerErrs, onProgress, cancelled)
	}

	var retErr error
	sending := true

	for fileIndex := range signature.Container.Files {
		if !sending {
			break
		}

		select {
		case workerErr := <-workerErrs:
			workerErrs <- nil
			retErr = workerErr
			close(cancelled)
			sending = false

		case consumerErr := <-consumerErrs:
			consumerErrs <- nil
			retErr = consumerErr
			close(cancelled)
			sending = false

		case fileIndices <- int64(fileIndex):
			// just queued another file
		}
	}

	close(fileIndices)

	// wait for all workers to finish
	for i := 0; i < numWorkers; i++ {
		err := <-workerErrs
		if err != nil {
			if retErr == nil {
				retErr = err
			}
		}
	}

	close(vctx.Wounds)

	// wait for wound consumer to finish
	cErr := <-consumerErrs
	if cErr != nil {
		if retErr == nil {
			retErr = cErr
		}
	}

	return retErr
}

type onProgressFunc func(delta int64)

func (vctx *ValidatorContext) validate(target string, signature *SignatureInfo, fileIndices chan int64,
	errs chan error, onProgress onProgressFunc, cancelled chan struct{}) {

	var retErr error

	targetPool, err := pools.New(signature.Container, target)
	if err != nil {
		errs <- err
		return
	}

	defer func() {
		err := targetPool.Close()
		if err != nil {
			retErr = errors.Wrap(err, 1)
			return
		}

		errs <- retErr
	}()

	validatingPool := &ValidatingPool{
		Pool:      nullpool.New(signature.Container),
		Container: signature.Container,
		Signature: signature,

		Wounds: vctx.Wounds,
		WoundsFilter: func(wounds chan *Wound) chan *Wound {
			return AggregateWounds(wounds, MaxWoundSize)
		},
	}

	doOne := func(fileIndex int64) error {
		file := signature.Container.Files[fileIndex]

		var reader io.Reader
		reader, err = targetPool.GetReader(fileIndex)
		if err != nil {
			if os.IsNotExist(err) {
				// whole file is missing
				wound := &Wound{
					Kind:  WoundKind_FILE,
					Index: fileIndex,
					Start: 0,
					End:   file.Size,
				}
				onProgress(file.Size)

				select {
				case vctx.Wounds <- wound:
				case <-cancelled:
				}
				return nil
			}
			return err
		}

		var writer io.WriteCloser
		writer, err = validatingPool.GetWriter(fileIndex)
		if err != nil {
			return err
		}

		defer writer.Close()

		lastCount := int64(0)
		countingWriter := counter.NewWriterCallback(func(count int64) {
			delta := count - lastCount
			onProgress(delta)
			lastCount = count
		}, writer)

		var writtenBytes int64
		writtenBytes, err = io.Copy(countingWriter, reader)
		if err != nil {
			return err
		}

		if writtenBytes != file.Size {
			onProgress(file.Size - writtenBytes)
			wound := &Wound{
				Kind:  WoundKind_FILE,
				Index: fileIndex,
				Start: writtenBytes,
				End:   file.Size,
			}

			select {
			case vctx.Wounds <- wound:
			case <-cancelled:
			}
		}

		return nil
	}

	for {
		select {
		case fileIndex, ok := <-fileIndices:
			if !ok {
				// no more work
				return
			}

			err := doOne(fileIndex)
			if err != nil {
				if retErr == nil {
					retErr = err
				}
				return
			}
		case <-cancelled:
			// cancelled
			return
		}
	}
}

// AssertValid validates target in FailFast mode - it's a shorthand
// so that setting up ValidatorContext isn't needed
func AssertValid(target string, signature *SignatureInfo) error {
	vctx := &ValidatorContext{
		FailFast: true,
		Consumer: &state.Consumer{},
	}

	err := vctx.Validate(target, signature)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Use IsNotExist variant that handles nested errors<commit_after>package pwr

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"sync/atomic"

	"github.com/go-errors/errors"
	"github.com/itchio/wharf/counter"
	"github.com/itchio/wharf/pools"
	"github.com/itchio/wharf/pools/nullpool"
	"github.com/itchio/wharf/state"
)

// MaxWoundSize is how large AggregateWounds will let an aggregat
// wound get before passing it along to its consumer. The idea is, when
// we're verifying a large file, we can start healing it before it's done
// verifying. In practice, this would require sharing file descriptors,
// and so it isn't done yet.
const MaxWoundSize int64 = 4 * 1024 * 1024 // 4MB

// ValidatorContext holds both input and output parameters to the validation
// process (checking that a container corresponds to its signature: that all
// directories exist, symlinks exist and point to the right destinations, files
// exist and have the right content)
type ValidatorContext struct {
	WoundsPath string
	HealPath   string
	NumWorkers int

	Consumer *state.Consumer

	// FailFast makes Validate return Wounds as errors and stop checking
	FailFast bool

	// Result

	// internal
	Wounds         chan *Wound
	WoundsConsumer WoundsConsumer
}

// Validate checks the directory at target using the container info and hashes
// contained in signature. FailFast mode returns an error on the first corruption
// seen, other modes write wounds to a file or for a wounds consumer, like a healer.
func (vctx *ValidatorContext) Validate(target string, signature *SignatureInfo) error {
	if vctx.Consumer == nil {
		vctx.Consumer = &state.Consumer{}
	}

	numWorkers := vctx.NumWorkers
	if numWorkers == 0 {
		numWorkers = runtime.NumCPU() + 1
	}

	vctx.Wounds = make(chan *Wound, 1024)
	workerErrs := make(chan error, numWorkers)
	consumerErrs := make(chan error, 1)
	cancelled := make(chan struct{})

	var woundsStateConsumer *state.Consumer
	var healerProgress float64
	var bytesDone int64

	updateProgress := func() {
		currentBytesDone := atomic.LoadInt64(&bytesDone)
		scanProgress := float64(currentBytesDone) / float64(signature.Container.Size)

		if woundsStateConsumer == nil {
			vctx.Consumer.Progress(scanProgress)
		} else {
			vctx.Consumer.Progress(healerProgress)
		}
	}

	onProgress := func(delta int64) {
		atomic.AddInt64(&bytesDone, delta)
		updateProgress()
	}

	if vctx.FailFast {
		if vctx.WoundsPath != "" {
			return fmt.Errorf("ValidatorContext: FailFast is not compatible with WoundsPath")
		}
		if vctx.HealPath != "" {
			return fmt.Errorf("ValidatorContext: FailFast is not compatible with HealPath")
		}

		vctx.WoundsConsumer = &WoundsGuardian{}
	} else if vctx.WoundsPath != "" {
		vctx.WoundsConsumer = &WoundsWriter{
			WoundsPath: vctx.WoundsPath,
		}
	} else if vctx.HealPath != "" {
		// healers can deal with "everything missing"
		err := os.MkdirAll(target, 0755)
		if err != nil {
			return err
		}

		healer, err := NewHealer(vctx.HealPath, target)
		if err != nil {
			return err
		}

		woundsStateConsumer = &state.Consumer{
			OnProgress: func(progress float64) {
				healerProgress = progress
				updateProgress()
			},
			OnProgressLabel: func(label string) {
				vctx.Consumer.ProgressLabel(label)
			},
		}
		healer.SetConsumer(woundsStateConsumer)

		vctx.WoundsConsumer = healer
	} else {
		vctx.WoundsConsumer = &WoundsPrinter{
			Consumer: vctx.Consumer,
		}
	}

	go func() {
		consumerErrs <- vctx.WoundsConsumer.Do(signature.Container, vctx.Wounds)

		// throw away wounds until closed
		for range vctx.Wounds {
			// muffin
		}
	}()

	// validate dirs and symlinks first
	for dirIndex, dir := range signature.Container.Dirs {
		path := filepath.Join(target, filepath.FromSlash(dir.Path))
		stats, err := os.Lstat(path)
		if err != nil {
			if IsNotExist(err) {
				vctx.Wounds <- &Wound{
					Kind:  WoundKind_DIR,
					Index: int64(dirIndex),
				}
				continue
			} else {
				return err
			}
		}

		if !stats.IsDir() {
			vctx.Wounds <- &Wound{
				Kind:  WoundKind_DIR,
				Index: int64(dirIndex),
			}
			continue
		}
	}

	for symlinkIndex, symlink := range signature.Container.Symlinks {
		path := filepath.Join(target, filepath.FromSlash(symlink.Path))
		dest, err := os.Readlink(path)
		if err != nil {
			if os.IsNotExist(err) {
				vctx.Wounds <- &Wound{
					Kind:  WoundKind_SYMLINK,
					Index: int64(symlinkIndex),
				}
				continue
			} else {
				return err
			}
		}

		if dest != filepath.FromSlash(symlink.Dest) {
			vctx.Wounds <- &Wound{
				Kind:  WoundKind_SYMLINK,
				Index: int64(symlinkIndex),
			}
			continue
		}
	}

	fileIndices := make(chan int64)

	for i := 0; i < numWorkers; i++ {
		go vctx.validate(target, signature, fileIndices, workerErrs, onProgress, cancelled)
	}

	var retErr error
	sending := true

	for fileIndex := range signature.Container.Files {
		if !sending {
			break
		}

		select {
		case workerErr := <-workerErrs:
			workerErrs <- nil
			retErr = workerErr
			close(cancelled)
			sending = false

		case consumerErr := <-consumerErrs:
			consumerErrs <- nil
			retErr = consumerErr
			close(cancelled)
			sending = false

		case fileIndices <- int64(fileIndex):
			// just queued another file
		}
	}

	close(fileIndices)

	// wait for all workers to finish
	for i := 0; i < numWorkers; i++ {
		err := <-workerErrs
		if err != nil {
			if retErr == nil {
				retErr = err
			}
		}
	}

	close(vctx.Wounds)

	// wait for wound consumer to finish
	cErr := <-consumerErrs
	if cErr != nil {
		if retErr == nil {
			retErr = cErr
		}
	}

	return retErr
}

type onProgressFunc func(delta int64)

func (vctx *ValidatorContext) validate(target string, signature *SignatureInfo, fileIndices chan int64,
	errs chan error, onProgress onProgressFunc, cancelled chan struct{}) {

	var retErr error

	targetPool, err := pools.New(signature.Container, target)
	if err != nil {
		errs <- err
		return
	}

	defer func() {
		err := targetPool.Close()
		if err != nil {
			retErr = errors.Wrap(err, 1)
			return
		}

		errs <- retErr
	}()

	validatingPool := &ValidatingPool{
		Pool:      nullpool.New(signature.Container),
		Container: signature.Container,
		Signature: signature,

		Wounds: vctx.Wounds,
		WoundsFilter: func(wounds chan *Wound) chan *Wound {
			return AggregateWounds(wounds, MaxWoundSize)
		},
	}

	doOne := func(fileIndex int64) error {
		file := signature.Container.Files[fileIndex]

		var reader io.Reader
		reader, err = targetPool.GetReader(fileIndex)
		if err != nil {
			if IsNotExist(err) {
				// whole file is missing
				wound := &Wound{
					Kind:  WoundKind_FILE,
					Index: fileIndex,
					Start: 0,
					End:   file.Size,
				}
				onProgress(file.Size)

				select {
				case vctx.Wounds <- wound:
				case <-cancelled:
				}
				return nil
			}
			return err
		}

		var writer io.WriteCloser
		writer, err = validatingPool.GetWriter(fileIndex)
		if err != nil {
			return err
		}

		defer writer.Close()

		lastCount := int64(0)
		countingWriter := counter.NewWriterCallback(func(count int64) {
			delta := count - lastCount
			onProgress(delta)
			lastCount = count
		}, writer)

		var writtenBytes int64
		writtenBytes, err = io.Copy(countingWriter, reader)
		if err != nil {
			return err
		}

		if writtenBytes != file.Size {
			onProgress(file.Size - writtenBytes)
			wound := &Wound{
				Kind:  WoundKind_FILE,
				Index: fileIndex,
				Start: writtenBytes,
				End:   file.Size,
			}

			select {
			case vctx.Wounds <- wound:
			case <-cancelled:
			}
		}

		return nil
	}

	for {
		select {
		case fileIndex, ok := <-fileIndices:
			if !ok {
				// no more work
				return
			}

			err := doOne(fileIndex)
			if err != nil {
				if retErr == nil {
					retErr = err
				}
				return
			}
		case <-cancelled:
			// cancelled
			return
		}
	}
}

// AssertValid validates target in FailFast mode - it's a shorthand
// so that setting up ValidatorContext isn't needed
func AssertValid(target string, signature *SignatureInfo) error {
	vctx := &ValidatorContext{
		FailFast: true,
		Consumer: &state.Consumer{},
	}

	err := vctx.Validate(target, signature)
	if err != nil {
		return err
	}

	return nil
}

// IsNotExist is a variant of os.IsNotExist that works with nested errors
func IsNotExist(err error) bool {
	if se, ok := err.(*errors.Error); ok {
		return IsNotExist(se.Err)
	}
	return os.IsNotExist(err)
}
<|endoftext|>


<commit_before>package audit

import (
	"github.com/stretchr/testify/assert"
	"gopkg.in/DATA-DOG/go-sqlmock.v1"
	"testing"

	"github.com/eirka/eirka-libs/db"
)

func TestAudit(t *testing.T) {

	var err error

	mock, err := db.NewTestDb()
	assert.NoError(t, err, "An error was not expected")

	rows := sqlmock.NewRows([]string{"role"}).AddRow(3)

	mock.ExpectPrepare(`INSERT INTO audit \(user_id,ib_id,audit_type,audit_ip,audit_time,audit_action,audit_info\)`).
		ExpectExec().
		WithArgs(1, 1, UserLog, "10.0.0.1", "NOW()", AuditEmailUpdate, "meta info")

	audit := Audit{
		User:   1,
		Ib:     1,
		Type:   UserLog,
		Ip:     "10.0.0.1",
		Action: AuditEmailUpdate,
		Info:   "meta info",
	}

	// submit audit
	err = audit.Submit()
	assert.NoError(t, err, "An error was not expected")

}
<commit_msg>add audit test<commit_after>package audit

import (
	"github.com/stretchr/testify/assert"
	"gopkg.in/DATA-DOG/go-sqlmock.v1"
	"testing"

	"github.com/eirka/eirka-libs/db"
)

func TestAudit(t *testing.T) {

	var err error

	mock, err := db.NewTestDb()
	assert.NoError(t, err, "An error was not expected")

	mock.ExpectPrepare(`INSERT INTO audit \(user_id,ib_id,audit_type,audit_ip,audit_time,audit_action,audit_info\)`).
		ExpectExec().
		WithArgs(1, 1, UserLog, "10.0.0.1", "NOW()", AuditEmailUpdate, "meta info")

	audit := Audit{
		User:   1,
		Ib:     1,
		Type:   UserLog,
		Ip:     "10.0.0.1",
		Action: AuditEmailUpdate,
		Info:   "meta info",
	}

	// submit audit
	err = audit.Submit()
	assert.NoError(t, err, "An error was not expected")

}
<|endoftext|>


<commit_before>package gate

import (
	"github.com/golang/protobuf/proto"
	"github.com/name5566/leaf/log"
	"github.com/name5566/leaf/network"
	"github.com/name5566/leaf/network/json"
	"github.com/name5566/leaf/network/protobuf"
	"reflect"
)

type TCPGate struct {
	Addr              string
	MaxConnNum        int
	PendingWriteNum   int
	LenMsgLen         int
	MinMsgLen         uint32
	MaxMsgLen         uint32
	LittleEndian      bool
	JSONProcessor     *json.Processor
	ProtobufProcessor *protobuf.Processor
}

func (gate *TCPGate) Run(closeSig chan bool) {
	server := new(network.TCPServer)
	server.Addr = gate.Addr
	server.MaxConnNum = gate.MaxConnNum
	server.PendingWriteNum = gate.PendingWriteNum
	server.LenMsgLen = gate.LenMsgLen
	server.MinMsgLen = gate.MinMsgLen
	server.MaxMsgLen = gate.MaxMsgLen
	server.LittleEndian = gate.LittleEndian
	server.NewAgent = func(conn *network.TCPConn) network.Agent {
		a := new(TCPAgent)
		a.conn = conn
		a.gate = gate
		return a
	}

	server.Start()
	<-closeSig
	server.Close()
}

func (gate *TCPGate) OnDestroy() {}

type TCPAgent struct {
	conn *network.TCPConn
	gate *TCPGate
}

func (a *TCPAgent) Run() {
	for {
		data, err := a.conn.ReadMsg()
		if err != nil {
			log.Debug("read message error: %v", err)
			break
		}

		if a.gate.JSONProcessor != nil {
			// json
			msg, err := a.gate.JSONProcessor.Unmarshal(data)
			if err != nil {
				log.Debug("unmarshal json error: %v", err)
				break
			}
			err = a.gate.JSONProcessor.Route(msg, Agent(a))
			if err != nil {
				log.Debug("route message error: %v", err)
				break
			}
		} else if a.gate.ProtobufProcessor != nil {
			// protobuf
			msg, err := a.gate.ProtobufProcessor.Unmarshal(data)
			if err != nil {
				log.Debug("unmarshal protobuf error: %v", err)
				break
			}
			err = a.gate.ProtobufProcessor.Route(msg, Agent(a))
			if err != nil {
				log.Debug("route message error: %v", err)
				break
			}
		}
	}
}

func (a *TCPAgent) OnClose() {}

func (a *TCPAgent) WriteMsg(msg interface{}) {
	if a.gate.JSONProcessor != nil {
		// json
		data, err := a.gate.JSONProcessor.Marshal(msg)
		if err != nil {
			log.Error("marshal json %v error: %v", reflect.TypeOf(msg), err)
			return
		}
		a.conn.WriteMsg(data)
	} else if a.gate.ProtobufProcessor != nil {
		// protobuf
		id, data, err := a.gate.ProtobufProcessor.Marshal(msg.(proto.Message))
		if err != nil {
			log.Error("marshal protobuf %v error: %v", reflect.TypeOf(msg), err)
			return
		}
		a.conn.WriteMsg(id, data)
	}
}

func (a *TCPAgent) Close() {
	a.conn.Close()
}
<commit_msg>chanrpc for a agent<commit_after>package gate

import (
	"github.com/golang/protobuf/proto"
	"github.com/name5566/leaf/chanrpc"
	"github.com/name5566/leaf/log"
	"github.com/name5566/leaf/network"
	"github.com/name5566/leaf/network/json"
	"github.com/name5566/leaf/network/protobuf"
	"reflect"
)

type TCPGate struct {
	Addr              string
	MaxConnNum        int
	PendingWriteNum   int
	LenMsgLen         int
	MinMsgLen         uint32
	MaxMsgLen         uint32
	LittleEndian      bool
	JSONProcessor     *json.Processor
	ProtobufProcessor *protobuf.Processor
	AgentChanRPC      *chanrpc.Server
}

func (gate *TCPGate) Run(closeSig chan bool) {
	server := new(network.TCPServer)
	server.Addr = gate.Addr
	server.MaxConnNum = gate.MaxConnNum
	server.PendingWriteNum = gate.PendingWriteNum
	server.LenMsgLen = gate.LenMsgLen
	server.MinMsgLen = gate.MinMsgLen
	server.MaxMsgLen = gate.MaxMsgLen
	server.LittleEndian = gate.LittleEndian
	server.NewAgent = func(conn *network.TCPConn) network.Agent {
		a := new(TCPAgent)
		a.conn = conn
		a.gate = gate
		return a
	}

	server.Start()
	<-closeSig
	server.Close()
}

func (gate *TCPGate) OnDestroy() {}

type TCPAgent struct {
	conn *network.TCPConn
	gate *TCPGate
}

func (a *TCPAgent) Run() {
	for {
		data, err := a.conn.ReadMsg()
		if err != nil {
			log.Debug("read message error: %v", err)
			break
		}

		if a.gate.JSONProcessor != nil {
			// json
			msg, err := a.gate.JSONProcessor.Unmarshal(data)
			if err != nil {
				log.Debug("unmarshal json error: %v", err)
				break
			}
			err = a.gate.JSONProcessor.Route(msg, Agent(a))
			if err != nil {
				log.Debug("route message error: %v", err)
				break
			}
		} else if a.gate.ProtobufProcessor != nil {
			// protobuf
			msg, err := a.gate.ProtobufProcessor.Unmarshal(data)
			if err != nil {
				log.Debug("unmarshal protobuf error: %v", err)
				break
			}
			err = a.gate.ProtobufProcessor.Route(msg, Agent(a))
			if err != nil {
				log.Debug("route message error: %v", err)
				break
			}
		}
	}
}

func (a *TCPAgent) OnClose() {
	if a.gate.AgentChanRPC != nil {
		err := a.gate.AgentChanRPC.Open(0).Call0("Close", a)
		if err != nil {
			log.Error("chanrpc error: %v", err)
		}
	}
}

func (a *TCPAgent) WriteMsg(msg interface{}) {
	if a.gate.JSONProcessor != nil {
		// json
		data, err := a.gate.JSONProcessor.Marshal(msg)
		if err != nil {
			log.Error("marshal json %v error: %v", reflect.TypeOf(msg), err)
			return
		}
		a.conn.WriteMsg(data)
	} else if a.gate.ProtobufProcessor != nil {
		// protobuf
		id, data, err := a.gate.ProtobufProcessor.Marshal(msg.(proto.Message))
		if err != nil {
			log.Error("marshal protobuf %v error: %v", reflect.TypeOf(msg), err)
			return
		}
		a.conn.WriteMsg(id, data)
	}
}

func (a *TCPAgent) Close() {
	a.conn.Close()
}
<|endoftext|>


<commit_before>// MIT License
//
// Copyright (c) 2018 Stefan Wichmann
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
package main

import (
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"runtime"
	"strings"
)

func downloadLatestReleaseInfo(url string) (releaseName string, assetURL string, err error) {
	resp, err := http.Get(url)
	if err != nil {
		return "", "", err
	}
	defer resp.Body.Close()

	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", "", err
	}

	var data interface{}
	err = json.Unmarshal(b, &data)
	if err != nil {
		return "", "", err
	}

	releaseInfo := data.(map[string]interface{})
	var name string
	if releaseInfo["tag_name"] != nil {
		name = releaseInfo["tag_name"].(string)
	} else if releaseInfo["name"] != nil {
		name = releaseInfo["name"].(string)
	} else {
		return "", "", errors.New("No releases available")
	}

	releaseAssets := releaseInfo["assets"].([]interface{})
	for _, asset := range releaseAssets {
		jsonAsset := asset.(map[string]interface{})

		match, url := assetMatchesPlattform(jsonAsset)
		if match {
			return name, url, nil
		}
	}

	return "", "", errors.New("No matching release found")
}

func assetMatchesPlattform(asset map[string]interface{}) (bool, string) {
	// match content type
	contentType := asset["content_type"].(string)
	if !(strings.Contains(contentType, "application/gzip") || strings.Contains(contentType, "application/zip")) {
		return false, ""
	}

	// match os and arch
	os := runtime.GOOS
	plattform := runtime.GOARCH
	assetName := asset["name"].(string)
	if !strings.Contains(assetName, os) || !strings.Contains(assetName, plattform) {
		return false, ""
	}

	// match file extension
	if !(strings.Contains(assetName, "zip") || strings.Contains(assetName, "tar.gz")) {
		return false, ""
	}

	return true, asset["browser_download_url"].(string)
}

func downloadReleaseArchive(url string) (archive string, err error) {
	// Create the tempfile in default location
	out, err := ioutil.TempFile("", "update")
	if err != nil {
		return "", err
	}
	defer out.Close()

	// Get the data
	resp, err := http.Get(url)
	if err != nil {
		os.Remove(out.Name())
		return "", err
	}
	defer resp.Body.Close()

	// Writer the body to file
	_, err = io.Copy(out, resp.Body)
	if err != nil {
		os.Remove(out.Name())
		return "", err
	}

	return out.Name(), nil
}
<commit_msg>Add check for arm64 build<commit_after>// MIT License
//
// Copyright (c) 2018 Stefan Wichmann
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
package main

import (
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"runtime"
	"strings"
)

func downloadLatestReleaseInfo(url string) (releaseName string, assetURL string, err error) {
	resp, err := http.Get(url)
	if err != nil {
		return "", "", err
	}
	defer resp.Body.Close()

	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", "", err
	}

	var data interface{}
	err = json.Unmarshal(b, &data)
	if err != nil {
		return "", "", err
	}

	releaseInfo := data.(map[string]interface{})
	var name string
	if releaseInfo["tag_name"] != nil {
		name = releaseInfo["tag_name"].(string)
	} else if releaseInfo["name"] != nil {
		name = releaseInfo["name"].(string)
	} else {
		return "", "", errors.New("No releases available")
	}

	releaseAssets := releaseInfo["assets"].([]interface{})
	for _, asset := range releaseAssets {
		jsonAsset := asset.(map[string]interface{})

		match, url := assetMatchesPlattform(jsonAsset)
		if match {
			return name, url, nil
		}
	}

	return "", "", errors.New("No matching release found")
}

func assetMatchesPlattform(asset map[string]interface{}) (bool, string) {
	// match content type
	contentType := asset["content_type"].(string)
	if !(strings.Contains(contentType, "application/gzip") || strings.Contains(contentType, "application/zip")) {
		return false, ""
	}

	// match os and arch
	os := runtime.GOOS
	plattform := runtime.GOARCH
	assetName := asset["name"].(string)

	if !strings.Contains(assetName, os) || !strings.Contains(assetName, plattform) {
		return false, ""
	}

	// special case for arm64 vs arm, skip arm64 builds
	if plattform == "arm" && strings.Contains(assetName, "arm64") {
		return false, ""
	}

	// match file extension
	if !(strings.Contains(assetName, "zip") || strings.Contains(assetName, "tar.gz")) {
		return false, ""
	}

	return true, asset["browser_download_url"].(string)
}

func downloadReleaseArchive(url string) (archive string, err error) {
	// Create the tempfile in default location
	out, err := ioutil.TempFile("", "update")
	if err != nil {
		return "", err
	}
	defer out.Close()

	// Get the data
	resp, err := http.Get(url)
	if err != nil {
		os.Remove(out.Name())
		return "", err
	}
	defer resp.Body.Close()

	// Writer the body to file
	_, err = io.Copy(out, resp.Body)
	if err != nil {
		os.Remove(out.Name())
		return "", err
	}

	return out.Name(), nil
}
<|endoftext|>


<commit_before>// Copyright 2016 The go-vgo Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// https://github.com/go-vgo/robotgo/blob/master/LICENSE
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

// +build darwin windows

package robotgo

import (
	"testing"

	"github.com/vcaesar/tt"
)

func TestColor(t *testing.T) {
	s := GetPixelColor(10, 10)
	tt.IsType(t, "string", s)
	tt.NotEmpty(t, s)

	c := GetPxColor(10, 10)
	s1 := PadHex(c)
	tt.Equal(t, s, s1)
}

func TestSize(t *testing.T) {
	x, y := GetScreenSize()
	tt.NotZero(t, x)
	tt.NotZero(t, y)

	x, y = GetScaleSize()
	tt.NotZero(t, x)
	tt.NotZero(t, y)
}

func TestMoveMouse(t *testing.T) {
	MoveMouse(20, 20)
	MilliSleep(10)
	x, y := GetMousePos()

	tt.Equal(t, 20, x)
	tt.Equal(t, 20, y)
}

func TestMoveMouseSmooth(t *testing.T) {
	b := MoveMouseSmooth(100, 100)
	MilliSleep(10)
	x, y := GetMousePos()

	tt.True(t, b)
	tt.Equal(t, 100, x)
	tt.Equal(t, 100, y)
}

func TestDragMouse(t *testing.T) {
	DragMouse(500, 500)
	MilliSleep(10)
	x, y := GetMousePos()

	tt.Equal(t, 500, x)
	tt.Equal(t, 500, y)
}

func TestScrollMouse(t *testing.T) {
	ScrollMouse(120, "up")
	MilliSleep(100)

	Scroll(210, 210)
}

func TestMoveRelative(t *testing.T) {
	Move(200, 200)
	MilliSleep(10)

	MoveRelative(10, -10)
	MilliSleep(10)

	x, y := GetMousePos()
	tt.Equal(t, 210, x)
	tt.Equal(t, 190, y)
}

func TestMoveSmoothRelative(t *testing.T) {
	Move(200, 200)
	MilliSleep(10)

	MoveSmoothRelative(10, -10)
	MilliSleep(10)

	x, y := GetMousePos()
	tt.Equal(t, 210, x)
	tt.Equal(t, 190, y)
}

func TestMouseToggle(t *testing.T) {
	e := MouseToggle("up", "right")
	tt.Zero(t, e)
}

func TestKey(t *testing.T) {
	e := KeyTap("v", "cmd")
	tt.Empty(t, e)

	e = KeyToggle("v", "up")
	tt.Empty(t, e)
}

func TestClip(t *testing.T) {
	err := WriteAll("s")
	tt.Nil(t, err)

	s, e := ReadAll()
	tt.Equal(t, "s", s)
	tt.Nil(t, e)
}

func TestTypeStr(t *testing.T) {
	c := CharCodeAt("s", 0)
	tt.Equal(t, 115, c)

	e := PasteStr("s")
	tt.Empty(t, e)

	uc := toUC("s")
	tt.Equal(t, "[s]", uc)
}

func TestKeyCode(t *testing.T) {
	m := MouseMap["left"]
	tt.Equal(t, 1, m)

	k := Keycode["1"]
	tt.Equal(t, 2, k)
}

func TestBitmap(t *testing.T) {
	bit := CaptureScreen()
	tt.NotNil(t, bit)
	e := SaveBitmap(bit, "robot_test.png")
	tt.Empty(t, e)

	bit1 := OpenBitmap("robot_test.png")
	b := tt.TypeOf(bit, bit1)
	tt.True(t, b)
	tt.NotNil(t, bit1)
}

func TestPs(t *testing.T) {
	id, err := Pids()
	tt.Not(t, "[]", id)
	tt.IsType(t, "[]int32", id)
	tt.Nil(t, err)

	b, e := PidExists(id[0])
	tt.Bool(t, b)
	tt.Nil(t, e)

	n, e := FindName(id[0])
	tt.NotEmpty(t, n)
	tt.Nil(t, e)
}

// func TestAlert(t *testing.T) {
// 	go func() {
// 		MilliSleep(200)
// 		KeyTap("enter")
// 		log.Println("tap...")
// 	}()

// 	i := ShowAlert("t", "msg")
// 	tt.Zero(t, i)
// }
<commit_msg>add Process() function test code<commit_after>// Copyright 2016 The go-vgo Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// https://github.com/go-vgo/robotgo/blob/master/LICENSE
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

// +build darwin windows

package robotgo

import (
	"testing"

	"github.com/vcaesar/tt"
)

func TestColor(t *testing.T) {
	s := GetPixelColor(10, 10)
	tt.IsType(t, "string", s)
	tt.NotEmpty(t, s)

	c := GetPxColor(10, 10)
	s1 := PadHex(c)
	tt.Equal(t, s, s1)
}

func TestSize(t *testing.T) {
	x, y := GetScreenSize()
	tt.NotZero(t, x)
	tt.NotZero(t, y)

	x, y = GetScaleSize()
	tt.NotZero(t, x)
	tt.NotZero(t, y)
}

func TestMoveMouse(t *testing.T) {
	MoveMouse(20, 20)
	MilliSleep(10)
	x, y := GetMousePos()

	tt.Equal(t, 20, x)
	tt.Equal(t, 20, y)
}

func TestMoveMouseSmooth(t *testing.T) {
	b := MoveMouseSmooth(100, 100)
	MilliSleep(10)
	x, y := GetMousePos()

	tt.True(t, b)
	tt.Equal(t, 100, x)
	tt.Equal(t, 100, y)
}

func TestDragMouse(t *testing.T) {
	DragMouse(500, 500)
	MilliSleep(10)
	x, y := GetMousePos()

	tt.Equal(t, 500, x)
	tt.Equal(t, 500, y)
}

func TestScrollMouse(t *testing.T) {
	ScrollMouse(120, "up")
	MilliSleep(100)

	Scroll(210, 210)
}

func TestMoveRelative(t *testing.T) {
	Move(200, 200)
	MilliSleep(10)

	MoveRelative(10, -10)
	MilliSleep(10)

	x, y := GetMousePos()
	tt.Equal(t, 210, x)
	tt.Equal(t, 190, y)
}

func TestMoveSmoothRelative(t *testing.T) {
	Move(200, 200)
	MilliSleep(10)

	MoveSmoothRelative(10, -10)
	MilliSleep(10)

	x, y := GetMousePos()
	tt.Equal(t, 210, x)
	tt.Equal(t, 190, y)
}

func TestMouseToggle(t *testing.T) {
	e := MouseToggle("up", "right")
	tt.Zero(t, e)
}

func TestKey(t *testing.T) {
	e := KeyTap("v", "cmd")
	tt.Empty(t, e)

	e = KeyToggle("v", "up")
	tt.Empty(t, e)
}

func TestClip(t *testing.T) {
	err := WriteAll("s")
	tt.Nil(t, err)

	s, e := ReadAll()
	tt.Equal(t, "s", s)
	tt.Nil(t, e)
}

func TestTypeStr(t *testing.T) {
	c := CharCodeAt("s", 0)
	tt.Equal(t, 115, c)

	e := PasteStr("s")
	tt.Empty(t, e)

	uc := toUC("s")
	tt.Equal(t, "[s]", uc)
}

func TestKeyCode(t *testing.T) {
	m := MouseMap["left"]
	tt.Equal(t, 1, m)

	k := Keycode["1"]
	tt.Equal(t, 2, k)
}

func TestBitmap(t *testing.T) {
	bit := CaptureScreen()
	tt.NotNil(t, bit)
	e := SaveBitmap(bit, "robot_test.png")
	tt.Empty(t, e)

	bit1 := OpenBitmap("robot_test.png")
	b := tt.TypeOf(bit, bit1)
	tt.True(t, b)
	tt.NotNil(t, bit1)
}

func TestPs(t *testing.T) {
	id, err := Pids()
	tt.Not(t, "[]", id)
	tt.IsType(t, "[]int32", id)
	tt.Nil(t, err)

	ps, e := Process()
	tt.Not(t, "[]", ps)
	tt.IsType(t, "[]robotgo.Nps", ps)
	tt.Nil(t, err)

	b, e := PidExists(id[0])
	tt.Bool(t, b)
	tt.Nil(t, e)

	n, e := FindName(id[0])
	tt.NotEmpty(t, n)
	tt.Nil(t, e)
}

// func TestAlert(t *testing.T) {
// 	go func() {
// 		MilliSleep(200)
// 		KeyTap("enter")
// 		log.Println("tap...")
// 	}()

// 	i := ShowAlert("t", "msg")
// 	tt.Zero(t, i)
// }
<|endoftext|>


<commit_before>package scaffold

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"text/template"
)

// FuncMap provides functions to the template.
// New functions can be added as needed. The usual restrictions for
// text/template.FuncMap apply (see http://golang.org/pkg/text/template/#FuncMap)
var FuncMap = template.FuncMap{
	"replace":    Replace,
	"camelCase1": CamelCase1,
	"camelCase2": CamelCase2,
	"title":      strings.Title,
	"toLower":    strings.ToLower,
	"toUpper":    strings.ToUpper,
	"trim":       strings.Trim,
}

// CamelCase1 converts a string in snake_case to CamelCase where the first letter of each word is capitalized
func CamelCase1(src string) string {
	s := strings.Split(src, "_")
	for i, _ := range s {
		s[i] = strings.Title(s[i])
	}
	return strings.Join(s, "")
}

// CamelCase2 converts a string in snake_case to camelCase where the first letter of each but the first word is capitalized
func CamelCase2(src string) string {
	s := strings.Split(src, "_")
	for i, _ := range s {
		if i != 0 {
			s[i] = strings.Title(s[i])
		}
	}
	return strings.Join(s, "")
}

// Replace replaces every occurence of old in s by new
func Replace(s, old, new string) string {
	return strings.Replace(s, old, new, -1)
}

// writeFile creates the given file with the given content if isTest is true.
// Needed directories are created on the fly and each file name is written to log
// if log is not nil.
// If isTest is true, no files and directories are created.
func writeFile(file string, content []byte, log io.Writer, isTest bool) error {
	dir := filepath.Dir(file)
	s, err := os.Stat(dir)
	if err != nil {
		if !isTest {
			os.MkdirAll(dir, 0770)
		}
	} else {
		if !s.IsDir() {
			return fmt.Errorf("not a directory: %#v", dir)
		}
	}
	if log != nil {
		log.Write([]byte(file + "\n"))
	}

	if !isTest {
		return ioutil.WriteFile(file, content, 0664)
	}
	return nil
}

// parseGenerator creates files and directories beneath baseDir as defined in the reader.
// The file names are written to log if it is not nil.
// If isTest is true, no files and directories are created.
func parseGenerator(baseDir string, rd io.Reader, log io.Writer, isTest bool) error {
	scanner := bufio.NewScanner(rd)
	var file string
	var dir = baseDir
	var bf bytes.Buffer
	var line = -1
	for scanner.Scan() {
		line++
		s := scanner.Text()
		if strings.HasPrefix(s, ">>>") {
			fd := strings.TrimSpace(strings.TrimPrefix(s, ">>>"))
			if fd[len(fd)-1] == '/' {
				dir = filepath.Join(dir, fd)
				file = ""
			} else {
				if file != "" {
					return fmt.Errorf("syntax error in line %d embedding file within file is not allowed (%#v inside %#v)", line, fd, file)
				}
				file = filepath.Join(dir, fd)
			}
			continue
		}

		if strings.HasPrefix(s, "<<<") {
			fd := strings.TrimSpace(strings.TrimPrefix(s, "<<<"))
			if fd[len(fd)-1] == '/' {
				dirName := filepath.Base(dir) + "/"
				if dirName != fd {
					return fmt.Errorf("syntax error in line %d closing dir %#v but should close dir %#v", line, fd, dirName)
				}
				dir = filepath.Dir(dir)
			} else {
				base := filepath.Base(file)
				if base != fd {
					return fmt.Errorf("syntax error in line %d closing file %#v but should close file %#v", line, fd, base)
				}
				err := writeFile(file, bf.Bytes(), log, isTest)
				if err != nil {
					return err
				}
				file = ""
				bf.Reset()
			}
			continue
		}

		bf.WriteString(s + "\n")
		// fmt.Println(scanner.Text()) // Println will add back the final '\n'
	}
	if err := scanner.Err(); err != nil {
		return err
	}
	return nil
}

// SplitTemplate splits the given template on the first empty line.
// It returns the head and body of the template.
// Templates must be UTF8 without byte order marker and have \n (linefeed) as line terminator.
func SplitTemplate(template string) (head, body string) {
	spl := strings.SplitN(template, "\n\n", 2)
	return spl[0], spl[1]
}

// convertJSON converts json to a map
func convertJSON(rd io.Reader) (data map[string]interface{}, err error) {
	data = map[string]interface{}{}
	err = json.NewDecoder(rd).Decode(&data)
	return
}

// mix mixes the given data to the template body
func mix(body string, data map[string]interface{}) (rd io.Reader, err error) {
	var bf bytes.Buffer
	t := template.New("x").Funcs(FuncMap)
	t, err = t.Parse(body)
	if err != nil {
		return
	}
	err = t.Execute(&bf, data)
	rd = &bf
	return
}

// Run mixes the properties of the json object to the template body. The result is then used
// to create files and directories beneath baseDir.
// If isTest is true the files and directories are not really created.
// If log is not nil a list of files that will be created is written to log.
func Run(baseDir string, body string, json io.Reader, log io.Writer, isTest bool) error {

	var (
		err          error
		placeholders map[string]interface{}
		generator    io.Reader
	)

steps:
	for jump := 1; err == nil; jump++ {
		switch jump - 1 {
		default:
			break steps
		case 0:
			placeholders, err = convertJSON(json)
		case 1:
			generator, err = mix(body, placeholders)
		case 2:
			err = parseGenerator(baseDir, generator, log, isTest)
		}
	}
	return err
}
<commit_msg>fix check on not existing dir inside writeFile<commit_after>package scaffold

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"text/template"
)

// FuncMap provides functions to the template.
// New functions can be added as needed. The usual restrictions for
// text/template.FuncMap apply (see http://golang.org/pkg/text/template/#FuncMap)
var FuncMap = template.FuncMap{
	"replace":    Replace,
	"camelCase1": CamelCase1,
	"camelCase2": CamelCase2,
	"title":      strings.Title,
	"toLower":    strings.ToLower,
	"toUpper":    strings.ToUpper,
	"trim":       strings.Trim,
}

// CamelCase1 converts a string in snake_case to CamelCase where the first letter of each word is capitalized
func CamelCase1(src string) string {
	s := strings.Split(src, "_")
	for i, _ := range s {
		s[i] = strings.Title(s[i])
	}
	return strings.Join(s, "")
}

// CamelCase2 converts a string in snake_case to camelCase where the first letter of each but the first word is capitalized
func CamelCase2(src string) string {
	s := strings.Split(src, "_")
	for i, _ := range s {
		if i != 0 {
			s[i] = strings.Title(s[i])
		}
	}
	return strings.Join(s, "")
}

// Replace replaces every occurence of old in s by new
func Replace(s, old, new string) string {
	return strings.Replace(s, old, new, -1)
}

// writeFile creates the given file with the given content if isTest is true.
// Needed directories are created on the fly and each file name is written to log
// if log is not nil.
// If isTest is true, no files and directories are created.
func writeFile(file string, content []byte, log io.Writer, isTest bool) error {
	dir := filepath.Dir(file)

	if s, err := os.Stat(dir); err != nil || !s.IsDir() {
		if err != nil {
			if os.IsNotExist(err) {
				if !isTest {
					err = os.MkdirAll(dir, 0770)
				} else {
					err = nil
				}
			}
			if err != nil {
				return err
			}
		} else {
			return fmt.Errorf("not a directory: %#v", dir)
		}
	}

	if log != nil {
		log.Write([]byte(file + "\n"))
	}

	if !isTest {
		return ioutil.WriteFile(file, content, 0664)
	}
	return nil
}

// parseGenerator creates files and directories beneath baseDir as defined in the reader.
// The file names are written to log if it is not nil.
// If isTest is true, no files and directories are created.
func parseGenerator(baseDir string, rd io.Reader, log io.Writer, isTest bool) error {
	scanner := bufio.NewScanner(rd)
	var file string
	var dir = baseDir
	var bf bytes.Buffer
	var line = -1
	for scanner.Scan() {
		line++
		s := scanner.Text()
		if strings.HasPrefix(s, ">>>") {
			fd := strings.TrimSpace(strings.TrimPrefix(s, ">>>"))
			if fd[len(fd)-1] == '/' {
				dir = filepath.Join(dir, fd)
				file = ""
			} else {
				if file != "" {
					return fmt.Errorf("syntax error in line %d embedding file within file is not allowed (%#v inside %#v)", line, fd, file)
				}
				file = filepath.Join(dir, fd)
			}
			continue
		}

		if strings.HasPrefix(s, "<<<") {
			fd := strings.TrimSpace(strings.TrimPrefix(s, "<<<"))
			if fd[len(fd)-1] == '/' {
				dirName := filepath.Base(dir) + "/"
				if dirName != fd {
					return fmt.Errorf("syntax error in line %d closing dir %#v but should close dir %#v", line, fd, dirName)
				}
				dir = filepath.Dir(dir)
			} else {
				base := filepath.Base(file)
				if base != fd {
					return fmt.Errorf("syntax error in line %d closing file %#v but should close file %#v", line, fd, base)
				}
				err := writeFile(file, bf.Bytes(), log, isTest)
				if err != nil {
					return err
				}
				file = ""
				bf.Reset()
			}
			continue
		}

		bf.WriteString(s + "\n")
		// fmt.Println(scanner.Text()) // Println will add back the final '\n'
	}
	if err := scanner.Err(); err != nil {
		return err
	}
	return nil
}

// SplitTemplate splits the given template on the first empty line.
// It returns the head and body of the template.
// Templates must be UTF8 without byte order marker and have \n (linefeed) as line terminator.
func SplitTemplate(template string) (head, body string) {
	spl := strings.SplitN(template, "\n\n", 2)
	return spl[0], spl[1]
}

// convertJSON converts json to a map
func convertJSON(rd io.Reader) (data map[string]interface{}, err error) {
	data = map[string]interface{}{}
	err = json.NewDecoder(rd).Decode(&data)
	return
}

// mix mixes the given data to the template body
func mix(body string, data map[string]interface{}) (rd io.Reader, err error) {
	var bf bytes.Buffer
	t := template.New("x").Funcs(FuncMap)
	t, err = t.Parse(body)
	if err != nil {
		return
	}
	err = t.Execute(&bf, data)
	rd = &bf
	return
}

// Run mixes the properties of the json object to the template body. The result is then used
// to create files and directories beneath baseDir.
// If isTest is true the files and directories are not really created.
// If log is not nil a list of files that will be created is written to log.
func Run(baseDir string, body string, json io.Reader, log io.Writer, isTest bool) error {

	var (
		err          error
		placeholders map[string]interface{}
		generator    io.Reader
	)

steps:
	for jump := 1; err == nil; jump++ {
		switch jump - 1 {
		default:
			break steps
		case 0:
			placeholders, err = convertJSON(json)
		case 1:
			generator, err = mix(body, placeholders)
		case 2:
			err = parseGenerator(baseDir, generator, log, isTest)
		}
	}
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"strconv"
)

type Fraction struct {
	numerator   int
	denominator int
}

func NewFrac(num, denom int) Fraction {
	return Fraction{
		numerator:   num,
		denominator: denom,
	}
}

func NewIFrac(integer int) Fraction {
	return Fraction{
		numerator:   integer,
		denominator: 1,
	}
}

// This method operates on a struct instance and modifies it attempting to
// put it in to a simpler/reduced form
func (self *Fraction) simplify() {
	// TODO: Use Euclidean Algoritim for finding GCD to divide num and denom
	if self.denominator == 0 {
		return
	}

	if self.numerator%self.denominator == 0 {
		self.numerator /= self.denominator
		self.denominator = 1
	}
	greatest_divisor := gcd(self.numerator, self.denominator)
	if greatest_divisor > 1 {
		self.numerator /= greatest_divisor
		self.denominator /= greatest_divisor
	}
}

func gcd(a, b int) int {
	for b != 0 {
		t := b
		b = a % b
		a = t
	}
	return a
}

// Adding two fractions together
func (self Fraction) add(other Fraction) Fraction {
	// TODO: could potentiallly use a math.Min(denominators) and % operator
	// to only need to multiply one fraction before adding (maybe faster?)
	new_self := self.mulInt(other.denominator)
	new_other := other.mulInt(self.denominator)

	return Fraction{
		numerator:   new_self.numerator + new_other.numerator,
		denominator: new_self.denominator,
	}
}

// Adding an integer to a fraction
func (self Fraction) addInt(integer int) Fraction {
	return Fraction{
		numerator:   self.numerator + self.denominator*integer,
		denominator: self.denominator,
	}
}

// Multiplication of fraction objects
func (self Fraction) mul(other Fraction) Fraction {
	return Fraction{
		numerator:   self.numerator * other.numerator,
		denominator: self.denominator * other.denominator,
	}
}

// In-place multiply (similar to *=)
func (self *Fraction) imul(other Fraction) {
	self.numerator *= other.numerator
	self.denominator *= other.denominator
}

// Multiply a fraction by an integer
func (self Fraction) mulInt(integer int) Fraction {
	return Fraction{
		numerator:   self.numerator * integer,
		denominator: self.denominator * integer,
	}
}

// Division of fractions
func (self Fraction) div(other Fraction) Fraction {
	return Fraction{
		numerator:   self.numerator * other.denominator,
		denominator: self.denominator * other.numerator,
	}
}

func (self Fraction) divInt(integer int) Fraction {
	return Fraction{
		numerator:   self.numerator,
		denominator: self.denominator * integer,
	}
}

func (self Fraction) inv() Fraction {
	return Fraction{
		numerator:   self.denominator,
		denominator: self.numerator,
	}
}

// The folowing functions are for getting the value of the fraction in
// decimal or rounded to an integer
func (self Fraction) intVal() int {
	return self.numerator / self.denominator
}

func (self Fraction) floatVal() float32 {
	return float32(self.numerator) / float32(self.denominator)
}

// To implement the Stringy interface so it can be printed out
func (self Fraction) String() string {
	return strconv.FormatInt(int64(self.numerator), 10) + "/" +
		strconv.FormatInt(int64(self.denominator), 10)
}
<commit_msg>Add inplace fraction add; Fix mulInt<commit_after>package main

import (
	"strconv"
)

type Fraction struct {
	numerator   int
	denominator int
}

func NewFrac(num, denom int) Fraction {
	return Fraction{
		numerator:   num,
		denominator: denom,
	}
}

func NewIFrac(integer int) Fraction {
	return Fraction{
		numerator:   integer,
		denominator: 1,
	}
}

// This method operates on a struct instance and modifies it attempting to
// put it in to a simpler/reduced form
func (self *Fraction) simplify() {
	// TODO: Use Euclidean Algoritim for finding GCD to divide num and denom
	if self.denominator == 0 {
		return
	}

	if self.numerator%self.denominator == 0 {
		self.numerator /= self.denominator
		self.denominator = 1
	}
	greatest_divisor := gcd(self.numerator, self.denominator)
	if greatest_divisor > 1 {
		self.numerator /= greatest_divisor
		self.denominator /= greatest_divisor
	}
}

func gcd(a, b int) int {
	for b != 0 {
		t := b
		b = a % b
		a = t
	}
	return a
}

// Adding two fractions together
func (self Fraction) add(other Fraction) Fraction {
	// TODO: could potentiallly use a math.Min(denominators) and % operator
	// to only need to multiply one fraction before adding (maybe faster?)
	new_self := self.mulInt(other.denominator)
	new_other := other.mulInt(self.denominator)

	return Fraction{
		numerator:   new_self.numerator + new_other.numerator,
		denominator: new_self.denominator,
	}
}

func (self *Fraction) iadd(other Fraction) {
	// TODO: could potentiallly use a math.Min(denominators) and % operator
	// to only need to multiply one fraction before adding (maybe faster?)
	new_self := self.mulInt(other.denominator)
	new_other := other.mulInt(self.denominator)

	self.numerator = new_self.numerator + new_other.numerator
	self.denominator = new_self.denominator
}

// Adding an integer to a fraction
func (self Fraction) addInt(integer int) Fraction {
	return Fraction{
		numerator:   self.numerator + self.denominator*integer,
		denominator: self.denominator,
	}
}

// Multiplication of fraction objects
func (self Fraction) mul(other Fraction) Fraction {
	return Fraction{
		numerator:   self.numerator * other.numerator,
		denominator: self.denominator * other.denominator,
	}
}

// In-place multiply (similar to *=)
func (self *Fraction) imul(other Fraction) {
	self.numerator *= other.numerator
	self.denominator *= other.denominator
}

// Multiply a fraction by an integer
func (self Fraction) mulInt(integer int) Fraction {
	return Fraction{
		numerator:   self.numerator * integer,
		denominator: self.denominator,
	}
}

// Division of fractions
func (self Fraction) div(other Fraction) Fraction {
	return Fraction{
		numerator:   self.numerator * other.denominator,
		denominator: self.denominator * other.numerator,
	}
}

func (self Fraction) divInt(integer int) Fraction {
	return Fraction{
		numerator:   self.numerator,
		denominator: self.denominator * integer,
	}
}

func (self Fraction) inv() Fraction {
	return Fraction{
		numerator:   self.denominator,
		denominator: self.numerator,
	}
}

// The folowing functions are for getting the value of the fraction in
// decimal or rounded to an integer
func (self Fraction) intVal() int {
	return self.numerator / self.denominator
}

func (self Fraction) floatVal() float32 {
	return float32(self.numerator) / float32(self.denominator)
}

// To implement the Stringy interface so it can be printed out
func (self Fraction) String() string {
	return strconv.FormatInt(int64(self.numerator), 10) + "/" +
		strconv.FormatInt(int64(self.denominator), 10)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"html"
	"html/template"
	"io"
	"log"
	"net/http"
	"strings"
	"time"
)

// WebServerConfigs holds configs for this server
type WebServerConfigs struct {
	Host     string
	Port     int
	BasePath string
	// webServer will use this FileFetcher to serve requests.
	fetcher FileFetcher
}

// RunWebServer starts a proxy pypi server that handles HTTP requests by leveraging any implementation of FileFetcher to do the grunt work.
func RunWebServer(cfg WebServerConfigs, fileFetcher FileFetcher) {
	cfg.fetcher = fileFetcher

	http.HandleFunc(cfg.BasePath, cfg.handleRequest)
	hostPortStr := fmt.Sprintf("%v:%v", cfg.Host, cfg.Port)
	log.Printf("serving on HTTP://%v/%v\n", hostPortStr, cfg.BasePath)
	log.Fatal(http.ListenAndServe(hostPortStr, nil))
}

func (ctx WebServerConfigs) handleRequest(w http.ResponseWriter, req *http.Request) {

	path := html.EscapeString(req.URL.Path)
	if strings.HasSuffix(path, "/") {
		ctx.handleListDir(w, path)
	} else {
		// must be file, lets find it.
		ctx.handleServeS3File(w, path)
	}
}

func (ctx WebServerConfigs) handleServeS3File(w http.ResponseWriter, path string) {
	start := time.Now()

	fileName := handlePypiFileNames(path)
	file, err := ctx.fetcher.GetFile(fileName)
	if err != nil {
		desc := fmt.Sprintf("\"GET %v\" 404 - time: %.3fs, error: %v",
			fileName, time.Since(start).Seconds(), err)
		log.Println(desc)
		http.Error(w, desc, http.StatusNotFound)
		return
	}
	timeFetching := time.Since(start)

	defer file.Payload.Close()
	nbytes, err := io.Copy(w, file.Payload)
	if err != nil {
		desc := fmt.Sprintf("\"GET %v\" 404 - time: %.3fs, network write error: %v",
			fileName, time.Since(start).Seconds(), err)
		log.Println(desc)
		http.Error(w, desc, http.StatusNotFound)
		return
	}
	elapsed := time.Since(start)
	log.Printf("\"GET %v\" 200 - size: %v, fetch: %.3fs, total: %.3fs",
		path, nbytes, timeFetching.Seconds(), elapsed.Seconds())
	return
}

func (ctx WebServerConfigs) handleListDir(w http.ResponseWriter, path string) {
	start := time.Now()

	fileList, err := handlePypiListDir(ctx.fetcher, path)
	if err != nil {
		desc := fmt.Sprintf("\"LISTDIR %v\" 400 - err: %v", path, err)
		http.Error(w, desc, http.StatusBadRequest)
		return
	}

	type templateData struct {
		Results []ListDirEntry
		Elapsed time.Duration
		BaseDir string
	}

	if err := pypiResultsTemplate.Execute(w, templateData{
		Results: fileList,
		Elapsed: time.Since(start),
		BaseDir: ctx.BasePath,
	}); err != nil {
		desc := fmt.Sprintf("\"LISTDIR %v\" 404 - time: %.3fs, template write error: %v",
			path, time.Since(start).Seconds(), err)
		log.Print(desc)
		http.Error(w, desc, http.StatusInternalServerError)
	}
	log.Printf("\"LISTDIR %v\" 200 - items:%v, time: %.3fs",
		path, len(fileList), time.Since(start).Seconds())
}

// results just like a pypi server would return
var pypiResultsTemplate = template.Must(template.New("results").Parse(`

  {{range .Results}}
  {{.Name}}
  {{end}}

`))

var resultsTemplate = template.Must(template.New("results").Parse(`

  {{range .Results}}

{{.LastModified}} - {{.SizeKb}} Kb - {{.Name}}

  {{end}}

{{len .Results}} results in {{.Elapsed}}

`))
<commit_msg>Log if LISTDIR operation fails<commit_after>package main

import (
	"fmt"
	"html"
	"html/template"
	"io"
	"log"
	"net/http"
	"strings"
	"time"
)

// WebServerConfigs holds configs for this server
type WebServerConfigs struct {
	Host     string
	Port     int
	BasePath string
	// webServer will use this FileFetcher to serve requests.
	fetcher FileFetcher
}

// RunWebServer starts a proxy pypi server that handles HTTP requests by leveraging any implementation of FileFetcher to do the grunt work.
func RunWebServer(cfg WebServerConfigs, fileFetcher FileFetcher) {
	cfg.fetcher = fileFetcher

	http.HandleFunc(cfg.BasePath, cfg.handleRequest)
	hostPortStr := fmt.Sprintf("%v:%v", cfg.Host, cfg.Port)
	log.Printf("serving on HTTP://%v/%v\n", hostPortStr, cfg.BasePath)
	log.Fatal(http.ListenAndServe(hostPortStr, nil))
}

func (ctx WebServerConfigs) handleRequest(w http.ResponseWriter, req *http.Request) {

	path := html.EscapeString(req.URL.Path)
	if strings.HasSuffix(path, "/") {
		ctx.handleListDir(w, path)
	} else {
		// must be file, lets find it.
		ctx.handleServeS3File(w, path)
	}
}

func (ctx WebServerConfigs) handleServeS3File(w http.ResponseWriter, path string) {
	start := time.Now()

	fileName := handlePypiFileNames(path)
	file, err := ctx.fetcher.GetFile(fileName)
	if err != nil {
		desc := fmt.Sprintf("\"GET %v\" 404 - time: %.3fs, error: %v",
			fileName, time.Since(start).Seconds(), err)
		log.Println(desc)
		http.Error(w, desc, http.StatusNotFound)
		return
	}
	timeFetching := time.Since(start)

	defer file.Payload.Close()
	nbytes, err := io.Copy(w, file.Payload)
	if err != nil {
		desc := fmt.Sprintf("\"GET %v\" 404 - time: %.3fs, network write error: %v",
			fileName, time.Since(start).Seconds(), err)
		log.Println(desc)
		http.Error(w, desc, http.StatusNotFound)
		return
	}
	elapsed := time.Since(start)
	log.Printf("\"GET %v\" 200 - size: %v, fetch: %.3fs, total: %.3fs",
		path, nbytes, timeFetching.Seconds(), elapsed.Seconds())
	return
}

func (ctx WebServerConfigs) handleListDir(w http.ResponseWriter, path string) {
	start := time.Now()

	fileList, err := handlePypiListDir(ctx.fetcher, path)
	if err != nil {
		desc := fmt.Sprintf("\"LISTDIR %v\" 400 - err: %v", path, err)
		log.Println(desc)
		http.Error(w, desc, http.StatusBadRequest)
		return
	}

	type templateData struct {
		Results []ListDirEntry
		Elapsed time.Duration
		BaseDir string
	}

	if err := pypiResultsTemplate.Execute(w, templateData{
		Results: fileList,
		Elapsed: time.Since(start),
		BaseDir: ctx.BasePath,
	}); err != nil {
		desc := fmt.Sprintf("\"LISTDIR %v\" 404 - time: %.3fs, template write error: %v",
			path, time.Since(start).Seconds(), err)
		log.Print(desc)
		http.Error(w, desc, http.StatusInternalServerError)
	}
	log.Printf("\"LISTDIR %v\" 200 - items:%v, time: %.3fs",
		path, len(fileList), time.Since(start).Seconds())
}

// results just like a pypi server would return
var pypiResultsTemplate = template.Must(template.New("results").Parse(`

  {{range .Results}}
  {{.Name}}
  {{end}}

`))

var resultsTemplate = template.Must(template.New("results").Parse(`

  {{range .Results}}

{{.LastModified}} - {{.SizeKb}} Kb - {{.Name}}

  {{end}}

{{len .Results}} results in {{.Elapsed}}

`))
<|endoftext|>