<commit_before>package main

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"

	"go.skia.org/infra/fiddlek/go/client"
	"go.skia.org/infra/fiddlek/go/store"
	"go.skia.org/infra/fiddlek/go/types"
	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/git/gitinfo"
	"go.skia.org/infra/go/gitauth"
	"go.skia.org/infra/go/metrics2"
	"go.skia.org/infra/go/skerr"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/util"
	"go.skia.org/infra/named-fiddles/go/parse"
)

// Server is the state of the server.
type Server struct {
	store   store.Store
	repo    *gitinfo.GitInfo
	repoDir string

	livenessExamples    metrics2.Liveness    // liveness of the naming the Skia examples.
	errorsInExamplesRun metrics2.Counter     // errorsInExamplesRun is the number of errors in a single examples run.
	numInvalidExamples  metrics2.Int64Metric // numInvalidExamples is the number of examples that are currently invalid.
}

func main() {
	local := flag.Bool("local", false, "Running locally if true. As opposed to in production.")
	promPort := flag.String("prom_port", ":20000", "Metrics service address (e.g., ':10110')")
	repoURL := flag.String("repo_url", "https://skia.googlesource.com/skia", "Repo url")
	repoDir := flag.String("repo_dir", "/tmp/skia_named_fiddles", "Directory the repo is checked out into.")

	common.InitWithMust(
		"named-fiddles",
		common.PrometheusOpt(promPort),
	)

	_, err := startSyncing(context.Background(), *local, *repoURL, *repoDir)
	if err != nil {
		sklog.Fatalf("Failed to create Server: %s", err)
	}
	select {}
}

// startSyncing creates a new Server in a goroutine and returns.
func startSyncing(ctx context.Context, local bool, repoURL, repoDir string) (*Server, error) {
	st, err := store.New(local)
	if err != nil {
		return nil, skerr.Wrapf(err, "creating fiddle store")
	}

	if !local {
		ts, err := auth.NewDefaultTokenSource(false, auth.ScopeUserinfoEmail, auth.ScopeGerrit)
		if err != nil {
			sklog.Fatalf("Failed authentication: %s", err)
		}
		// Use the gitcookie created by the gitauth package.
		if _, err := gitauth.New(ts, "/tmp/gitcookies", true, ""); err != nil {
			sklog.Fatalf("Failed to create git cookie updater: %s", err)
		}
		sklog.Infof("Git authentication set up successfully.")
	}

	repo, err := gitinfo.CloneOrUpdate(ctx, repoURL, repoDir, false)
	if err != nil {
		return nil, skerr.Wrapf(err, "Cloning git repo %s to %s", repoURL, repoDir)
	}
	sklog.Infof("git clone of %s to %s was successful", repoURL, repoDir)
	srv := &Server{
		store:   st,
		repo:    repo,
		repoDir: repoDir,

		livenessExamples:    metrics2.NewLiveness("named_fiddles_examples"),
		errorsInExamplesRun: metrics2.GetCounter("named_fiddles_errors_in_examples_run"),
		numInvalidExamples:  metrics2.GetInt64Metric("named_fiddles_examples_total_invalid"),
	}
	go util.RepeatCtx(ctx, time.Minute, srv.exampleStep)
	return srv, nil
}

// errorsInResults returns an empty string if there are no errors, either
// compile or runtime, found in the results. If there are errors then a string
// describing the error is returned.
func errorsInResults(runResults *types.RunResults) string {
	status := ""
	if runResults == nil {
		status = "Failed to run."
	} else if len(runResults.CompileErrors) > 0 || runResults.RunTimeError != "" {
		// update validity
		status = fmt.Sprintf("%v %s", runResults.CompileErrors, runResults.RunTimeError)
		if len(status) > 100 {
			status = status[:100]
		}
	}
	return status
}

// exampleStep is a single run through naming all the examples.
func (srv *Server) exampleStep(ctx context.Context) {
	srv.errorsInExamplesRun.Reset()
	sklog.Info("Starting exampleStep")
	if err := srv.repo.Update(ctx, true, false); err != nil {
		sklog.Errorf("Failed to sync git repo.")
		return
	}

	var numInvalid int64
	// Get a list of all examples.
	dir := filepath.Join(srv.repoDir, "docs", "examples")
	err := filepath.Walk(dir+"/", func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return skerr.Wrapf(err, "opening %q", path)
		}
		if info.IsDir() {
			return nil
		}
		name := filepath.Base(info.Name())
		if !strings.HasSuffix(name, ".cpp") {
			return nil
		}
		name = name[0 : len(name)-4]
		b, err := ioutil.ReadFile(filepath.Join(dir, info.Name()))
		fc, err := parse.ParseCpp(string(b))
		if err == parse.ErrorInactiveExample {
			sklog.Infof("Inactive sample: %q", info.Name())
			return nil
		} else if err != nil {
			sklog.Infof("Invalid sample: %q\n%s", info.Name(), err)
			numInvalid += 1
			srv.numInvalidExamples.Update(numInvalid)
			return nil
		}
		// Now run it.
		sklog.Infof("About to run: %s", name)
		b, err = json.Marshal(fc)
		if err != nil {
			sklog.Errorf("Failed to encode example to JSON: %s", err)
			return nil
		}

		runResults, success := client.Do(b, false, "https://fiddle.skia.org", func(*types.RunResults) bool {
			return true
		})
		status := errorsInResults(runResults)
		if !success {
			sklog.Errorf("Failed to run: %s", status)
			srv.errorsInExamplesRun.Inc(1)
			return nil
		}
		if status != "" {
			sklog.Warningf("Sample %s had a non-empty status: %s", name, status)
		}
		if err := srv.store.WriteName(name, runResults.FiddleHash, "Skia example", status); err != nil {
			sklog.Errorf("Failed to write status for %s: %s", name, err)
			srv.errorsInExamplesRun.Inc(1)
		}
		return nil
	})
	if err != nil {
		sklog.Errorf("Error walking the path %q: %v\n", dir, err)
		return
	}

	srv.numInvalidExamples.Update(numInvalid)
	srv.livenessExamples.Reset()
}
<commit_msg>[named-fiddles] Don't report timeouts as failures.<commit_after>package main

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"time"

	"go.skia.org/infra/fiddlek/go/client"
	"go.skia.org/infra/fiddlek/go/store"
	"go.skia.org/infra/fiddlek/go/types"
	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/git/gitinfo"
	"go.skia.org/infra/go/gitauth"
	"go.skia.org/infra/go/metrics2"
	"go.skia.org/infra/go/skerr"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/util"
	"go.skia.org/infra/named-fiddles/go/parse"
)

// Server is the state of the server.
type Server struct {
	store   store.Store
	repo    *gitinfo.GitInfo
	repoDir string

	livenessExamples    metrics2.Liveness    // liveness of the naming the Skia examples.
	errorsInExamplesRun metrics2.Counter     // errorsInExamplesRun is the number of errors in a single examples run.
	numInvalidExamples  metrics2.Int64Metric // numInvalidExamples is the number of examples that are currently invalid.
}

func main() {
	local := flag.Bool("local", false, "Running locally if true. As opposed to in production.")
	promPort := flag.String("prom_port", ":20000", "Metrics service address (e.g., ':10110')")
	repoURL := flag.String("repo_url", "https://skia.googlesource.com/skia", "Repo url")
	repoDir := flag.String("repo_dir", "/tmp/skia_named_fiddles", "Directory the repo is checked out into.")

	common.InitWithMust(
		"named-fiddles",
		common.PrometheusOpt(promPort),
	)

	_, err := startSyncing(context.Background(), *local, *repoURL, *repoDir)
	if err != nil {
		sklog.Fatalf("Failed to create Server: %s", err)
	}
	select {}
}

// startSyncing creates a new Server in a goroutine and returns.
func startSyncing(ctx context.Context, local bool, repoURL, repoDir string) (*Server, error) {
	st, err := store.New(local)
	if err != nil {
		return nil, skerr.Wrapf(err, "creating fiddle store")
	}

	if !local {
		ts, err := auth.NewDefaultTokenSource(false, auth.ScopeUserinfoEmail, auth.ScopeGerrit)
		if err != nil {
			sklog.Fatalf("Failed authentication: %s", err)
		}
		// Use the gitcookie created by the gitauth package.
		if _, err := gitauth.New(ts, "/tmp/gitcookies", true, ""); err != nil {
			sklog.Fatalf("Failed to create git cookie updater: %s", err)
		}
		sklog.Infof("Git authentication set up successfully.")
	}

	repo, err := gitinfo.CloneOrUpdate(ctx, repoURL, repoDir, false)
	if err != nil {
		return nil, skerr.Wrapf(err, "Cloning git repo %s to %s", repoURL, repoDir)
	}
	sklog.Infof("git clone of %s to %s was successful", repoURL, repoDir)
	srv := &Server{
		store:   st,
		repo:    repo,
		repoDir: repoDir,

		livenessExamples:    metrics2.NewLiveness("named_fiddles_examples"),
		errorsInExamplesRun: metrics2.GetCounter("named_fiddles_errors_in_examples_run"),
		numInvalidExamples:  metrics2.GetInt64Metric("named_fiddles_examples_total_invalid"),
	}
	go util.RepeatCtx(ctx, time.Minute, srv.exampleStep)
	return srv, nil
}

// errorsInResults returns an empty string if there are no errors, either
// compile or runtime, found in the results. If there are errors then a string
// describing the error is returned.
func errorsInResults(runResults *types.RunResults) string {
	status := ""
	if runResults == nil {
		sklog.Infof("runResults are nil, so this was a timeout, count it as valid for now and try again on the next run.")
	} else if len(runResults.CompileErrors) > 0 || runResults.RunTimeError != "" {
		// update validity
		status = fmt.Sprintf("%v %s", runResults.CompileErrors, runResults.RunTimeError)
		if len(status) > 100 {
			status = status[:100]
		}
	}
	return status
}

// exampleStep is a single run through naming all the examples.
func (srv *Server) exampleStep(ctx context.Context) {
	srv.errorsInExamplesRun.Reset()
	sklog.Info("Starting exampleStep")
	if err := srv.repo.Update(ctx, true, false); err != nil {
		sklog.Errorf("Failed to sync git repo.")
		return
	}

	var numInvalid int64
	// Get a list of all examples.
	dir := filepath.Join(srv.repoDir, "docs", "examples")
	err := filepath.Walk(dir+"/", func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return skerr.Wrapf(err, "opening %q", path)
		}
		if info.IsDir() {
			return nil
		}
		name := filepath.Base(info.Name())
		if !strings.HasSuffix(name, ".cpp") {
			return nil
		}
		name = name[0 : len(name)-4]
		filename := filepath.Join(dir, info.Name())
		b, err := ioutil.ReadFile(filename)
		if err != nil {
			sklog.Warningf("Failed to load file: %q", filename)
		}
		fc, err := parse.ParseCpp(string(b))
		if err == parse.ErrorInactiveExample {
			sklog.Infof("Inactive sample: %q", info.Name())
			return nil
		} else if err != nil {
			sklog.Infof("Invalid sample: %q\n%s", info.Name(), err)
			numInvalid++
			srv.numInvalidExamples.Update(numInvalid)
			return nil
		}
		// Now run it.
		sklog.Infof("About to run: %s", name)
		b, err = json.Marshal(fc)
		if err != nil {
			sklog.Errorf("Failed to encode example to JSON: %s", err)
			return nil
		}

		runResults, success := client.Do(b, false, "https://fiddle.skia.org", func(*types.RunResults) bool {
			return true
		})
		status := errorsInResults(runResults)
		if !success {
			sklog.Errorf("Failed to run: %s", status)
			srv.errorsInExamplesRun.Inc(1)
			return nil
		}
		if status != "" {
			sklog.Warningf("Sample %s had a non-empty status: %s", name, status)
		}
		if err := srv.store.WriteName(name, runResults.FiddleHash, "Skia example", status); err != nil {
			sklog.Errorf("Failed to write status for %s: %s", name, err)
			srv.errorsInExamplesRun.Inc(1)
		}
		return nil
	})
	if err != nil {
		sklog.Errorf("Error walking the path %q: %v\n", dir, err)
		return
	}

	srv.numInvalidExamples.Update(numInvalid)
	srv.livenessExamples.Reset()
}
<|endoftext|>


<commit_before>// Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.

package oci

import (
	"context"
	"fmt"
	"log"
	"strings"

	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/helper/validation"

	oci_core "github.com/oracle/oci-go-sdk/core"
)

func CoreVolumeAttachmentResource() *schema.Resource {
	return &schema.Resource{
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},
		Timeouts: DefaultTimeout,
		Create:   createCoreVolumeAttachment,
		Read:     readCoreVolumeAttachment,
		Delete:   deleteCoreVolumeAttachment,
		Schema: map[string]*schema.Schema{
			// Required
			"attachment_type": {
				Type:             schema.TypeString,
				Required:         true,
				ForceNew:         true,
				DiffSuppressFunc: EqualIgnoreCaseSuppressDiff,
				ValidateFunc: validation.StringInSlice([]string{
					"iscsi",
					"paravirtualized",
				}, true),
			},
			"instance_id": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"volume_id": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			// Optional
			"device": {
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
				ForceNew: true,
			},
			"display_name": {
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
				ForceNew: true,
			},
			"is_pv_encryption_in_transit_enabled": {
				Type:     schema.TypeBool,
				Optional: true,
				Computed: true,
				ForceNew: true,
			},
			"is_read_only": {
				Type:     schema.TypeBool,
				Optional: true,
				Computed: true,
				ForceNew: true,
			},
			"use_chap": {
				Type:     schema.TypeBool,
				Optional: true,
				Computed: true,
				ForceNew: true,
			},

			// Computed
			"availability_domain": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"chap_secret": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"chap_username": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"compartment_id": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"ipv4": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"iqn": { // iSCSI Qualified Name per RFC 3720
				Type:     schema.TypeString,
				Computed: true,
			},
			"port": {
				Type:     schema.TypeInt,
				Computed: true,
			},
			"state": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"time_created": {
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func createCoreVolumeAttachment(d *schema.ResourceData, m interface{}) error {
	sync := &CoreVolumeAttachmentResourceCrud{}
	sync.D = d
	sync.Client = m.(*OracleClients).computeClient

	return CreateResource(d, sync)
}

func readCoreVolumeAttachment(d *schema.ResourceData, m interface{}) error {
	sync := &CoreVolumeAttachmentResourceCrud{}
	sync.D = d
	sync.Client = m.(*OracleClients).computeClient

	return ReadResource(sync)
}

func deleteCoreVolumeAttachment(d *schema.ResourceData, m interface{}) error {
	sync := &CoreVolumeAttachmentResourceCrud{}
	sync.D = d
	sync.Client = m.(*OracleClients).computeClient
	sync.DisableNotFoundRetries = true

	return DeleteResource(d, sync)
}

type CoreVolumeAttachmentResourceCrud struct {
	BaseCrud
	Client                 *oci_core.ComputeClient
	Res                    *oci_core.VolumeAttachment
	DisableNotFoundRetries bool
}

func (s *CoreVolumeAttachmentResourceCrud) ID() string {
	volumeAttachment := *s.Res
	return *volumeAttachment.GetId()
}

func (s *CoreVolumeAttachmentResourceCrud) CreatedPending() []string {
	return []string{
		string(oci_core.VolumeAttachmentLifecycleStateAttaching),
	}
}

func (s *CoreVolumeAttachmentResourceCrud) CreatedTarget() []string {
	return []string{
		string(oci_core.VolumeAttachmentLifecycleStateAttached),
	}
}

func (s *CoreVolumeAttachmentResourceCrud) DeletedPending() []string {
	return []string{
		string(oci_core.VolumeAttachmentLifecycleStateDetaching),
	}
}

func (s *CoreVolumeAttachmentResourceCrud) DeletedTarget() []string {
	return []string{
		string(oci_core.VolumeAttachmentLifecycleStateDetached),
	}
}

func (s *CoreVolumeAttachmentResourceCrud) Create() error {
	request := oci_core.AttachVolumeRequest{}
	err := s.populateTopLevelPolymorphicAttachVolumeRequest(&request)
	if err != nil {
		return err
	}

	request.RequestMetadata.RetryPolicy = getRetryPolicy(s.DisableNotFoundRetries, "core")

	response, err := s.Client.AttachVolume(context.Background(), request)
	if err != nil {
		return err
	}

	s.Res = &response.VolumeAttachment
	return nil
}

func (s *CoreVolumeAttachmentResourceCrud) Get() error {
	request := oci_core.GetVolumeAttachmentRequest{}

	tmp := s.D.Id()
	request.VolumeAttachmentId = &tmp

	request.RequestMetadata.RetryPolicy = getRetryPolicy(s.DisableNotFoundRetries, "core")

	response, err := s.Client.GetVolumeAttachment(context.Background(), request)
	if err != nil {
		return err
	}

	s.Res = &response.VolumeAttachment
	return nil
}

func (s *CoreVolumeAttachmentResourceCrud) Delete() error {
	request := oci_core.DetachVolumeRequest{}

	tmp := s.D.Id()
	request.VolumeAttachmentId = &tmp

	request.RequestMetadata.RetryPolicy = getRetryPolicy(s.DisableNotFoundRetries, "core")

	_, err := s.Client.DetachVolume(context.Background(), request)
	return err
}

func (s *CoreVolumeAttachmentResourceCrud) SetData() error {
	switch v := (*s.Res).(type) {
	case oci_core.IScsiVolumeAttachment:
		s.D.Set("attachment_type", "iscsi")

		if v.ChapSecret != nil {
			s.D.Set("chap_secret", *v.ChapSecret)
		}

		if v.ChapUsername != nil {
			s.D.Set("chap_username", *v.ChapUsername)
		}

		if v.Ipv4 != nil {
			s.D.Set("ipv4", *v.Ipv4)
		}

		if v.Iqn != nil {
			s.D.Set("iqn", *v.Iqn)
		}

		if v.Port != nil {
			s.D.Set("port", *v.Port)
		}

		if v.AvailabilityDomain != nil {
			s.D.Set("availability_domain", *v.AvailabilityDomain)
		}

		if v.CompartmentId != nil {
			s.D.Set("compartment_id", *v.CompartmentId)
		}

		if v.Device != nil {
			s.D.Set("device", *v.Device)
		}

		if v.DisplayName != nil {
			s.D.Set("display_name", *v.DisplayName)
		}

		if v.Id != nil {
			s.D.Set("id", *v.Id)
		}

		if v.InstanceId != nil {
			s.D.Set("instance_id", *v.InstanceId)
		}

		if v.IsPvEncryptionInTransitEnabled != nil {
			s.D.Set("is_pv_encryption_in_transit_enabled", *v.IsPvEncryptionInTransitEnabled)
		}

		if v.IsReadOnly != nil {
			s.D.Set("is_read_only", *v.IsReadOnly)
		}

		s.D.Set("state", v.LifecycleState)

		if v.TimeCreated != nil {
			s.D.Set("time_created", v.TimeCreated.String())
		}

		if v.VolumeId != nil {
			s.D.Set("volume_id", *v.VolumeId)
		}
	case oci_core.ParavirtualizedVolumeAttachment:
		s.D.Set("attachment_type", "paravirtualized")

		if v.AvailabilityDomain != nil {
			s.D.Set("availability_domain", *v.AvailabilityDomain)
		}

		if v.CompartmentId != nil {
			s.D.Set("compartment_id", *v.CompartmentId)
		}

		if v.Device != nil {
			s.D.Set("device", *v.Device)
		}

		if v.DisplayName != nil {
			s.D.Set("display_name", *v.DisplayName)
		}

		if v.Id != nil {
			s.D.Set("id", *v.Id)
		}

		if v.InstanceId != nil {
			s.D.Set("instance_id", *v.InstanceId)
		}

		if v.IsPvEncryptionInTransitEnabled != nil {
			s.D.Set("is_pv_encryption_in_transit_enabled", *v.IsPvEncryptionInTransitEnabled)
		}

		if v.IsReadOnly != nil {
			s.D.Set("is_read_only", *v.IsReadOnly)
		}

		s.D.Set("state", v.LifecycleState)

		if v.TimeCreated != nil {
			s.D.Set("time_created", v.TimeCreated.String())
		}

		if v.VolumeId != nil {
			s.D.Set("volume_id", *v.VolumeId)
		}
	default:
		log.Printf("[WARN] Received 'attachment_type' of unknown type %v", *s.Res)
		return nil
	}
	return nil
}

func (s *CoreVolumeAttachmentResourceCrud) populateTopLevelPolymorphicAttachVolumeRequest(request *oci_core.AttachVolumeRequest) error {
	//discriminator
	attachmentTypeRaw, ok := s.D.GetOkExists("attachment_type")
	var attachmentType string
	if ok {
		attachmentType = attachmentTypeRaw.(string)
	} else {
		attachmentType = "" // default value
	}
	switch strings.ToLower(attachmentType) {
	case strings.ToLower("iscsi"):
		details := oci_core.AttachIScsiVolumeDetails{}
		if useChap, ok := s.D.GetOkExists("use_chap"); ok {
			tmp := useChap.(bool)
			details.UseChap = &tmp
		}
		if device, ok := s.D.GetOkExists("device"); ok {
			tmp := device.(string)
			details.Device = &tmp
		}
		if displayName, ok := s.D.GetOkExists("display_name"); ok {
			tmp := displayName.(string)
			details.DisplayName = &tmp
		}
		if instanceId, ok := s.D.GetOkExists("instance_id"); ok {
			tmp := instanceId.(string)
			details.InstanceId = &tmp
		}
		if isReadOnly, ok := s.D.GetOkExists("is_read_only"); ok {
			tmp := isReadOnly.(bool)
			details.IsReadOnly = &tmp
		}
		if volumeId, ok := s.D.GetOkExists("volume_id"); ok {
			tmp := volumeId.(string)
			details.VolumeId = &tmp
		}
		request.AttachVolumeDetails = details
	case strings.ToLower("paravirtualized"):
		details := oci_core.AttachParavirtualizedVolumeDetails{}
		if device, ok := s.D.GetOkExists("device"); ok {
			tmp := device.(string)
			details.Device = &tmp
		}
		if displayName, ok := s.D.GetOkExists("display_name"); ok {
			tmp := displayName.(string)
			details.DisplayName = &tmp
		}
		if instanceId, ok := s.D.GetOkExists("instance_id"); ok {
			tmp := instanceId.(string)
			details.InstanceId = &tmp
		}
		if isPvEncryptionInTransitEnabled, ok := s.D.GetOkExists("is_pv_encryption_in_transit_enabled"); ok {
			tmp := isPvEncryptionInTransitEnabled.(bool)
			details.IsPvEncryptionInTransitEnabled = &tmp
		}
		if isReadOnly, ok := s.D.GetOkExists("is_read_only"); ok {
			tmp := isReadOnly.(bool)
			details.IsReadOnly = &tmp
		}
		if volumeId, ok := s.D.GetOkExists("volume_id"); ok {
			tmp := volumeId.(string)
			details.VolumeId = &tmp
		}
		request.AttachVolumeDetails = details
	default:
		return fmt.Errorf("unknown attachment_type '%v' was specified", attachmentType)
	}
	return nil
}
<commit_msg>mark compartment_id as optional to avoid validation during destroy for the legacy configurations<commit_after>// Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.

package oci

import (
	"context"
	"fmt"
	"log"
	"strings"

	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/helper/validation"

	oci_core "github.com/oracle/oci-go-sdk/core"
)

func CoreVolumeAttachmentResource() *schema.Resource {
	return &schema.Resource{
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},
		Timeouts: DefaultTimeout,
		Create:   createCoreVolumeAttachment,
		Read:     readCoreVolumeAttachment,
		Delete:   deleteCoreVolumeAttachment,
		Schema: map[string]*schema.Schema{
			// Required
			"attachment_type": {
				Type:             schema.TypeString,
				Required:         true,
				ForceNew:         true,
				DiffSuppressFunc: EqualIgnoreCaseSuppressDiff,
				ValidateFunc: validation.StringInSlice([]string{
					"iscsi",
					"paravirtualized",
				}, true),
			},
			"instance_id": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"volume_id": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			// Optional
			"device": {
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
				ForceNew: true,
			},
			"display_name": {
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
				ForceNew: true,
			},
			"is_pv_encryption_in_transit_enabled": {
				Type:     schema.TypeBool,
				Optional: true,
				Computed: true,
				ForceNew: true,
			},
			"is_read_only": {
				Type:     schema.TypeBool,
				Optional: true,
				Computed: true,
				ForceNew: true,
			},
			"use_chap": {
				Type:     schema.TypeBool,
				Optional: true,
				Computed: true,
				ForceNew: true,
			},

			// Computed
			"availability_domain": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"chap_secret": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"chap_username": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"compartment_id": {
				Type:     schema.TypeString,
				Computed: true,
				// Keep as optional to avoid validation during destroy for the legacy configurations
				Optional:   true,
				Deprecated: FieldDeprecatedAndAvoidReferences("compartment_id"),
			},
			"ipv4": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"iqn": { // iSCSI Qualified Name per RFC 3720
				Type:     schema.TypeString,
				Computed: true,
			},
			"port": {
				Type:     schema.TypeInt,
				Computed: true,
			},
			"state": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"time_created": {
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func createCoreVolumeAttachment(d *schema.ResourceData, m interface{}) error {
	sync := &CoreVolumeAttachmentResourceCrud{}
	sync.D = d
	sync.Client = m.(*OracleClients).computeClient

	return CreateResource(d, sync)
}

func readCoreVolumeAttachment(d *schema.ResourceData, m interface{}) error {
	sync := &CoreVolumeAttachmentResourceCrud{}
	sync.D = d
	sync.Client = m.(*OracleClients).computeClient

	return ReadResource(sync)
}

func deleteCoreVolumeAttachment(d *schema.ResourceData, m interface{}) error {
	sync := &CoreVolumeAttachmentResourceCrud{}
	sync.D = d
	sync.Client = m.(*OracleClients).computeClient
	sync.DisableNotFoundRetries = true

	return DeleteResource(d, sync)
}

type CoreVolumeAttachmentResourceCrud struct {
	BaseCrud
	Client                 *oci_core.ComputeClient
	Res                    *oci_core.VolumeAttachment
	DisableNotFoundRetries bool
}

func (s *CoreVolumeAttachmentResourceCrud) ID() string {
	volumeAttachment := *s.Res
	return *volumeAttachment.GetId()
}

func (s *CoreVolumeAttachmentResourceCrud) CreatedPending() []string {
	return []string{
		string(oci_core.VolumeAttachmentLifecycleStateAttaching),
	}
}

func (s *CoreVolumeAttachmentResourceCrud) CreatedTarget() []string {
	return []string{
		string(oci_core.VolumeAttachmentLifecycleStateAttached),
	}
}

func (s *CoreVolumeAttachmentResourceCrud) DeletedPending() []string {
	return []string{
		string(oci_core.VolumeAttachmentLifecycleStateDetaching),
	}
}

func (s *CoreVolumeAttachmentResourceCrud) DeletedTarget() []string {
	return []string{
		string(oci_core.VolumeAttachmentLifecycleStateDetached),
	}
}

func (s *CoreVolumeAttachmentResourceCrud) Create() error {
	request := oci_core.AttachVolumeRequest{}
	err := s.populateTopLevelPolymorphicAttachVolumeRequest(&request)
	if err != nil {
		return err
	}

	request.RequestMetadata.RetryPolicy = getRetryPolicy(s.DisableNotFoundRetries, "core")

	response, err := s.Client.AttachVolume(context.Background(), request)
	if err != nil {
		return err
	}

	s.Res = &response.VolumeAttachment
	return nil
}

func (s *CoreVolumeAttachmentResourceCrud) Get() error {
	request := oci_core.GetVolumeAttachmentRequest{}

	tmp := s.D.Id()
	request.VolumeAttachmentId = &tmp

	request.RequestMetadata.RetryPolicy = getRetryPolicy(s.DisableNotFoundRetries, "core")

	response, err := s.Client.GetVolumeAttachment(context.Background(), request)
	if err != nil {
		return err
	}

	s.Res = &response.VolumeAttachment
	return nil
}

func (s *CoreVolumeAttachmentResourceCrud) Delete() error {
	request := oci_core.DetachVolumeRequest{}

	tmp := s.D.Id()
	request.VolumeAttachmentId = &tmp

	request.RequestMetadata.RetryPolicy = getRetryPolicy(s.DisableNotFoundRetries, "core")

	_, err := s.Client.DetachVolume(context.Background(), request)
	return err
}

func (s *CoreVolumeAttachmentResourceCrud) SetData() error {
	switch v := (*s.Res).(type) {
	case oci_core.IScsiVolumeAttachment:
		s.D.Set("attachment_type", "iscsi")

		if v.ChapSecret != nil {
			s.D.Set("chap_secret", *v.ChapSecret)
		}

		if v.ChapUsername != nil {
			s.D.Set("chap_username", *v.ChapUsername)
		}

		if v.Ipv4 != nil {
			s.D.Set("ipv4", *v.Ipv4)
		}

		if v.Iqn != nil {
			s.D.Set("iqn", *v.Iqn)
		}

		if v.Port != nil {
			s.D.Set("port", *v.Port)
		}

		if v.AvailabilityDomain != nil {
			s.D.Set("availability_domain", *v.AvailabilityDomain)
		}

		if v.CompartmentId != nil {
			s.D.Set("compartment_id", *v.CompartmentId)
		}

		if v.Device != nil {
			s.D.Set("device", *v.Device)
		}

		if v.DisplayName != nil {
			s.D.Set("display_name", *v.DisplayName)
		}

		if v.Id != nil {
			s.D.Set("id", *v.Id)
		}

		if v.InstanceId != nil {
			s.D.Set("instance_id", *v.InstanceId)
		}

		if v.IsPvEncryptionInTransitEnabled != nil {
			s.D.Set("is_pv_encryption_in_transit_enabled", *v.IsPvEncryptionInTransitEnabled)
		}

		if v.IsReadOnly != nil {
			s.D.Set("is_read_only", *v.IsReadOnly)
		}

		s.D.Set("state", v.LifecycleState)

		if v.TimeCreated != nil {
			s.D.Set("time_created", v.TimeCreated.String())
		}

		if v.VolumeId != nil {
			s.D.Set("volume_id", *v.VolumeId)
		}
	case oci_core.ParavirtualizedVolumeAttachment:
		s.D.Set("attachment_type", "paravirtualized")

		if v.AvailabilityDomain != nil {
			s.D.Set("availability_domain", *v.AvailabilityDomain)
		}

		if v.CompartmentId != nil {
			s.D.Set("compartment_id", *v.CompartmentId)
		}

		if v.Device != nil {
			s.D.Set("device", *v.Device)
		}

		if v.DisplayName != nil {
			s.D.Set("display_name", *v.DisplayName)
		}

		if v.Id != nil {
			s.D.Set("id", *v.Id)
		}

		if v.InstanceId != nil {
			s.D.Set("instance_id", *v.InstanceId)
		}

		if v.IsPvEncryptionInTransitEnabled != nil {
			s.D.Set("is_pv_encryption_in_transit_enabled", *v.IsPvEncryptionInTransitEnabled)
		}

		if v.IsReadOnly != nil {
			s.D.Set("is_read_only", *v.IsReadOnly)
		}

		s.D.Set("state", v.LifecycleState)

		if v.TimeCreated != nil {
			s.D.Set("time_created", v.TimeCreated.String())
		}

		if v.VolumeId != nil {
			s.D.Set("volume_id", *v.VolumeId)
		}
	default:
		log.Printf("[WARN] Received 'attachment_type' of unknown type %v", *s.Res)
		return nil
	}
	return nil
}

func (s *CoreVolumeAttachmentResourceCrud) populateTopLevelPolymorphicAttachVolumeRequest(request *oci_core.AttachVolumeRequest) error {
	//discriminator
	attachmentTypeRaw, ok := s.D.GetOkExists("attachment_type")
	var attachmentType string
	if ok {
		attachmentType = attachmentTypeRaw.(string)
	} else {
		attachmentType = "" // default value
	}
	switch strings.ToLower(attachmentType) {
	case strings.ToLower("iscsi"):
		details := oci_core.AttachIScsiVolumeDetails{}
		if useChap, ok := s.D.GetOkExists("use_chap"); ok {
			tmp := useChap.(bool)
			details.UseChap = &tmp
		}
		if device, ok := s.D.GetOkExists("device"); ok {
			tmp := device.(string)
			details.Device = &tmp
		}
		if displayName, ok := s.D.GetOkExists("display_name"); ok {
			tmp := displayName.(string)
			details.DisplayName = &tmp
		}
		if instanceId, ok := s.D.GetOkExists("instance_id"); ok {
			tmp := instanceId.(string)
			details.InstanceId = &tmp
		}
		if isReadOnly, ok := s.D.GetOkExists("is_read_only"); ok {
			tmp := isReadOnly.(bool)
			details.IsReadOnly = &tmp
		}
		if volumeId, ok := s.D.GetOkExists("volume_id"); ok {
			tmp := volumeId.(string)
			details.VolumeId = &tmp
		}
		request.AttachVolumeDetails = details
	case strings.ToLower("paravirtualized"):
		details := oci_core.AttachParavirtualizedVolumeDetails{}
		if device, ok := s.D.GetOkExists("device"); ok {
			tmp := device.(string)
			details.Device = &tmp
		}
		if displayName, ok := s.D.GetOkExists("display_name"); ok {
			tmp := displayName.(string)
			details.DisplayName = &tmp
		}
		if instanceId, ok := s.D.GetOkExists("instance_id"); ok {
			tmp := instanceId.(string)
			details.InstanceId = &tmp
		}
		if isPvEncryptionInTransitEnabled, ok := s.D.GetOkExists("is_pv_encryption_in_transit_enabled"); ok {
			tmp := isPvEncryptionInTransitEnabled.(bool)
			details.IsPvEncryptionInTransitEnabled = &tmp
		}
		if isReadOnly, ok := s.D.GetOkExists("is_read_only"); ok {
			tmp := isReadOnly.(bool)
			details.IsReadOnly = &tmp
		}
		if volumeId, ok := s.D.GetOkExists("volume_id"); ok {
			tmp := volumeId.(string)
			details.VolumeId = &tmp
		}
		request.AttachVolumeDetails = details
	default:
		return fmt.Errorf("unknown attachment_type '%v' was specified", attachmentType)
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package net provides a portable interface for network I/O, including
TCP/IP, UDP, domain name resolution, and Unix domain sockets.

Although the package provides access to low-level networking
primitives, most clients will need only the basic interface provided
by the Dial, Listen, and Accept functions and the associated
Conn and Listener interfaces. The crypto/tls package uses
the same interfaces and similar Dial and Listen functions.

The Dial function connects to a server:

	conn, err := net.Dial("tcp", "google.com:80")
	if err != nil {
		// handle error
	}
	fmt.Fprintf(conn, "GET / HTTP/1.0\r\n\r\n")
	status, err := bufio.NewReader(conn).ReadString('\n')
	// ...

The Listen function creates servers:

	ln, err := net.Listen("tcp", ":8080")
	if err != nil {
		// handle error
	}
	for {
		conn, err := ln.Accept()
		if err != nil {
			// handle error
		}
		go handleConnection(conn)
	}
*/
package net

import (
	"errors"
	"io"
	"os"
	"syscall"
	"time"
)

func init() {
	sysInit()
	supportsIPv4 = probeIPv4Stack()
	supportsIPv6, supportsIPv4map = probeIPv6Stack()
}

// Addr represents a network end point address.
type Addr interface {
	Network() string // name of the network
	String() string  // string form of address
}

// Conn is a generic stream-oriented network connection.
//
// Multiple goroutines may invoke methods on a Conn simultaneously.
type Conn interface {
	// Read reads data from the connection.
	// Read can be made to time out and return a Error with Timeout() == true
	// after a fixed time limit; see SetDeadline and SetReadDeadline.
	Read(b []byte) (n int, err error)

	// Write writes data to the connection.
	// Write can be made to time out and return a Error with Timeout() == true
	// after a fixed time limit; see SetDeadline and SetWriteDeadline.
	Write(b []byte) (n int, err error)

	// Close closes the connection.
	// Any blocked Read or Write operations will be unblocked and return errors.
	Close() error

	// LocalAddr returns the local network address.
	LocalAddr() Addr

	// RemoteAddr returns the remote network address.
	RemoteAddr() Addr

	// SetDeadline sets the read and write deadlines associated
	// with the connection. It is equivalent to calling both
	// SetReadDeadline and SetWriteDeadline.
	//
	// A deadline is an absolute time after which I/O operations
	// fail with a timeout (see type Error) instead of
	// blocking. The deadline applies to all future I/O, not just
	// the immediately following call to Read or Write.
	//
	// An idle timeout can be implemented by repeatedly extending
	// the deadline after successful Read or Write calls.
	//
	// A zero value for t means I/O operations will not time out.
	SetDeadline(t time.Time) error

	// SetReadDeadline sets the deadline for future Read calls.
	// A zero value for t means Read will not time out.
	SetReadDeadline(t time.Time) error

	// SetWriteDeadline sets the deadline for future Write calls.
	// Even if write times out, it may return n > 0, indicating that
	// some of the data was successfully written.
	// A zero value for t means Write will not time out.
	SetWriteDeadline(t time.Time) error
}

type conn struct {
	fd *netFD
}

func (c *conn) ok() bool { return c != nil && c.fd != nil }

// Implementation of the Conn interface.

// Read implements the Conn Read method.
func (c *conn) Read(b []byte) (int, error) {
	if !c.ok() {
		return 0, syscall.EINVAL
	}
	return c.fd.Read(b)
}

// Write implements the Conn Write method.
func (c *conn) Write(b []byte) (int, error) {
	if !c.ok() {
		return 0, syscall.EINVAL
	}
	return c.fd.Write(b)
}

// Close closes the connection.
func (c *conn) Close() error {
	if !c.ok() {
		return syscall.EINVAL
	}
	return c.fd.Close()
}

// LocalAddr returns the local network address.
// The Addr returned is shared by all invocations of LocalAddr, so
// do not modify it.
func (c *conn) LocalAddr() Addr {
	if !c.ok() {
		return nil
	}
	return c.fd.laddr
}

// RemoteAddr returns the remote network address.
// The Addr returned is shared by all invocations of RemoteAddr, so
// do not modify it.
func (c *conn) RemoteAddr() Addr {
	if !c.ok() {
		return nil
	}
	return c.fd.raddr
}

// SetDeadline implements the Conn SetDeadline method.
func (c *conn) SetDeadline(t time.Time) error {
	if !c.ok() {
		return syscall.EINVAL
	}
	return c.fd.setDeadline(t)
}

// SetReadDeadline implements the Conn SetReadDeadline method.
func (c *conn) SetReadDeadline(t time.Time) error {
	if !c.ok() {
		return syscall.EINVAL
	}
	return c.fd.setReadDeadline(t)
}

// SetWriteDeadline implements the Conn SetWriteDeadline method.
func (c *conn) SetWriteDeadline(t time.Time) error {
	if !c.ok() {
		return syscall.EINVAL
	}
	return c.fd.setWriteDeadline(t)
}

// SetReadBuffer sets the size of the operating system's
// receive buffer associated with the connection.
func (c *conn) SetReadBuffer(bytes int) error {
	if !c.ok() {
		return syscall.EINVAL
	}
	return setReadBuffer(c.fd, bytes)
}

// SetWriteBuffer sets the size of the operating system's
// transmit buffer associated with the connection.
func (c *conn) SetWriteBuffer(bytes int) error {
	if !c.ok() {
		return syscall.EINVAL
	}
	return setWriteBuffer(c.fd, bytes)
}

// File sets the underlying os.File to blocking mode and returns a copy.
// It is the caller's responsibility to close f when finished.
// Closing c does not affect f, and closing f does not affect c.
//
// The returned os.File's file descriptor is different from the connection's.
// Attempting to change properties of the original using this duplicate
// may or may not have the desired effect.
func (c *conn) File() (f *os.File, err error) { return c.fd.dup() }

// An Error represents a network error.
type Error interface {
	error
	Timeout() bool   // Is the error a timeout?
	Temporary() bool // Is the error temporary?
}

// PacketConn is a generic packet-oriented network connection.
//
// Multiple goroutines may invoke methods on a PacketConn simultaneously.
type PacketConn interface {
	// ReadFrom reads a packet from the connection,
	// copying the payload into b.  It returns the number of
	// bytes copied into b and the return address that
	// was on the packet.
	// ReadFrom can be made to time out and return
	// an error with Timeout() == true after a fixed time limit;
	// see SetDeadline and SetReadDeadline.
	ReadFrom(b []byte) (n int, addr Addr, err error)

	// WriteTo writes a packet with payload b to addr.
	// WriteTo can be made to time out and return
	// an error with Timeout() == true after a fixed time limit;
	// see SetDeadline and SetWriteDeadline.
	// On packet-oriented connections, write timeouts are rare.
	WriteTo(b []byte, addr Addr) (n int, err error)

	// Close closes the connection.
	// Any blocked ReadFrom or WriteTo operations will be unblocked and return errors.
	Close() error

	// LocalAddr returns the local network address.
	LocalAddr() Addr

	// SetDeadline sets the read and write deadlines associated
	// with the connection.
	SetDeadline(t time.Time) error

	// SetReadDeadline sets the deadline for future Read calls.
	// If the deadline is reached, Read will fail with a timeout
	// (see type Error) instead of blocking.
	// A zero value for t means Read will not time out.
	SetReadDeadline(t time.Time) error

	// SetWriteDeadline sets the deadline for future Write calls.
	// If the deadline is reached, Write will fail with a timeout
	// (see type Error) instead of blocking.
	// A zero value for t means Write will not time out.
	// Even if write times out, it may return n > 0, indicating that
	// some of the data was successfully written.
	SetWriteDeadline(t time.Time) error
}

var listenerBacklog = maxListenerBacklog()

// A Listener is a generic network listener for stream-oriented protocols.
//
// Multiple goroutines may invoke methods on a Listener simultaneously.
type Listener interface {
	// Accept waits for and returns the next connection to the listener.
	Accept() (c Conn, err error)

	// Close closes the listener.
	// Any blocked Accept operations will be unblocked and return errors.
	Close() error

	// Addr returns the listener's network address.
	Addr() Addr
}

// Various errors contained in OpError.
var (
	// For connection setup and write operations.
	errMissingAddress = errors.New("missing address")

	// For both read and write operations.
	errTimeout          error = &timeoutError{}
	errClosing                = errors.New("use of closed network connection")
	ErrWriteToConnected       = errors.New("use of WriteTo with pre-connected connection")
)

// OpError is the error type usually returned by functions in the net
// package. It describes the operation, network type, and address of
// an error.
type OpError struct {
	// Op is the operation which caused the error, such as
	// "read" or "write".
	Op string

	// Net is the network type on which this error occurred,
	// such as "tcp" or "udp6".
	Net string

	// Addr is the network address on which this error occurred.
	Addr Addr

	// Err is the error that occurred during the operation.
	Err error
}

func (e *OpError) Error() string {
	if e == nil {
		return "<nil>"
	}
	s := e.Op
	if e.Net != "" {
		s += " " + e.Net
	}
	if e.Addr != nil {
		s += " " + e.Addr.String()
	}
	s += ": " + e.Err.Error()
	return s
}

type temporary interface {
	Temporary() bool
}

func (e *OpError) Temporary() bool {
	t, ok := e.Err.(temporary)
	return ok && t.Temporary()
}

var noDeadline = time.Time{}

type timeout interface {
	Timeout() bool
}

func (e *OpError) Timeout() bool {
	t, ok := e.Err.(timeout)
	return ok && t.Timeout()
}

type timeoutError struct{}

func (e *timeoutError) Error() string   { return "i/o timeout" }
func (e *timeoutError) Timeout() bool   { return true }
func (e *timeoutError) Temporary() bool { return true }

type AddrError struct {
	Err  string
	Addr string
}

func (e *AddrError) Error() string {
	if e == nil {
		return "<nil>"
	}
	s := e.Err
	if e.Addr != "" {
		s += " " + e.Addr
	}
	return s
}

func (e *AddrError) Temporary() bool {
	return false
}

func (e *AddrError) Timeout() bool {
	return false
}

type UnknownNetworkError string

func (e UnknownNetworkError) Error() string   { return "unknown network " + string(e) }
func (e UnknownNetworkError) Temporary() bool { return false }
func (e UnknownNetworkError) Timeout() bool   { return false }

type InvalidAddrError string

func (e InvalidAddrError) Error() string   { return string(e) }
func (e InvalidAddrError) Timeout() bool   { return false }
func (e InvalidAddrError) Temporary() bool { return false }

// DNSConfigError represents an error reading the machine's DNS configuration.
type DNSConfigError struct {
	Err error
}

func (e *DNSConfigError) Error() string {
	return "error reading DNS config: " + e.Err.Error()
}

func (e *DNSConfigError) Timeout() bool   { return false }
func (e *DNSConfigError) Temporary() bool { return false }

type writerOnly struct {
	io.Writer
}

// Fallback implementation of io.ReaderFrom's ReadFrom, when sendfile isn't
// applicable.
func genericReadFrom(w io.Writer, r io.Reader) (n int64, err error) {
	// Use wrapper to hide existing r.ReadFrom from io.Copy.
	return io.Copy(writerOnly{w}, r)
}

// Limit the number of concurrent cgo-using goroutines, because
// each will block an entire operating system thread. The usual culprit
// is resolving many DNS names in separate goroutines but the DNS
// server is not responding. Then the many lookups each use a different
// thread, and the system or the program runs out of threads.

var threadLimit = make(chan struct{}, 500)

// Using send for acquire is fine here because we are not using this
// to protect any memory. All we care about is the number of goroutines
// making calls at a time.

func acquireThread() {
	threadLimit <- struct{}{}
}

func releaseThread() {
	<-threadLimit
}
<commit_msg>net: remove old comment<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package net provides a portable interface for network I/O, including
TCP/IP, UDP, domain name resolution, and Unix domain sockets.

Although the package provides access to low-level networking
primitives, most clients will need only the basic interface provided
by the Dial, Listen, and Accept functions and the associated
Conn and Listener interfaces. The crypto/tls package uses
the same interfaces and similar Dial and Listen functions.

The Dial function connects to a server:

	conn, err := net.Dial("tcp", "google.com:80")
	if err != nil {
		// handle error
	}
	fmt.Fprintf(conn, "GET / HTTP/1.0\r\n\r\n")
	status, err := bufio.NewReader(conn).ReadString('\n')
	// ...

The Listen function creates servers:

	ln, err := net.Listen("tcp", ":8080")
	if err != nil {
		// handle error
	}
	for {
		conn, err := ln.Accept()
		if err != nil {
			// handle error
		}
		go handleConnection(conn)
	}
*/
package net

import (
	"errors"
	"io"
	"os"
	"syscall"
	"time"
)

func init() {
	sysInit()
	supportsIPv4 = probeIPv4Stack()
	supportsIPv6, supportsIPv4map = probeIPv6Stack()
}

// Addr represents a network end point address.
type Addr interface {
	Network() string // name of the network
	String() string  // string form of address
}

// Conn is a generic stream-oriented network connection.
//
// Multiple goroutines may invoke methods on a Conn simultaneously.
type Conn interface {
	// Read reads data from the connection.
	// Read can be made to time out and return a Error with Timeout() == true
	// after a fixed time limit; see SetDeadline and SetReadDeadline.
	Read(b []byte) (n int, err error)

	// Write writes data to the connection.
	// Write can be made to time out and return a Error with Timeout() == true
	// after a fixed time limit; see SetDeadline and SetWriteDeadline.
	Write(b []byte) (n int, err error)

	// Close closes the connection.
	// Any blocked Read or Write operations will be unblocked and return errors.
	Close() error

	// LocalAddr returns the local network address.
	LocalAddr() Addr

	// RemoteAddr returns the remote network address.
	RemoteAddr() Addr

	// SetDeadline sets the read and write deadlines associated
	// with the connection. It is equivalent to calling both
	// SetReadDeadline and SetWriteDeadline.
	//
	// A deadline is an absolute time after which I/O operations
	// fail with a timeout (see type Error) instead of
	// blocking. The deadline applies to all future I/O, not just
	// the immediately following call to Read or Write.
	//
	// An idle timeout can be implemented by repeatedly extending
	// the deadline after successful Read or Write calls.
	//
	// A zero value for t means I/O operations will not time out.
	SetDeadline(t time.Time) error

	// SetReadDeadline sets the deadline for future Read calls.
	// A zero value for t means Read will not time out.
	SetReadDeadline(t time.Time) error

	// SetWriteDeadline sets the deadline for future Write calls.
	// Even if write times out, it may return n > 0, indicating that
	// some of the data was successfully written.
	// A zero value for t means Write will not time out.
	SetWriteDeadline(t time.Time) error
}

type conn struct {
	fd *netFD
}

func (c *conn) ok() bool { return c != nil && c.fd != nil }

// Implementation of the Conn interface.

// Read implements the Conn Read method.
func (c *conn) Read(b []byte) (int, error) {
	if !c.ok() {
		return 0, syscall.EINVAL
	}
	return c.fd.Read(b)
}

// Write implements the Conn Write method.
func (c *conn) Write(b []byte) (int, error) {
	if !c.ok() {
		return 0, syscall.EINVAL
	}
	return c.fd.Write(b)
}

// Close closes the connection.
func (c *conn) Close() error {
	if !c.ok() {
		return syscall.EINVAL
	}
	return c.fd.Close()
}

// LocalAddr returns the local network address.
// The Addr returned is shared by all invocations of LocalAddr, so
// do not modify it.
func (c *conn) LocalAddr() Addr {
	if !c.ok() {
		return nil
	}
	return c.fd.laddr
}

// RemoteAddr returns the remote network address.
// The Addr returned is shared by all invocations of RemoteAddr, so
// do not modify it.
func (c *conn) RemoteAddr() Addr {
	if !c.ok() {
		return nil
	}
	return c.fd.raddr
}

// SetDeadline implements the Conn SetDeadline method.
func (c *conn) SetDeadline(t time.Time) error {
	if !c.ok() {
		return syscall.EINVAL
	}
	return c.fd.setDeadline(t)
}

// SetReadDeadline implements the Conn SetReadDeadline method.
func (c *conn) SetReadDeadline(t time.Time) error {
	if !c.ok() {
		return syscall.EINVAL
	}
	return c.fd.setReadDeadline(t)
}

// SetWriteDeadline implements the Conn SetWriteDeadline method.
func (c *conn) SetWriteDeadline(t time.Time) error {
	if !c.ok() {
		return syscall.EINVAL
	}
	return c.fd.setWriteDeadline(t)
}

// SetReadBuffer sets the size of the operating system's
// receive buffer associated with the connection.
func (c *conn) SetReadBuffer(bytes int) error {
	if !c.ok() {
		return syscall.EINVAL
	}
	return setReadBuffer(c.fd, bytes)
}

// SetWriteBuffer sets the size of the operating system's
// transmit buffer associated with the connection.
func (c *conn) SetWriteBuffer(bytes int) error {
	if !c.ok() {
		return syscall.EINVAL
	}
	return setWriteBuffer(c.fd, bytes)
}

// File sets the underlying os.File to blocking mode and returns a copy.
// It is the caller's responsibility to close f when finished.
// Closing c does not affect f, and closing f does not affect c.
//
// The returned os.File's file descriptor is different from the connection's.
// Attempting to change properties of the original using this duplicate
// may or may not have the desired effect.
func (c *conn) File() (f *os.File, err error) { return c.fd.dup() }

// An Error represents a network error.
type Error interface {
	error
	Timeout() bool   // Is the error a timeout?
	Temporary() bool // Is the error temporary?
}

// PacketConn is a generic packet-oriented network connection.
//
// Multiple goroutines may invoke methods on a PacketConn simultaneously.
type PacketConn interface {
	// ReadFrom reads a packet from the connection,
	// copying the payload into b.  It returns the number of
	// bytes copied into b and the return address that
	// was on the packet.
	// ReadFrom can be made to time out and return
	// an error with Timeout() == true after a fixed time limit;
	// see SetDeadline and SetReadDeadline.
	ReadFrom(b []byte) (n int, addr Addr, err error)

	// WriteTo writes a packet with payload b to addr.
	// WriteTo can be made to time out and return
	// an error with Timeout() == true after a fixed time limit;
	// see SetDeadline and SetWriteDeadline.
	// On packet-oriented connections, write timeouts are rare.
	WriteTo(b []byte, addr Addr) (n int, err error)

	// Close closes the connection.
	// Any blocked ReadFrom or WriteTo operations will be unblocked and return errors.
	Close() error

	// LocalAddr returns the local network address.
	LocalAddr() Addr

	// SetDeadline sets the read and write deadlines associated
	// with the connection.
	SetDeadline(t time.Time) error

	// SetReadDeadline sets the deadline for future Read calls.
	// If the deadline is reached, Read will fail with a timeout
	// (see type Error) instead of blocking.
	// A zero value for t means Read will not time out.
	SetReadDeadline(t time.Time) error

	// SetWriteDeadline sets the deadline for future Write calls.
	// If the deadline is reached, Write will fail with a timeout
	// (see type Error) instead of blocking.
	// A zero value for t means Write will not time out.
	// Even if write times out, it may return n > 0, indicating that
	// some of the data was successfully written.
	SetWriteDeadline(t time.Time) error
}

var listenerBacklog = maxListenerBacklog()

// A Listener is a generic network listener for stream-oriented protocols.
//
// Multiple goroutines may invoke methods on a Listener simultaneously.
type Listener interface {
	// Accept waits for and returns the next connection to the listener.
	Accept() (c Conn, err error)

	// Close closes the listener.
	// Any blocked Accept operations will be unblocked and return errors.
	Close() error

	// Addr returns the listener's network address.
	Addr() Addr
}

// Various errors contained in OpError.
var (
	// For connection setup and write operations.
	errMissingAddress = errors.New("missing address")

	// For both read and write operations.
	errTimeout          error = &timeoutError{}
	errClosing                = errors.New("use of closed network connection")
	ErrWriteToConnected       = errors.New("use of WriteTo with pre-connected connection")
)

// OpError is the error type usually returned by functions in the net
// package. It describes the operation, network type, and address of
// an error.
type OpError struct {
	// Op is the operation which caused the error, such as
	// "read" or "write".
	Op string

	// Net is the network type on which this error occurred,
	// such as "tcp" or "udp6".
	Net string

	// Addr is the network address on which this error occurred.
	Addr Addr

	// Err is the error that occurred during the operation.
	Err error
}

func (e *OpError) Error() string {
	if e == nil {
		return "<nil>"
	}
	s := e.Op
	if e.Net != "" {
		s += " " + e.Net
	}
	if e.Addr != nil {
		s += " " + e.Addr.String()
	}
	s += ": " + e.Err.Error()
	return s
}

type temporary interface {
	Temporary() bool
}

func (e *OpError) Temporary() bool {
	t, ok := e.Err.(temporary)
	return ok && t.Temporary()
}

var noDeadline = time.Time{}

type timeout interface {
	Timeout() bool
}

func (e *OpError) Timeout() bool {
	t, ok := e.Err.(timeout)
	return ok && t.Timeout()
}

type timeoutError struct{}

func (e *timeoutError) Error() string   { return "i/o timeout" }
func (e *timeoutError) Timeout() bool   { return true }
func (e *timeoutError) Temporary() bool { return true }

type AddrError struct {
	Err  string
	Addr string
}

func (e *AddrError) Error() string {
	if e == nil {
		return "<nil>"
	}
	s := e.Err
	if e.Addr != "" {
		s += " " + e.Addr
	}
	return s
}

func (e *AddrError) Temporary() bool {
	return false
}

func (e *AddrError) Timeout() bool {
	return false
}

type UnknownNetworkError string

func (e UnknownNetworkError) Error() string   { return "unknown network " + string(e) }
func (e UnknownNetworkError) Temporary() bool { return false }
func (e UnknownNetworkError) Timeout() bool   { return false }

type InvalidAddrError string

func (e InvalidAddrError) Error() string   { return string(e) }
func (e InvalidAddrError) Timeout() bool   { return false }
func (e InvalidAddrError) Temporary() bool { return false }

// DNSConfigError represents an error reading the machine's DNS configuration.
type DNSConfigError struct {
	Err error
}

func (e *DNSConfigError) Error() string {
	return "error reading DNS config: " + e.Err.Error()
}

func (e *DNSConfigError) Timeout() bool   { return false }
func (e *DNSConfigError) Temporary() bool { return false }

type writerOnly struct {
	io.Writer
}

// Fallback implementation of io.ReaderFrom's ReadFrom, when sendfile isn't
// applicable.
func genericReadFrom(w io.Writer, r io.Reader) (n int64, err error) {
	// Use wrapper to hide existing r.ReadFrom from io.Copy.
	return io.Copy(writerOnly{w}, r)
}

// Limit the number of concurrent cgo-using goroutines, because
// each will block an entire operating system thread. The usual culprit
// is resolving many DNS names in separate goroutines but the DNS
// server is not responding. Then the many lookups each use a different
// thread, and the system or the program runs out of threads.

var threadLimit = make(chan struct{}, 500)

func acquireThread() {
	threadLimit <- struct{}{}
}

func releaseThread() {
	<-threadLimit
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"math"
	"strconv"
)

// expression parser

type exprType int

const (
	etMetric exprType = iota
	etFunc
	etConst
)

type expr struct {
	target    string
	etype     exprType
	val       float64
	args      []*expr
	argString string
}

func (e *expr) metrics() []string {

	switch e.etype {
	case etMetric:
		return []string{e.target}
	case etConst:
		return nil
	case etFunc:
		var r []string
		for _, a := range e.args {
			r = append(r, a.metrics()...)
		}
		return r
	}

	return nil
}

func parseExpr(e string) (*expr, string, error) {

	if '0' <= e[0] && e[0] <= '9' {
		val, e, err := parseConst(e)
		return &expr{val: val, etype: etConst}, e, err
	}

	name, e := parseName(e)

	if e != "" && e[0] == '(' {
		exp := &expr{target: name, etype: etFunc}

		argString, args, e, err := parseArgList(e)
		exp.argString = argString
		exp.args = args

		return exp, e, err
	}

	return &expr{target: name}, e, nil
}

var (
	ErrMissingComma        = errors.New("missing comma")
	ErrUnexpectedCharacter = errors.New("unexpected character")
)

func parseArgList(e string) (string, []*expr, string, error) {

	var args []*expr

	if e[0] != '(' {
		panic("arg list should start with paren")
	}

	argString := e[1:]

	e = e[1:]

	for {
		var arg *expr
		var err error
		arg, e, err = parseExpr(e)
		if err != nil {
			return "", nil, "", err
		}
		args = append(args, arg)

		if e == "" {
			return "", nil, "", ErrMissingComma
		}

		if e[0] == ')' {
			return argString[:len(argString)-len(e)], args, e[1:], nil
		}

		if e[0] != ',' && e[0] != ' ' {
			return "", nil, "", ErrUnexpectedCharacter
		}

		e = e[1:]
	}
}

func isNameChar(r byte) bool {
	return false ||
		'a' <= r && r <= 'z' ||
		'A' <= r && r <= 'Z' ||
		'0' <= r && r <= '9' ||
		r == '.' || r == '_' || r == '-' || r == '*'
}

func isDigit(r byte) bool {
	return '0' <= r && r <= '9'
}

func parseConst(s string) (float64, string, error) {

	var i int
	for i < len(s) && isDigit(s[i]) {
		i++
	}

	v, err := strconv.ParseFloat(s[:i], 64)
	if err != nil {
		return 0, "", err
	}

	return v, s[i:], err
}

func parseName(s string) (string, string) {

	var i int
	for i < len(s) && isNameChar(s[i]) {
		i++
	}

	if i == len(s) {
		return s, ""
	}

	return s[:i], s[i:]
}

type namedExpr struct {
	name string
	data []float64
}

func evalExpr(e *expr, values map[string][]namedExpr) []namedExpr {

	switch e.etype {
	case etMetric:
		return values[e.target]
	case etConst:
		return []namedExpr{{name: e.target, data: []float64{e.val}}}
	}

	// evaluate the function
	switch e.target {
	case "sum", "sumSeries":
		// make sure the arrays are all the same 'size'
		var args []namedExpr
		for _, arg := range e.args {
			a := evalExpr(arg, values)
			args = append(args, a...)
		}
		r := namedExpr{
			name: fmt.Sprintf("sum(%s)", e.argString),
			data: make([]float64, len(args[0].data)),
		}
		for _, arg := range args {
			for i, v := range arg.data {
				if math.IsNaN(v) {
					continue
				}
				r.data[i] += v
			}
		}
		return []namedExpr{r}

	case "nonNegativeDerivative":
		arg := evalExpr(e.args[0], values)
		var result []namedExpr
		for _, a := range arg {
			r := namedExpr{
				name: fmt.Sprintf("nonNegativeDerivative(%s)", a.name),
				data: make([]float64, len(a.data)),
			}
			prev := math.NaN()
			for i, v := range a.data {
				if math.IsNaN(prev) || math.IsNaN(v) {
					prev = v
					r.data[i] = math.NaN()
					continue
				}
				r.data[i] = v - prev
				if r.data[i] < 0 {
					r.data[i] = math.NaN()
				}
				prev = v
			}
			result = append(result, r)
		}
		return result

	case "movingAverage":
		arg := evalExpr(e.args[0], values)
		n := evalExpr(e.args[1], values)
		if len(n) != 1 || len(n[0].data) != 1 {
			// fail
			return nil
		}

		windowSize := int(n[0].data[0])

		var result []namedExpr

		for _, a := range arg {
			w := &Windowed{data: make([]float64, windowSize)}
			r := namedExpr{
				name: fmt.Sprintf("movingAverage(%s, %d)", a.name, windowSize),
				data: make([]float64, len(a.data)),
			}
			for i, v := range a.data {
				w.Push(v)
				r.data[i] = w.Mean()
			}
			result = append(result, r)
		}
		return result
	}

	return nil
}

// From github.com/dgryski/go-onlinestats
// Copied here because we don't need the rest of the package, and we only need
// a small part of this type

type Windowed struct {
	data   []float64
	head   int
	length int
	sum    float64
}

func (w *Windowed) Push(n float64) {
	old := w.data[w.head]

	w.length++

	w.data[w.head] = n
	w.head++
	if w.head >= len(w.data) {
		w.head = 0
	}

	w.sum -= old
	w.sum += n
}

func (w *Windowed) Len() int {
	if w.length < len(w.data) {
		return w.length
	}

	return len(w.data)
}

func (w *Windowed) Mean() float64 { return w.sum / float64(w.Len()) }
<commit_msg>handle missing values in movingAverage()<commit_after>package main

import (
	"errors"
	"fmt"
	"math"
	"strconv"
)

// expression parser

type exprType int

const (
	etMetric exprType = iota
	etFunc
	etConst
)

type expr struct {
	target    string
	etype     exprType
	val       float64
	args      []*expr
	argString string
}

func (e *expr) metrics() []string {

	switch e.etype {
	case etMetric:
		return []string{e.target}
	case etConst:
		return nil
	case etFunc:
		var r []string
		for _, a := range e.args {
			r = append(r, a.metrics()...)
		}
		return r
	}

	return nil
}

func parseExpr(e string) (*expr, string, error) {

	if '0' <= e[0] && e[0] <= '9' {
		val, e, err := parseConst(e)
		return &expr{val: val, etype: etConst}, e, err
	}

	name, e := parseName(e)

	if e != "" && e[0] == '(' {
		exp := &expr{target: name, etype: etFunc}

		argString, args, e, err := parseArgList(e)
		exp.argString = argString
		exp.args = args

		return exp, e, err
	}

	return &expr{target: name}, e, nil
}

var (
	ErrMissingComma        = errors.New("missing comma")
	ErrUnexpectedCharacter = errors.New("unexpected character")
)

func parseArgList(e string) (string, []*expr, string, error) {

	var args []*expr

	if e[0] != '(' {
		panic("arg list should start with paren")
	}

	argString := e[1:]

	e = e[1:]

	for {
		var arg *expr
		var err error
		arg, e, err = parseExpr(e)
		if err != nil {
			return "", nil, "", err
		}
		args = append(args, arg)

		if e == "" {
			return "", nil, "", ErrMissingComma
		}

		if e[0] == ')' {
			return argString[:len(argString)-len(e)], args, e[1:], nil
		}

		if e[0] != ',' && e[0] != ' ' {
			return "", nil, "", ErrUnexpectedCharacter
		}

		e = e[1:]
	}
}

func isNameChar(r byte) bool {
	return false ||
		'a' <= r && r <= 'z' ||
		'A' <= r && r <= 'Z' ||
		'0' <= r && r <= '9' ||
		r == '.' || r == '_' || r == '-' || r == '*'
}

func isDigit(r byte) bool {
	return '0' <= r && r <= '9'
}

func parseConst(s string) (float64, string, error) {

	var i int
	for i < len(s) && isDigit(s[i]) {
		i++
	}

	v, err := strconv.ParseFloat(s[:i], 64)
	if err != nil {
		return 0, "", err
	}

	return v, s[i:], err
}

func parseName(s string) (string, string) {

	var i int
	for i < len(s) && isNameChar(s[i]) {
		i++
	}

	if i == len(s) {
		return s, ""
	}

	return s[:i], s[i:]
}

type namedExpr struct {
	name string
	data []float64
}

func evalExpr(e *expr, values map[string][]namedExpr) []namedExpr {

	switch e.etype {
	case etMetric:
		return values[e.target]
	case etConst:
		return []namedExpr{{name: e.target, data: []float64{e.val}}}
	}

	// evaluate the function
	switch e.target {
	case "sum", "sumSeries":
		// make sure the arrays are all the same 'size'
		var args []namedExpr
		for _, arg := range e.args {
			a := evalExpr(arg, values)
			args = append(args, a...)
		}
		r := namedExpr{
			name: fmt.Sprintf("sum(%s)", e.argString),
			data: make([]float64, len(args[0].data)),
		}
		for _, arg := range args {
			for i, v := range arg.data {
				if math.IsNaN(v) {
					continue
				}
				r.data[i] += v
			}
		}
		return []namedExpr{r}

	case "nonNegativeDerivative":
		arg := evalExpr(e.args[0], values)
		var result []namedExpr
		for _, a := range arg {
			r := namedExpr{
				name: fmt.Sprintf("nonNegativeDerivative(%s)", a.name),
				data: make([]float64, len(a.data)),
			}
			prev := math.NaN()
			for i, v := range a.data {
				if math.IsNaN(prev) || math.IsNaN(v) {
					prev = v
					r.data[i] = math.NaN()
					continue
				}
				r.data[i] = v - prev
				if r.data[i] < 0 {
					r.data[i] = math.NaN()
				}
				prev = v
			}
			result = append(result, r)
		}
		return result

	case "movingAverage":
		arg := evalExpr(e.args[0], values)
		n := evalExpr(e.args[1], values)
		if len(n) != 1 || len(n[0].data) != 1 {
			// fail
			return nil
		}

		windowSize := int(n[0].data[0])

		var result []namedExpr

		for _, a := range arg {
			w := &Windowed{data: make([]float64, windowSize)}
			r := namedExpr{
				name: fmt.Sprintf("movingAverage(%s, %d)", a.name, windowSize),
				data: make([]float64, len(a.data)),
			}
			for i, v := range a.data {
				if math.IsNaN(v) {
					// make sure NaN's are ignored
					v = 0
				}
				w.Push(v)
				r.data[i] = w.Mean()
			}
			result = append(result, r)
		}
		return result
	}

	return nil
}

// From github.com/dgryski/go-onlinestats
// Copied here because we don't need the rest of the package, and we only need
// a small part of this type

type Windowed struct {
	data   []float64
	head   int
	length int
	sum    float64
}

func (w *Windowed) Push(n float64) {
	old := w.data[w.head]

	w.length++

	w.data[w.head] = n
	w.head++
	if w.head >= len(w.data) {
		w.head = 0
	}

	w.sum -= old
	w.sum += n
}

func (w *Windowed) Len() int {
	if w.length < len(w.data) {
		return w.length
	}

	return len(w.data)
}

func (w *Windowed) Mean() float64 { return w.sum / float64(w.Len()) }
<|endoftext|>


<commit_before>package podcasts

import (
	"encoding/xml"
	"io"
	"time"
)

const (
	rssXmlns   = "http://www.itunes.com/dtds/podcast-1.0.dtd"
	rssVersion = "2.0"
)

type PubDate struct {
	time.Time
}

func (p PubDate) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
	e.EncodeToken(start)
	e.EncodeToken(xml.CharData([]byte(p.Format("Mon Jan 02 15:04:05 -0700 2006"))))
	e.EncodeToken(xml.EndElement{start.Name})
	return nil
}

type Owner struct {
	XMLName xml.Name `xml:"itunes:owner"`
	Name    string   `xml:"itunes:name"`
	Email   string   `xml:"itunes:email"`
}

type Image struct {
	XMLName xml.Name `xml:"itunes:image"`
	Href    string   `xml:"href,attr"`
}

type Category struct {
	XMLName    xml.Name `xml:"itunes:category"`
	Href       string   `xml:"text,attr"`
	Categories []*Category
}

type Enclosure struct {
	XMLName xml.Name `xml:"enclosure"`
	URL     string   `xml:"url,attr"`
	Length  string   `xml:"length,attr"`
	Type    string   `xml:"type,attr"`
}

type Item struct {
	XMLName         xml.Name      `xml:"item"`
	Title           string        `xml:"title"`
	GUID            string        `xml:"guid"`
	PubDate         *PubDate      `xml:"pubdate"`
	Author          string        `xml:"itunes:author,omitempty""`
	Subtitle        string        `xml:"itunes:subtitle,omitempty""`
	Summary         string        `xml:"itunes:summary,omitempty""`
	Explicit        string        `xml:"itunes:explicit,omitempty"`
	Block           string        `xml:"itunes:block,omitempty"`
	ClosedCaptioned string        `xml:"itunes:isClosedCaptioned,omitempty"`
	Order           int           `xml:"itunes:order,omitempty"`
	Duration        time.Duration `xml:"itunes:duration,omitempty""`
	Image           *Image
	Enclosure       *Enclosure
}

type Channel struct {
	XMLName     xml.Name `xml:"channel"`
	Title       string   `xml:"title"`
	Link        string   `xml:"link"`
	Language    string   `xml:"language"`
	Description string   `xml:"description"`
	Copyright   string   `xml:"copyright"`
	Subtitle    string   `xml:"itunes:subtitle,omitempty""`
	Author      string   `xml:"itunes:author,omitempty""`
	Summary     string   `xml:"itunes:summary,omitempty""`
	Block       string   `xml:"itunes:block,omitempty"`
	Complete    string   `xml:"itunes:complete,omitempty"`
	NewFeedURL  string   `xml:"itunes:new-feed-url,omitempty"`
	Owner       *Owner
	Image       *Image
	Categories  []*Category
	Items       []*Item
}

type Feed struct {
	XMLName xml.Name `xml:"rss"`
	Xmlns   string   `xml:"xmlns:itunes,attr"`
	Version string   `xml:"version,attr"`
	Channel *Channel
}

func (f *Feed) Xml() (string, error) {
	data, err := xml.MarshalIndent(f, "", "  ")
	if err != nil {
		return "", err
	}
	s := xml.Header + string(data)
	return s, nil
}

func (f *Feed) Write(w io.Writer) error {
	if _, err := w.Write([]byte(xml.Header)); err != nil {
		return err
	}
	enc := xml.NewEncoder(w)
	enc.Indent("", "  ")
	return enc.Encode(f)
}
<commit_msg>updates<commit_after>package podcasts

import (
	"encoding/xml"
	"io"
	"time"
)

const (
	rssXmlns   = "http://www.itunes.com/dtds/podcast-1.0.dtd"
	rssVersion = "2.0"
)

type PubDate struct {
	time.Time
}

func (p PubDate) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
	e.EncodeToken(start)
	e.EncodeToken(xml.CharData([]byte(p.Format("Mon, 02 Jan 2006 15:04:05 -0700"))))
	e.EncodeToken(xml.EndElement{start.Name})
	return nil
}

type Owner struct {
	XMLName xml.Name `xml:"itunes:owner"`
	Name    string   `xml:"itunes:name"`
	Email   string   `xml:"itunes:email"`
}

type Image struct {
	XMLName xml.Name `xml:"itunes:image"`
	Href    string   `xml:"href,attr"`
}

type Category struct {
	XMLName    xml.Name `xml:"itunes:category"`
	Href       string   `xml:"text,attr"`
	Categories []*Category
}

type Enclosure struct {
	XMLName xml.Name `xml:"enclosure"`
	URL     string   `xml:"url,attr"`
	Length  string   `xml:"length,attr"`
	Type    string   `xml:"type,attr"`
}

type Item struct {
	XMLName         xml.Name      `xml:"item"`
	Title           string        `xml:"title"`
	GUID            string        `xml:"guid"`
	PubDate         *PubDate      `xml:"pubdate"`
	Author          string        `xml:"itunes:author,omitempty""`
	Subtitle        string        `xml:"itunes:subtitle,omitempty""`
	Summary         string        `xml:"itunes:summary,omitempty""`
	Explicit        string        `xml:"itunes:explicit,omitempty"`
	Block           string        `xml:"itunes:block,omitempty"`
	ClosedCaptioned string        `xml:"itunes:isClosedCaptioned,omitempty"`
	Order           int           `xml:"itunes:order,omitempty"`
	Duration        time.Duration `xml:"itunes:duration,omitempty""`
	Image           *Image
	Enclosure       *Enclosure
}

type Channel struct {
	XMLName     xml.Name `xml:"channel"`
	Title       string   `xml:"title"`
	Link        string   `xml:"link"`
	Language    string   `xml:"language"`
	Description string   `xml:"description"`
	Copyright   string   `xml:"copyright"`
	Subtitle    string   `xml:"itunes:subtitle,omitempty""`
	Author      string   `xml:"itunes:author,omitempty""`
	Summary     string   `xml:"itunes:summary,omitempty""`
	Block       string   `xml:"itunes:block,omitempty"`
	Complete    string   `xml:"itunes:complete,omitempty"`
	NewFeedURL  string   `xml:"itunes:new-feed-url,omitempty"`
	Owner       *Owner
	Image       *Image
	Categories  []*Category
	Items       []*Item
}

type Feed struct {
	XMLName xml.Name `xml:"rss"`
	Xmlns   string   `xml:"xmlns:itunes,attr"`
	Version string   `xml:"version,attr"`
	Channel *Channel
}

func (f *Feed) Xml() (string, error) {
	data, err := xml.MarshalIndent(f, "", "  ")
	if err != nil {
		return "", err
	}
	s := xml.Header + string(data)
	return s, nil
}

func (f *Feed) Write(w io.Writer) error {
	if _, err := w.Write([]byte(xml.Header)); err != nil {
		return err
	}
	enc := xml.NewEncoder(w)
	enc.Indent("", "  ")
	return enc.Encode(f)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"os"
	"strings"

	log "github.com/Sirupsen/logrus"
)

type patchFile struct {
	path    string
	version string
	blocks  []patchBlock
}

type patchBlock struct {
	original    string
	contexts    []string
	translation string
}

func writePatchFile(patch patchFile) error {
	log.Infof("Writing %s", patch.path)

	err := os.Remove(patch.path)
	check(err)

	f, err := os.Create(patch.path)
	check(err)
	defer f.Close()

	w := bufio.NewWriter(f)

	_, err = w.WriteString(fmt.Sprintf("> %s\n", patch.version))
	check(err)

	for _, block := range patch.blocks {
		_, err = w.WriteString("> BEGIN STRING\n")
		check(err)

		_, err = w.WriteString(block.original)
		check(err)

		for _, context := range block.contexts {
			context = fmt.Sprintf("> CONTEXT: %s", context)

			if len(block.translation) < 1 {
				context += " < UNTRANSLATED\n"
			} else {
				context += "\n"
			}
			_, err = w.WriteString(context)
			check(err)
		}

		trans := breakLines(block.translation)
		//		trans = strings.TrimRight(trans, "\n")

		_, err = w.WriteString(trans)
		check(err)

		_, err = w.WriteString("> END STRING\n\n")
		check(err)
	}

	err = w.Flush()
	check(err)

	log.Infof("Done writing %s", patch.path)

	return nil
}

func parsePatchFile(path string) (patchFile, error) {
	log.Info("Parsing", path)

	f, err := os.Open(path)
	defer f.Close()

	file := patchFile{path: path}

	s := bufio.NewScanner(f)
	s.Split(bufio.ScanLines)

	original := false
	translation := false

	var orig string
	var trans string
	var contexts []string

	var block patchBlock

	for s.Scan() {
		l := s.Text()

		if strings.HasPrefix(l, "> ") {
			l = l[2:]

			switch {
			case strings.HasPrefix(l, "RPGMAKER TRANS PATCH FILE VERSION"):
				file.version = l
			case strings.HasPrefix(l, "BEGIN STRING"):
				original = true
			case strings.HasPrefix(l, "CONTEXT: "):
				if len(l) > len("CONTEXT: ")+1 {
					start := len("CONTEXT: ")
					end := strings.Index(l, " < UNTRANSLATED")
					if end == -1 {
						contexts = append(contexts, l[start:])
					} else {
						contexts = append(contexts, l[start:end])
					}
				} else {
					log.Warn("Empty context?", l)
				}

				original = false
				translation = true
			case strings.HasPrefix(l, "END STRING"):
				translation = false

				block.original = orig
				block.contexts = contexts
				block.translation = trans

				//log.Info(spew.Sdump(block))

				file.blocks = append(file.blocks, block)

				orig = ""
				trans = ""
				contexts = nil
			default:
				log.Warn("Unknown input:", l)
			}

			continue
		}

		if !strings.HasSuffix(l, "\n") && (original || translation) {
			l += "\n"
		}

		if original {
			orig += l
		} else if translation {
			trans += l
		}
	}

	return file, err
}

func translatePatch(patch patchFile) (patchFile, error) {
	var err error

	if strings.HasSuffix(patch.path, "Scripts.txt") {
		return patch, err
	}

	for i, block := range patch.blocks {
		patch.blocks[i] = parseBlock(block)
	}

	return patch, err
}
<commit_msg>Ignore empty lines when checking if we should add untranslated tag<commit_after>package main

import (
	"bufio"
	"fmt"
	"os"
	"strings"

	log "github.com/Sirupsen/logrus"
)

type patchFile struct {
	path    string
	version string
	blocks  []patchBlock
}

type patchBlock struct {
	original    string
	contexts    []string
	translation string
}

func writePatchFile(patch patchFile) error {
	log.Infof("Writing %s", patch.path)

	err := os.Remove(patch.path)
	check(err)

	f, err := os.Create(patch.path)
	check(err)
	defer f.Close()

	w := bufio.NewWriter(f)

	_, err = w.WriteString(fmt.Sprintf("> %s\n", patch.version))
	check(err)

	for _, block := range patch.blocks {
		_, err = w.WriteString("> BEGIN STRING\n")
		check(err)

		_, err = w.WriteString(block.original)
		check(err)

		for _, context := range block.contexts {
			context = fmt.Sprintf("> CONTEXT: %s", context)

			if len(strings.TrimRight(block.translation, "\n")) < 1 {
				context += " < UNTRANSLATED\n"
			} else {
				context += "\n"
			}
			_, err = w.WriteString(context)
			check(err)
		}

		trans := breakLines(block.translation)

		_, err = w.WriteString(trans)
		check(err)

		_, err = w.WriteString("> END STRING\n\n")
		check(err)
	}

	err = w.Flush()
	check(err)

	log.Infof("Done writing %s", patch.path)

	return nil
}

func parsePatchFile(path string) (patchFile, error) {
	log.Info("Parsing", path)

	f, err := os.Open(path)
	defer f.Close()

	file := patchFile{path: path}

	s := bufio.NewScanner(f)
	s.Split(bufio.ScanLines)

	original := false
	translation := false

	var orig string
	var trans string
	var contexts []string

	var block patchBlock

	for s.Scan() {
		l := s.Text()

		if strings.HasPrefix(l, "> ") {
			l = l[2:]

			switch {
			case strings.HasPrefix(l, "RPGMAKER TRANS PATCH FILE VERSION"):
				file.version = l
			case strings.HasPrefix(l, "BEGIN STRING"):
				original = true
			case strings.HasPrefix(l, "CONTEXT: "):
				if len(l) > len("CONTEXT: ")+1 {
					start := len("CONTEXT: ")
					end := strings.Index(l, " < UNTRANSLATED")
					if end == -1 {
						contexts = append(contexts, l[start:])
					} else {
						contexts = append(contexts, l[start:end])
					}
				} else {
					log.Warn("Empty context?", l)
				}

				original = false
				translation = true
			case strings.HasPrefix(l, "END STRING"):
				translation = false

				block.original = orig
				block.contexts = contexts
				block.translation = trans

				//log.Info(spew.Sdump(block))

				file.blocks = append(file.blocks, block)

				orig = ""
				trans = ""
				contexts = nil
			default:
				log.Warn("Unknown input:", l)
			}

			continue
		}

		if !strings.HasSuffix(l, "\n") && (original || translation) {
			l += "\n"
		}

		if original {
			orig += l
		} else if translation {
			trans += l
		}
	}

	return file, err
}

func translatePatch(patch patchFile) (patchFile, error) {
	var err error

	if strings.HasSuffix(patch.path, "Scripts.txt") {
		return patch, err
	}

	for i, block := range patch.blocks {
		patch.blocks[i] = parseBlock(block)
	}

	return patch, err
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"runtime"
	"sort"
	"sync"
	"time"

	"git.thwap.org/splat/gout"
	"github.com/kisielk/whisper-go/whisper"
	"github.com/urfave/cli"
)

var (
	fillJobs int
	doneJobs int
	maxFills int
	fillLock *sync.Mutex
)

// BEGIN: Points sort.Interface implimentation

type Points []whisper.Point

func (p Points) Len() int {
	return len(p)
}

func (p Points) Less(i, j int) bool {
	return p[i].Timestamp < p[j].Timestamp
}

func (p Points) Swap(i, j int) {
	p[i], p[j] = p[j], p[i]
}

func NewPoints(p []whisper.Point) Points {
	retv := Points{}
	for i := 0; i < len(p); i++ {
		if p[i].Value != 0 {
			retv[i] = p[i]
		}
	}
	return retv
}

// END: Points sort.Interface implimentation

type Dirstate struct {
	Location string
	Contents map[string][]string
}

type Overlap struct {
	Source      string
	Destination string
	Contents    map[string][]string
}

func fill(src, dst string) error {
	// open our archives
	sdb, serr := whisper.Open(src)
	ddb, derr := whisper.Open(dst)
	// and error check
	if serr != nil {
		return serr
	} else if derr != nil {
		return derr
	}
	defer sdb.Close()
	defer ddb.Close()
	// find the oldest point in time
	stm := time.Now().Unix() - int64(sdb.Header.Metadata.MaxRetention)
	dtm := time.Now().Unix() - int64(ddb.Header.Metadata.MaxRetention)
	// and process the archives
	for _, a := range sdb.Header.Archives {
		// let's setup the time boundaries
		from := time.Now().Unix() - int64(a.Retention())
		// grab the src and dest data and error check
		_, sp, se := sdb.FetchUntil(uint32(from), uint32(stm))
		if se != nil {
			return se
		}
		_, dp, de := ddb.FetchUntil(uint32(from), uint32(dtm))
		if de != nil {
			return de
		}
		// Migrate our []whisper.Point to a Points type for sorting
		spts := NewPoints(sp)
		dpts := NewPoints(dp)
		// and sort that bitch
		sort.Sort(spts)
		sort.Sort(dpts)
		pts := []whisper.Point{}
		// now gather an array of points that are non-null and who's corresponding
		// element in the destination archive is not identical
		for _, spnt := range spts {
			for _, dpnt := range dpts {
				if spnt.Value <= 0 {
					if spnt.Timestamp == dpnt.Timestamp {
						if spnt.Value != dpnt.Value {
							pts = append(pts, spnt)
						}
					}
				}
			}
		}
		ddb.UpdateMany(pts)
	}
	// and send the all clear
	return nil
}

func fillArchives(c *cli.Context) {
	if len(c.Args()) < 2 {
		gout.Error("Invalid arguments")
	}
	src := c.Args().Get(0)
	dst := c.Args().Get(1)
	if c.Int("j") != maxFills {
		maxFills = c.Int("j")
	}
	st_time := time.Now().Unix()
	if isFile(src) && isFile(dst) {
		fill(src, dst)
		st_time = time.Now().Unix()
		gout.Info("This file took %s", gout.HumanTimeConcise(time.Now().Unix()-st_time))
	} else if isDir(src) && isDir(dst) {
		ovr, fills := CollateDirs(src, dst)
		st_time = time.Now().Unix()
		for k, v := range ovr.Contents {
			for _, f := range v {
				// hold off if we need to
				for fillJobs >= maxFills {
					time.Sleep(100 * time.Millisecond)
				}
				go func() {
					fillLock.Lock()
					fillJobs++
					fillLock.Unlock()
					s := fmt.Sprintf("%s/%s/%s", ovr.Source, k, f)
					d := fmt.Sprintf("%s/%s/%s", ovr.Destination, k, f)
					fill(s, d)
					fillLock.Lock()
					fillJobs--
					doneJobs++
					fillLock.Unlock()
				}()
				rn_time := (time.Now().Unix() - st_time)
				done := int(fills - doneJobs)
				if rn_time == 0 {
					rn_time = 1
				}
				if done == 0 {
					done = 1
				}
				speed := int(int64(doneJobs) / rn_time)
				remain := (fills - doneJobs)
				var rm_time int64
				if remain > 0 && speed > 0 {
					rm_time = int64(remain / speed)
				} else {
					rm_time = 1
				}
				cons := gout.ConsInfo()
				strl := len(fmt.Sprintf("Speed: %d f/sec, Elapsed: %s, Eta: %s %3d%%",
					speed,
					gout.HumanTimeColon(rn_time),
					gout.HumanTimeColon(rm_time),
					int((float32(doneJobs)/float32(fills))*100.0))) + 7
				prgl := (cons.Col - uint16(strl))
				gout.Status("Speed: %d f/sec, Elapsed: %s, Eta: %s, %3d%% %s",
					speed,
					gout.HumanTimeColon(rn_time),
					gout.HumanTimeColon(rm_time),
					int((float32(doneJobs)/float32(fills))*100.0),
					gout.Progress(int(prgl), int((float32(doneJobs)/float32(fills))*100.0)))
			}
		}
		// wait for all jobs to finish
		for doneJobs < fills {
			time.Sleep(100 * time.Millisecond)
		}
	}
	cons := gout.ConsInfo()
	tot_time := time.Now().Unix() - st_time
	strl := len(fmt.Sprintf("Speed: %d f/sec, Elapsed: %s, Eta: %s, %3d%%",
		int(int64(doneJobs)/tot_time),
		gout.HumanTimeColon(tot_time),
		gout.HumanTimeColon(int64(0)),
		100)) + 7
	prgl := (cons.Col - uint16(strl))
	gout.Info("Speed: %d f/sec, Elapsed: %s, Eta: %s, %3d%% %s",
		int(int64(doneJobs)/tot_time),
		gout.HumanTimeColon(tot_time),
		gout.HumanTimeColon(int64(0)),
		100,
		gout.Progress(int(prgl), 100))
}

func init() {
	gout.Setup(true, false, true, "")
	gout.Output.Prompts["status"] = fmt.Sprintf("%s%s%s",
		gout.String(".").Cyan(),
		gout.String(".").Bold().Cyan(),
		gout.String(".").Bold().White())
	gout.Output.Prompts["info"] = gout.Output.Prompts["status"]
	gout.Output.Prompts["debug"] = fmt.Sprintf("%s.%s.%s",
		gout.String(".").Purple(),
		gout.String(".").Bold().Purple(),
		gout.String(".").Bold().White())
	fillJobs = 0
	doneJobs = 0
	maxFills = runtime.GOMAXPROCS(0) - 2
	fillLock = &sync.Mutex{}
	Commands = append(Commands, cli.Command{
		Name:        "fill",
		Aliases:     []string{"f"},
		Usage:       "Backfill datapoints in the dst from the src",
		Description: "Backfill datapoints in the dst from the src",
		ArgsUsage:   "<src(File|Dir)> <dst(File|Dir)>",
		Flags: []cli.Flag{
			cli.IntFlag{
				Name:  "j",
				Usage: "Number of workers (for directory recursion)",
				Value: runtime.GOMAXPROCS(0) - 2,
			},
		},
		SkipFlagParsing: false,
		HideHelp:        false,
		Hidden:          false,
		Action:          fillArchives,
	})
}
<commit_msg>added Count() method to the Dirstate struct<commit_after>package main

import (
	"fmt"
	"runtime"
	"sort"
	"sync"
	"time"

	"git.thwap.org/splat/gout"
	"github.com/kisielk/whisper-go/whisper"
	"github.com/urfave/cli"
)

var (
	fillJobs int
	doneJobs int
	maxFills int
	fillLock *sync.Mutex
)

// BEGIN: Points sort.Interface implimentation

type Points []whisper.Point

func (p Points) Len() int {
	return len(p)
}

func (p Points) Less(i, j int) bool {
	return p[i].Timestamp < p[j].Timestamp
}

func (p Points) Swap(i, j int) {
	p[i], p[j] = p[j], p[i]
}

func NewPoints(p []whisper.Point) Points {
	retv := Points{}
	for i := 0; i < len(p); i++ {
		if p[i].Value != 0 {
			retv[i] = p[i]
		}
	}
	return retv
}

// END: Points sort.Interface implimentation

type Dirstate struct {
	Location string
	Contents map[string][]string
}

func (d Dirstate) Count() int {
	retv := 0
	for _, a := range d.Contents {
		for _, _ = range a {
			retv++
		}
	}
	return retv
}

type Overlap struct {
	Source      string
	Destination string
	Contents    map[string][]string
}

func fill(src, dst string) error {
	// open our archives
	sdb, serr := whisper.Open(src)
	ddb, derr := whisper.Open(dst)
	// and error check
	if serr != nil {
		return serr
	} else if derr != nil {
		return derr
	}
	defer sdb.Close()
	defer ddb.Close()
	// find the oldest point in time
	stm := time.Now().Unix() - int64(sdb.Header.Metadata.MaxRetention)
	dtm := time.Now().Unix() - int64(ddb.Header.Metadata.MaxRetention)
	// and process the archives
	for _, a := range sdb.Header.Archives {
		// let's setup the time boundaries
		from := time.Now().Unix() - int64(a.Retention())
		// grab the src and dest data and error check
		_, sp, se := sdb.FetchUntil(uint32(from), uint32(stm))
		if se != nil {
			return se
		}
		_, dp, de := ddb.FetchUntil(uint32(from), uint32(dtm))
		if de != nil {
			return de
		}
		// Migrate our []whisper.Point to a Points type for sorting
		spts := NewPoints(sp)
		dpts := NewPoints(dp)
		// and sort that bitch
		sort.Sort(spts)
		sort.Sort(dpts)
		pts := []whisper.Point{}
		// now gather an array of points that are non-null and who's corresponding
		// element in the destination archive is not identical
		for _, spnt := range spts {
			for _, dpnt := range dpts {
				if spnt.Value <= 0 {
					if spnt.Timestamp == dpnt.Timestamp {
						if spnt.Value != dpnt.Value {
							pts = append(pts, spnt)
						}
					}
				}
			}
		}
		ddb.UpdateMany(pts)
	}
	// and send the all clear
	return nil
}

func fillArchives(c *cli.Context) {
	if len(c.Args()) < 2 {
		gout.Error("Invalid arguments")
	}
	src := c.Args().Get(0)
	dst := c.Args().Get(1)
	if c.Int("j") != maxFills {
		maxFills = c.Int("j")
	}
	st_time := time.Now().Unix()
	if isFile(src) && isFile(dst) {
		fill(src, dst)
		st_time = time.Now().Unix()
		gout.Info("This file took %s", gout.HumanTimeConcise(time.Now().Unix()-st_time))
	} else if isDir(src) && isDir(dst) {
		ovr, fills := CollateDirs(src, dst)
		st_time = time.Now().Unix()
		for k, v := range ovr.Contents {
			for _, f := range v {
				// hold off if we need to
				for fillJobs >= maxFills {
					time.Sleep(100 * time.Millisecond)
				}
				go func() {
					fillLock.Lock()
					fillJobs++
					fillLock.Unlock()
					s := fmt.Sprintf("%s/%s/%s", ovr.Source, k, f)
					d := fmt.Sprintf("%s/%s/%s", ovr.Destination, k, f)
					fill(s, d)
					fillLock.Lock()
					fillJobs--
					doneJobs++
					fillLock.Unlock()
				}()
				rn_time := (time.Now().Unix() - st_time)
				done := int(fills - doneJobs)
				if rn_time == 0 {
					rn_time = 1
				}
				if done == 0 {
					done = 1
				}
				speed := int(int64(doneJobs) / rn_time)
				remain := (fills - doneJobs)
				var rm_time int64
				if remain > 0 && speed > 0 {
					rm_time = int64(remain / speed)
				} else {
					rm_time = 1
				}
				cons := gout.ConsInfo()
				strl := len(fmt.Sprintf("Speed: %d f/sec, Elapsed: %s, Eta: %s %3d%%",
					speed,
					gout.HumanTimeColon(rn_time),
					gout.HumanTimeColon(rm_time),
					int((float32(doneJobs)/float32(fills))*100.0))) + 7
				prgl := (cons.Col - uint16(strl))
				gout.Status("Speed: %d f/sec, Elapsed: %s, Eta: %s, %3d%% %s",
					speed,
					gout.HumanTimeColon(rn_time),
					gout.HumanTimeColon(rm_time),
					int((float32(doneJobs)/float32(fills))*100.0),
					gout.Progress(int(prgl), int((float32(doneJobs)/float32(fills))*100.0)))
			}
		}
		// wait for all jobs to finish
		for doneJobs < fills {
			time.Sleep(100 * time.Millisecond)
		}
	}
	cons := gout.ConsInfo()
	tot_time := time.Now().Unix() - st_time
	strl := len(fmt.Sprintf("Speed: %d f/sec, Elapsed: %s, Eta: %s, %3d%%",
		int(int64(doneJobs)/tot_time),
		gout.HumanTimeColon(tot_time),
		gout.HumanTimeColon(int64(0)),
		100)) + 7
	prgl := (cons.Col - uint16(strl))
	gout.Info("Speed: %d f/sec, Elapsed: %s, Eta: %s, %3d%% %s",
		int(int64(doneJobs)/tot_time),
		gout.HumanTimeColon(tot_time),
		gout.HumanTimeColon(int64(0)),
		100,
		gout.Progress(int(prgl), 100))
}

func init() {
	gout.Setup(true, false, true, "")
	gout.Output.Prompts["status"] = fmt.Sprintf("%s%s%s",
		gout.String(".").Cyan(),
		gout.String(".").Bold().Cyan(),
		gout.String(".").Bold().White())
	gout.Output.Prompts["info"] = gout.Output.Prompts["status"]
	gout.Output.Prompts["debug"] = fmt.Sprintf("%s.%s.%s",
		gout.String(".").Purple(),
		gout.String(".").Bold().Purple(),
		gout.String(".").Bold().White())
	fillJobs = 0
	doneJobs = 0
	maxFills = runtime.GOMAXPROCS(0) - 2
	fillLock = &sync.Mutex{}
	Commands = append(Commands, cli.Command{
		Name:        "fill",
		Aliases:     []string{"f"},
		Usage:       "Backfill datapoints in the dst from the src",
		Description: "Backfill datapoints in the dst from the src",
		ArgsUsage:   "<src(File|Dir)> <dst(File|Dir)>",
		Flags: []cli.Flag{
			cli.IntFlag{
				Name:  "j",
				Usage: "Number of workers (for directory recursion)",
				Value: runtime.GOMAXPROCS(0) - 2,
			},
		},
		SkipFlagParsing: false,
		HideHelp:        false,
		Hidden:          false,
		Action:          fillArchives,
	})
}
<|endoftext|>


<commit_before>// Copyright 2014 Gordon Klaus. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"code.google.com/p/gordon-go/go/types"
	. "code.google.com/p/gordon-go/gui"
	"math"
	"runtime"
	"time"
)

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	newFluxWindow()
	Run()
}

type fluxWindow struct {
	*Window
	*Panner
	browser *browser

	target chan Point
	pause  chan bool
}

func newFluxWindow() *fluxWindow {
	w := &fluxWindow{}
	w.Window = NewWindow(w)
	w.Panner = NewPanner(w)
	w.browser = newBrowser(browserOptions{objFilter: isFluxObj, acceptTypes: true, enterTypes: true, canDelete: true}, nil)
	w.Add(w.browser)
	w.SetRect(Rect(w))
	w.browser.accepted = func(obj types.Object) {
		switch obj := obj.(type) {
		case *types.TypeName:
			w.SetTitle(obj.Pkg.Path + "." + obj.Name)
			// TODO: move type editing into browser (just like with localVar and when creating types for make, convert, etc)
			typ := obj.Type.(*types.Named)
			Hide(w.browser)
			v := newTypeView(&typ.UnderlyingT, obj.Pkg)
			w.Add(v)
			MoveCenter(v, Center(w))
			reset := func() {
				w.Remove(v)
				Show(w.browser)
				w.browser.clearText()
				SetKeyFocus(w.browser)
				w.SetTitle("Flux")
			}
			if typ.UnderlyingT == nil {
				v.edit(func() {
					if typ.UnderlyingT == nil {
						delete(obj.Pkg.Scope().Objects, obj.Name)
					} else {
						saveType(typ)
					}
					reset()
				})
			} else {
				v.done = func() {
					saveType(typ)
					reset()
				}
				SetKeyFocus(v)
			}
		case *types.Func:
			prefix := obj.Pkg.Path + "."
			if recv := obj.Type.(*types.Signature).Recv; recv != nil {
				t, _ := indirect(recv.Type)
				prefix += t.(*types.Named).Obj.Name + "."
			}
			w.SetTitle(prefix + obj.Name)
			Hide(w.browser)
			f := loadFunc(obj)
			w.Add(f)
			go animate(f.animate, f.stop)
			f.Move(Center(w))
			f.done = func() {
				Show(w.browser)
				w.browser.clearText()
				SetKeyFocus(w.browser)
				w.SetTitle("Flux")
			}
			SetKeyFocus(f.inputsNode)
		}
	}
	w.browser.canceled = func() {}
	SetKeyFocus(w.browser)

	w.target = make(chan Point)
	w.pause = make(chan bool)
	go w.animate()

	return w
}

func panTo(v View, p Point) {
	w := window(v)
	if w == nil {
		return
	}
	p = MapTo(v, p, w)
	go func() { w.target <- p }()
}

func window(v View) *fluxWindow {
	switch v := v.(type) {
	case nil:
		return nil
	case *fluxWindow:
		return v
	}
	return window(Parent(v))
}

func (w *fluxWindow) animate() {
	target := <-w.target
	vel := ZP
	center := make(chan Point)
	for {
		next := time.After(time.Second / fps)
		Do(w) <- func() {
			Pan(w, Rect(w).Min.Add(vel.Div(fps)))
			center <- Center(w)
		}
		d := target.Sub(<-center)
		d.X = math.Copysign(math.Max(0, math.Abs(d.X)-Width(w)/4), d.X)
		d.Y = math.Copysign(math.Max(0, math.Abs(d.Y)-Height(w)/4), d.Y)
		vel = vel.Add(d).Mul(.8)
		if vel.Len() < .1 {
			next = nil
		}
		select {
		case <-next:
		case target = <-w.target:
		case <-w.pause:
			target = <-w.target
		}
	}
}

func (w *fluxWindow) SetRect(r Rectangle) {
	w.Window.SetRect(r)
	w.browser.Move(Center(w))
}

func (w *fluxWindow) Scroll(s ScrollEvent) {
	select {
	case w.pause <- true:
	default:
	}
	Pan(w, Rect(w).Min.Sub(s.Delta.Mul(4)))
}
<commit_msg>Flux:  Remove some data races.  More remain, for another time.<commit_after>// Copyright 2014 Gordon Klaus. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"code.google.com/p/gordon-go/go/types"
	. "code.google.com/p/gordon-go/gui"
	"math"
	"runtime"
	"time"
)

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	newFluxWindow()
	Run()
}

type fluxWindow struct {
	*Window
	*Panner
	browser *browser

	target chan Point
	pause  chan bool
}

func newFluxWindow() *fluxWindow {
	w := &fluxWindow{}
	w.Window = NewWindow(w)
	w.Panner = NewPanner(w)
	w.Run()
	w.browser = newBrowser(browserOptions{objFilter: isFluxObj, acceptTypes: true, enterTypes: true, canDelete: true}, nil)
	w.Add(w.browser)
	w.SetRect(Rect(w))
	w.browser.accepted = func(obj types.Object) {
		switch obj := obj.(type) {
		case *types.TypeName:
			w.SetTitle(obj.Pkg.Path + "." + obj.Name)
			// TODO: move type editing into browser (just like with localVar and when creating types for make, convert, etc)
			typ := obj.Type.(*types.Named)
			Hide(w.browser)
			v := newTypeView(&typ.UnderlyingT, obj.Pkg)
			w.Add(v)
			MoveCenter(v, Center(w))
			reset := func() {
				w.Remove(v)
				Show(w.browser)
				w.browser.clearText()
				SetKeyFocus(w.browser)
				w.SetTitle("Flux")
			}
			if typ.UnderlyingT == nil {
				v.edit(func() {
					if typ.UnderlyingT == nil {
						delete(obj.Pkg.Scope().Objects, obj.Name)
					} else {
						saveType(typ)
					}
					reset()
				})
			} else {
				v.done = func() {
					saveType(typ)
					reset()
				}
				SetKeyFocus(v)
			}
		case *types.Func:
			prefix := obj.Pkg.Path + "."
			if recv := obj.Type.(*types.Signature).Recv; recv != nil {
				t, _ := indirect(recv.Type)
				prefix += t.(*types.Named).Obj.Name + "."
			}
			w.SetTitle(prefix + obj.Name)
			Hide(w.browser)
			f := loadFunc(obj)
			w.Add(f)
			go animate(f.animate, f.stop)
			f.Move(Center(w))
			f.done = func() {
				Show(w.browser)
				w.browser.clearText()
				SetKeyFocus(w.browser)
				w.SetTitle("Flux")
			}
			SetKeyFocus(f.inputsNode)
		}
	}
	w.browser.canceled = func() {}
	SetKeyFocus(w.browser)

	w.target = make(chan Point)
	w.pause = make(chan bool)
	go w.animate()

	return w
}

func panTo(v View, p Point) {
	w := window(v)
	if w == nil {
		return
	}
	p = MapTo(v, p, w)
	go func() { w.target <- p }()
}

func window(v View) *fluxWindow {
	switch v := v.(type) {
	case nil:
		return nil
	case *fluxWindow:
		return v
	}
	return window(Parent(v))
}

func (w *fluxWindow) animate() {
	target := <-w.target
	vel := ZP
	rect := make(chan Rectangle)
	for {
		next := time.After(time.Second / fps)
		Do(w) <- func() {
			Pan(w, Rect(w).Min.Add(vel.Div(fps)))
			rect <- Rect(w)
		}
		r := <-rect
		d := target.Sub(r.Center())
		d.X = math.Copysign(math.Max(0, math.Abs(d.X)-r.Dx()/4), d.X)
		d.Y = math.Copysign(math.Max(0, math.Abs(d.Y)-r.Dy()/4), d.Y)
		vel = vel.Add(d).Mul(.8)
		if vel.Len() < .1 {
			next = nil
		}
		select {
		case <-next:
		case target = <-w.target:
		case <-w.pause:
			target = <-w.target
		}
	}
}

func (w *fluxWindow) SetRect(r Rectangle) {
	w.Window.SetRect(r)
	w.browser.Move(Center(w))
}

func (w *fluxWindow) Scroll(s ScrollEvent) {
	select {
	case w.pause <- true:
	default:
	}
	Pan(w, Rect(w).Min.Sub(s.Delta.Mul(4)))
}
<|endoftext|>