<commit_before>package ops

import (
	"errors"
	"github.com/phzfi/RIC/server/images"
	"github.com/phzfi/RIC/server/logging"
	"os"
	"path/filepath"
	"strings"
	"sync"
)

type dim [2]int
type idToSize map[string]dim

type ImageSource struct {
	roots []string
	sizes idToSize
	mutex *sync.RWMutex
}

func MakeImageSource() ImageSource {
	return ImageSource{
		sizes: make(idToSize),
		mutex: new(sync.RWMutex),
	}
}

func (i ImageSource) LoadImageOp(id string) Operation {
	return loadImageOp{&i, id}
}

// Search root for an image. Returned image should be destroyed by image.Destroy, image.Resized or image.ToBlob or other.
func (i ImageSource) searchRoots(filename string, img images.Image) (err error) {
	if len(i.roots) == 0 {
		logging.Debug("No roots")
		err = os.ErrNotExist
		return
	}
	// Extract requested type/extension and id from filename
	ext := strings.TrimLeft(filepath.Ext(filename), ".")
	id := strings.TrimRight(filename[0:len(filename)-len(ext)], ".")
	// Search requested image from all roots by trial and error
	for _, root := range i.roots {
		// TODO: Fix escape vulnerability (sanitize filename from at least ".." etc)
		// Assume image is stored as .jpg -> change extension to .jpg
		trial := filepath.Join(root, id) + ".jpg"
		err = img.FromFile(trial)
		if err == nil {
			logging.Debug("Found: " + trial)
			break
		}
		logging.Debug("Not found: " + trial)
	}
	return
}

// TODO: This is a temp solution for ImageSize creating too many Images.
// Limit to creating only one at time for finding the image size

func (i ImageSource) ImageSize(fn string) (w int, h int, err error) {
	i.mutex.RLock()
	s, ok := i.sizes[fn]
	i.mutex.RUnlock()

	if ok {
		return s[0], s[1], nil
	}

	image := images.NewImage()
	defer image.Destroy()

	err = i.searchRoots(fn, image)
	if err != nil {
		return
	}

	w = image.GetWidth()
	h = image.GetHeight()

	i.mutex.Lock()
	i.sizes[fn] = dim{w, h}
	i.mutex.Unlock()

	return
}

// A very trivial (and inefficient way to handle roots)
// Can be used for development work, however.
func (i *ImageSource) AddRoot(root string) error {
	abspath, err := filepath.Abs(root)
	if err != nil {
		return err
	}

	logging.Debug("Adding root: " + root + " -> " + abspath)
	for _, path := range i.roots {
		if path == abspath {
			return errors.New("Root is already served")
		}
	}

	i.roots = append(i.roots, abspath)
	return nil
}

// A very trivial (and inefficient way to handle roots)
// Can be used for development work, however.
func (is *ImageSource) RemoveRoot(root string) error {
	abspath, err := filepath.Abs(root)
	if err != nil {
		return err
	}

	for i, path := range is.roots {
		if path == abspath {
			is.roots = append(is.roots[:i], is.roots[i+1:]...)
			return nil
		}
	}

	return errors.New("Root not found")
}
<commit_msg>ImageSize no longer reads the whole file<commit_after>package ops

import (
	"errors"
	"github.com/phzfi/RIC/server/images"
	"github.com/phzfi/RIC/server/logging"
	"os"
	"path/filepath"
	"strings"
	"sync"
)

type dim [2]int
type idToSize map[string]dim

type ImageSource struct {
	roots []string
	sizes idToSize
	mutex *sync.RWMutex
}

func MakeImageSource() ImageSource {
	return ImageSource{
		sizes: make(idToSize),
		mutex: new(sync.RWMutex),
	}
}

func (i ImageSource) LoadImageOp(id string) Operation {
	return loadImageOp{&i, id}
}

// Searches root for an image. If found loads the image to img. Otherwise does nothing and returns an error.
func (i ImageSource) searchRoots(fn string, img images.Image) (err error) {
	return i.searchRootsCustomTrialFunc(fn, img.FromFile)
}

// Searches root for an image. Calls the given trialFunc with the given fn for every root until trialFunc does not return an error. Returns if trialFunc succeeds. returns with error if no trialFunc succeeds.
func (i ImageSource) searchRootsCustomTrialFunc(fn string, trialFunc func (fn string) (err error)) (err error) {
	if len(i.roots) == 0 {
		logging.Debug("No roots")
		err = os.ErrNotExist
		return
	}
	// Extract requested type/extension and id from filename
	ext := strings.TrimLeft(filepath.Ext(fn), ".")
	id := strings.TrimRight(fn[0:len(fn)-len(ext)], ".")
	// Search requested image from all roots by trial and error
	for _, root := range i.roots {
		// TODO: Fix escape vulnerability (sanitize filename from at least ".." etc)
		// Assume image is stored as .jpg -> change extension to .jpg
		trial := filepath.Join(root, id) + ".jpg"
		err = trialFunc(trial)
		if err == nil {
			logging.Debug("Found: " + trial)
			break
		}
		logging.Debug("Not found: " + trial)
	}
	return
}

// Get image size
func (i ImageSource) ImageSize(fn string) (w int, h int, err error) {
	i.mutex.RLock()
	s, ok := i.sizes[fn]
	i.mutex.RUnlock()

	if ok {
		return s[0], s[1], nil
	}

	image := images.NewImage()
	defer image.Destroy()

	err = i.pingRoots(fn, image)
	if err != nil {
		return
	}

	w = image.GetWidth()
	h = image.GetHeight()

	i.mutex.Lock()
	i.sizes[fn] = dim{w, h}
	i.mutex.Unlock()

	return
}

// Searches root for an image. If found, loads only the image metadata to img. Otherwise does nothing and returns an error.
func (i ImageSource) pingRoots(fn string, img images.Image) (err error) {
	return i.searchRootsCustomTrialFunc(fn, img.PingImage)
}

// A very trivial (and inefficient way to handle roots)
// Can be used for development work, however.
func (i *ImageSource) AddRoot(root string) error {
	abspath, err := filepath.Abs(root)
	if err != nil {
		return err
	}

	logging.Debug("Adding root: " + root + " -> " + abspath)
	for _, path := range i.roots {
		if path == abspath {
			return errors.New("Root is already served")
		}
	}

	i.roots = append(i.roots, abspath)
	return nil
}

// A very trivial (and inefficient way to handle roots)
// Can be used for development work, however.
func (is *ImageSource) RemoveRoot(root string) error {
	abspath, err := filepath.Abs(root)
	if err != nil {
		return err
	}

	for i, path := range is.roots {
		if path == abspath {
			is.roots = append(is.roots[:i], is.roots[i+1:]...)
			return nil
		}
	}

	return errors.New("Root not found")
}
<|endoftext|>


<commit_before>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package http

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"strings"
	"testing"
	"time"

	"github.com/google/cloudprober/logger"
	"github.com/google/cloudprober/metrics"
	"github.com/google/cloudprober/targets/lameduck"
	"google3/go/context/context"
)

const testExportInterval = 2 * time.Second

type fakeLameduckLister struct {
	lameducked []string
	err        error
}

func (f *fakeLameduckLister) List() ([]string, error) {
	return f.lameducked, f.err
}

func TestMain(m *testing.M) {
	os.Exit(m.Run())
}

func testServer(ctx context.Context, t *testing.T, insName string, ldLister lameduck.Lister) (*Server, chan *metrics.EventMetrics) {
	ln, err := net.Listen("tcp", ":0")
	if err != nil {
		t.Fatalf("Listen error: %v.", err)
	}

	dataChan := make(chan *metrics.EventMetrics, 10)
	s := &Server{
		l:             &logger.Logger{},
		ln:            ln,
		statsInterval: 2 * time.Second,
		instanceName:  insName,
		ldLister:      ldLister,
		staticURLResTable: map[string]string{
			"/":         OK,
			"/instance": insName,
		},
	}

	go func() {
		s.Start(ctx, dataChan)
	}()

	return s, dataChan
}

// get preforms HTTP GET request and return the response body and status
func get(t *testing.T, ln net.Listener, path string) (string, string) {
	t.Helper()
	resp, err := http.Get(fmt.Sprintf("http://%s/%s", listenerAddr(ln), path))
	if err != nil {
		t.Errorf("HTTP server returned an error for the URL '/%s'. Err: %v", path, err)
		return "", ""
	}
	status := resp.Status
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		t.Errorf("Error while reading response for the URL '/%s': Err: %v", path, err)
		return "", status
	}
	return string(body), status
}

func listenerAddr(ln net.Listener) string {
	return fmt.Sprintf("localhost:%d", ln.Addr().(*net.TCPAddr).Port)
}

func TestListenAndServeStats(t *testing.T) {
	testIns := "testInstance"
	ctx, cancelFunc := context.WithCancel(context.Background())
	s, dataChan := testServer(ctx, t, testIns, &fakeLameduckLister{})
	defer cancelFunc()

	urlsAndExpectedResponse := map[string]string{
		"/":            OK,
		"/instance":    "testInstance",
		"/lameduck":    "false",
		"/healthcheck": OK,
	}
	for url, expectedResponse := range urlsAndExpectedResponse {
		if response, _ := get(t, s.ln, url); response != expectedResponse {
			t.Errorf("Didn't get the expected response for URL '%s'. Got: %s, Expected: %s", url, response, expectedResponse)
		}
	}
	// Sleep for the export interval and a second extra to allow for the stats to
	// come in.
	time.Sleep(s.statsInterval)
	time.Sleep(time.Second)

	// Build a map of expected URL stats
	expectedURLStats := make(map[string]int64)
	for url := range urlsAndExpectedResponse {
		expectedURLStats[url]++
	}
	if len(dataChan) != 1 {
		t.Errorf("Wrong number of stats on the stats channel. Got: %d, Expected: %d", len(dataChan), 1)
	}
	em := <-dataChan

	// See if we got stats for the all URLs
	for url, expectedCount := range expectedURLStats {
		count := em.Metric("req").(*metrics.Map).GetKey(url).Int64()
		if count != expectedCount {
			t.Errorf("Didn't get the expected stats for the URL: %s. Got: %d, Expected: %d", url, count, expectedCount)
		}
	}
}

func TestLameduckingTestInstance(t *testing.T) {
	ctx, cancelFunc := context.WithCancel(context.Background())
	s, _ := testServer(ctx, t, "testInstance", &fakeLameduckLister{})
	defer cancelFunc()

	if resp, _ := get(t, s.ln, "lameduck"); !strings.Contains(resp, "false") {
		t.Errorf("Didn't get the expected response for the URL '/lameduck'. got: %q, want it to contain: %q", resp, "false")
	}
	if resp, status := get(t, s.ln, "healthcheck"); resp != OK || status != "200 OK" {
		t.Errorf("Didn't get the expected response for the URL '/healthcheck'. got: %q, %q , want: %q, %q", resp, status, OK, "200 OK")
	}

	s.ldLister = &fakeLameduckLister{[]string{"testInstance"}, nil}

	if resp, _ := get(t, s.ln, "lameduck"); !strings.Contains(resp, "true") {
		t.Errorf("Didn't get the expected response for the URL '/lameduck'. got: %q, want it to contain: %q", resp, "true")
	}
	if _, status := get(t, s.ln, "healthcheck"); status != "503 Service Unavailable" {
		t.Errorf("Didn't get the expected response for the URL '/healthcheck'. got: %q , want: %q", status, "200 OK")
	}
}

func TestLameduckListerNil(t *testing.T) {
	expectedErrMsg := "not initialized"

	ctx, cancelFunc := context.WithCancel(context.Background())
	s, _ := testServer(ctx, t, "testInstance", nil)
	defer cancelFunc()

	if resp, status := get(t, s.ln, "lameduck"); !strings.Contains(resp, expectedErrMsg) || status != "200 OK" {
		t.Errorf("Didn't get the expected response for the URL '/lameduck'. got: %q, %q. want it to contain: %q, %q", resp, status, expectedErrMsg, "200 OK")
	}
	if resp, status := get(t, s.ln, "healthcheck"); resp != OK || status != "200 OK" {
		t.Errorf("Didn't get the expected response for the URL '/healthcheck'. got: %q, %q , want: %q, %q", resp, status, OK, "200 OK")
	}
}

func TestErrorToGetLameduckList(t *testing.T) {
	expectedErrMsg := "fake lameduck error message"

	ctx, cancelFunc := context.WithCancel(context.Background())
	s, _ := testServer(ctx, t, "testInstance", &fakeLameduckLister{nil, errors.New(expectedErrMsg)})
	defer cancelFunc()

	if resp, status := get(t, s.ln, "lameduck"); !strings.Contains(resp, expectedErrMsg) || status != "200 OK" {
		t.Errorf("Didn't get the expected response for the URL '/lameduck'. got: %q, %q. want it to contain: %q, %q", resp, status, expectedErrMsg, "200 OK")
	}
	if resp, status := get(t, s.ln, "healthcheck"); resp != OK || status != "200 OK" {
		t.Errorf("Didn't get the expected response for the URL '/healthcheck'. got: %q, %q , want: %q, %q", resp, status, OK, "200 OK")
	}
}
<commit_msg>Use standard context package in http_test.<commit_after>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package http

import (
	"context"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"strings"
	"testing"
	"time"

	"github.com/google/cloudprober/logger"
	"github.com/google/cloudprober/metrics"
	"github.com/google/cloudprober/targets/lameduck"
)

const testExportInterval = 2 * time.Second

type fakeLameduckLister struct {
	lameducked []string
	err        error
}

func (f *fakeLameduckLister) List() ([]string, error) {
	return f.lameducked, f.err
}

func TestMain(m *testing.M) {
	os.Exit(m.Run())
}

func testServer(ctx context.Context, t *testing.T, insName string, ldLister lameduck.Lister) (*Server, chan *metrics.EventMetrics) {
	ln, err := net.Listen("tcp", ":0")
	if err != nil {
		t.Fatalf("Listen error: %v.", err)
	}

	dataChan := make(chan *metrics.EventMetrics, 10)
	s := &Server{
		l:             &logger.Logger{},
		ln:            ln,
		statsInterval: 2 * time.Second,
		instanceName:  insName,
		ldLister:      ldLister,
		staticURLResTable: map[string]string{
			"/":         OK,
			"/instance": insName,
		},
	}

	go func() {
		s.Start(ctx, dataChan)
	}()

	return s, dataChan
}

// get preforms HTTP GET request and return the response body and status
func get(t *testing.T, ln net.Listener, path string) (string, string) {
	t.Helper()
	resp, err := http.Get(fmt.Sprintf("http://%s/%s", listenerAddr(ln), path))
	if err != nil {
		t.Errorf("HTTP server returned an error for the URL '/%s'. Err: %v", path, err)
		return "", ""
	}
	status := resp.Status
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		t.Errorf("Error while reading response for the URL '/%s': Err: %v", path, err)
		return "", status
	}
	return string(body), status
}

func listenerAddr(ln net.Listener) string {
	return fmt.Sprintf("localhost:%d", ln.Addr().(*net.TCPAddr).Port)
}

func TestListenAndServeStats(t *testing.T) {
	testIns := "testInstance"
	ctx, cancelFunc := context.WithCancel(context.Background())
	s, dataChan := testServer(ctx, t, testIns, &fakeLameduckLister{})
	defer cancelFunc()

	urlsAndExpectedResponse := map[string]string{
		"/":            OK,
		"/instance":    "testInstance",
		"/lameduck":    "false",
		"/healthcheck": OK,
	}
	for url, expectedResponse := range urlsAndExpectedResponse {
		if response, _ := get(t, s.ln, url); response != expectedResponse {
			t.Errorf("Didn't get the expected response for URL '%s'. Got: %s, Expected: %s", url, response, expectedResponse)
		}
	}
	// Sleep for the export interval and a second extra to allow for the stats to
	// come in.
	time.Sleep(s.statsInterval)
	time.Sleep(time.Second)

	// Build a map of expected URL stats
	expectedURLStats := make(map[string]int64)
	for url := range urlsAndExpectedResponse {
		expectedURLStats[url]++
	}
	if len(dataChan) != 1 {
		t.Errorf("Wrong number of stats on the stats channel. Got: %d, Expected: %d", len(dataChan), 1)
	}
	em := <-dataChan

	// See if we got stats for the all URLs
	for url, expectedCount := range expectedURLStats {
		count := em.Metric("req").(*metrics.Map).GetKey(url).Int64()
		if count != expectedCount {
			t.Errorf("Didn't get the expected stats for the URL: %s. Got: %d, Expected: %d", url, count, expectedCount)
		}
	}
}

func TestLameduckingTestInstance(t *testing.T) {
	ctx, cancelFunc := context.WithCancel(context.Background())
	s, _ := testServer(ctx, t, "testInstance", &fakeLameduckLister{})
	defer cancelFunc()

	if resp, _ := get(t, s.ln, "lameduck"); !strings.Contains(resp, "false") {
		t.Errorf("Didn't get the expected response for the URL '/lameduck'. got: %q, want it to contain: %q", resp, "false")
	}
	if resp, status := get(t, s.ln, "healthcheck"); resp != OK || status != "200 OK" {
		t.Errorf("Didn't get the expected response for the URL '/healthcheck'. got: %q, %q , want: %q, %q", resp, status, OK, "200 OK")
	}

	s.ldLister = &fakeLameduckLister{[]string{"testInstance"}, nil}

	if resp, _ := get(t, s.ln, "lameduck"); !strings.Contains(resp, "true") {
		t.Errorf("Didn't get the expected response for the URL '/lameduck'. got: %q, want it to contain: %q", resp, "true")
	}
	if _, status := get(t, s.ln, "healthcheck"); status != "503 Service Unavailable" {
		t.Errorf("Didn't get the expected response for the URL '/healthcheck'. got: %q , want: %q", status, "200 OK")
	}
}

func TestLameduckListerNil(t *testing.T) {
	expectedErrMsg := "not initialized"

	ctx, cancelFunc := context.WithCancel(context.Background())
	s, _ := testServer(ctx, t, "testInstance", nil)
	defer cancelFunc()

	if resp, status := get(t, s.ln, "lameduck"); !strings.Contains(resp, expectedErrMsg) || status != "200 OK" {
		t.Errorf("Didn't get the expected response for the URL '/lameduck'. got: %q, %q. want it to contain: %q, %q", resp, status, expectedErrMsg, "200 OK")
	}
	if resp, status := get(t, s.ln, "healthcheck"); resp != OK || status != "200 OK" {
		t.Errorf("Didn't get the expected response for the URL '/healthcheck'. got: %q, %q , want: %q, %q", resp, status, OK, "200 OK")
	}
}

func TestErrorToGetLameduckList(t *testing.T) {
	expectedErrMsg := "fake lameduck error message"

	ctx, cancelFunc := context.WithCancel(context.Background())
	s, _ := testServer(ctx, t, "testInstance", &fakeLameduckLister{nil, errors.New(expectedErrMsg)})
	defer cancelFunc()

	if resp, status := get(t, s.ln, "lameduck"); !strings.Contains(resp, expectedErrMsg) || status != "200 OK" {
		t.Errorf("Didn't get the expected response for the URL '/lameduck'. got: %q, %q. want it to contain: %q, %q", resp, status, expectedErrMsg, "200 OK")
	}
	if resp, status := get(t, s.ln, "healthcheck"); resp != OK || status != "200 OK" {
		t.Errorf("Didn't get the expected response for the URL '/healthcheck'. got: %q, %q , want: %q, %q", resp, status, OK, "200 OK")
	}
}
<|endoftext|>


<commit_before>package create

import (
	"encoding/json"
	"fmt"
	"sync"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	awssession "github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/aws/aws-sdk-go/service/kms"
	"github.com/giantswarm/awstpr"
	awsinfo "github.com/giantswarm/awstpr/aws"
	"github.com/giantswarm/clustertpr/node"
	"github.com/giantswarm/k8scloudconfig"
	microerror "github.com/giantswarm/microkit/error"
	micrologger "github.com/giantswarm/microkit/logger"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/pkg/api"
	"k8s.io/client-go/pkg/runtime"
	"k8s.io/client-go/pkg/watch"
	"k8s.io/client-go/tools/cache"

	awsutil "github.com/giantswarm/aws-operator/client/aws"
	k8sutil "github.com/giantswarm/aws-operator/client/k8s"
)

const (
	ClusterListAPIEndpoint  string = "/apis/cluster.giantswarm.io/v1/awses"
	ClusterWatchAPIEndpoint string = "/apis/cluster.giantswarm.io/v1/watch/awses"
	// The format of instance's name is "[name of cluster]-[prefix ('master' or 'worker')]-[number]".
	instanceNameFormat string = "%s-%s-%d"
	// Period or re-synchronizing the list of objects in k8s watcher. 0 means that re-sync will be
	// delayed as long as possible, until the watch will be closed or timed out.
	resyncPeriod time.Duration = 0
	// Prefixes used for machine names.
	prefixMaster string = "master"
	prefixWorker string = "worker"
	// EC2 instance tag keys.
	tagKeyName    string = "Name"
	tagKeyCluster string = "Cluster"
)

type EC2StateCode int

const (
	// http://docs.aws.amazon.com/sdk-for-go/api/service/ec2/#InstanceState
	EC2PendingState      EC2StateCode = 0
	EC2RunningState      EC2StateCode = 16
	EC2ShuttingDownState EC2StateCode = 32
	EC2TerminatedState   EC2StateCode = 48
	EC2StoppingState     EC2StateCode = 64
	EC2StoppedState      EC2StateCode = 80
)

// Config represents the configuration used to create a version service.
type Config struct {
	// Dependencies.
	AwsConfig awsutil.Config
	K8sClient kubernetes.Interface
	Logger    micrologger.Logger
	CertsDir  string
}

// DefaultConfig provides a default configuration to create a new version service
// by best effort.
func DefaultConfig() Config {
	return Config{
		// Dependencies.
		K8sClient: nil,
		Logger:    nil,
		CertsDir:  "",
	}
}

// New creates a new configured version service.
func New(config Config) (*Service, error) {
	// Dependencies.
	if config.Logger == nil {
		return nil, microerror.MaskAnyf(invalidConfigError, "logger must not be empty")
	}

	newService := &Service{
		// Dependencies.
		awsConfig: config.AwsConfig,
		k8sClient: config.K8sClient,
		logger:    config.Logger,

		// AWS certificates options.
		certsDir: config.CertsDir,

		// Internals
		bootOnce: sync.Once{},
	}

	return newService, nil
}

// Service implements the version service interface.
type Service struct {
	// Dependencies.
	awsConfig awsutil.Config
	k8sClient kubernetes.Interface
	logger    micrologger.Logger

	// AWS certificates options.
	certsDir string

	// Internals.
	bootOnce sync.Once
}

type Event struct {
	Type   string
	Object *awstpr.CustomObject
}

func (s *Service) newClusterListWatch() *cache.ListWatch {
	client := s.k8sClient.Core().RESTClient()

	listWatch := &cache.ListWatch{
		ListFunc: func(options api.ListOptions) (runtime.Object, error) {
			req := client.Get().AbsPath(ClusterListAPIEndpoint)
			b, err := req.DoRaw()
			if err != nil {
				return nil, err
			}

			var c awstpr.List
			if err := json.Unmarshal(b, &c); err != nil {
				return nil, err
			}

			return &c, nil
		},

		WatchFunc: func(options api.ListOptions) (watch.Interface, error) {
			req := client.Get().AbsPath(ClusterWatchAPIEndpoint)
			stream, err := req.Stream()
			if err != nil {
				return nil, err
			}

			watcher := watch.NewStreamWatcher(&k8sutil.ClusterDecoder{
				Stream: stream,
			})

			return watcher, nil
		},
	}

	return listWatch
}

func (s *Service) Boot() {
	s.bootOnce.Do(func() {
		if err := s.createTPR(); err != nil {
			panic(err)
		}
		s.logger.Log("info", "successfully created third-party resource")

		_, clusterInformer := cache.NewInformer(
			s.newClusterListWatch(),
			&awstpr.CustomObject{},
			resyncPeriod,
			cache.ResourceEventHandlerFuncs{
				AddFunc: func(obj interface{}) {
					cluster := obj.(*awstpr.CustomObject)
					s.logger.Log("info", fmt.Sprintf("creating cluster '%s'", cluster.Name))

					if err := s.createClusterNamespace(cluster.Spec.Cluster); err != nil {
						s.logger.Log("error", fmt.Sprintf("could not create cluster namespace: %s", err))
						return
					}

					// Create AWS client
					s.awsConfig.Region = cluster.Spec.AWS.Region
					awsSession, ec2Client := awsutil.NewClient(s.awsConfig)

					// Create KMS key
					kmsSvc := kms.New(awsSession)
					key, err := kmsSvc.CreateKey(&kms.CreateKeyInput{})
					if err != nil {
						s.logger.Log("error", fmt.Sprintf("could not create KMS service client: %s", err))
						return
					}

					// Encode TLS assets
					tlsAssets, err := s.encodeTLSAssets(awsSession, *key.KeyMetadata.Arn)
					if err != nil {
						s.logger.Log("error", fmt.Sprintf("could not encode TLS assets: %s", err))
						return
					}

					// Run masters
					if err := s.runMachines(runMachinesInput{
						awsSession:  awsSession,
						ec2Client:   ec2Client,
						spec:        cluster.Spec,
						tlsAssets:   tlsAssets,
						clusterName: cluster.Name,
						prefix:      prefixMaster,
					}); err != nil {
						s.logger.Log("error", microerror.MaskAny(err))
						return
					}

					// Run workers
					if err := s.runMachines(runMachinesInput{
						awsSession:  awsSession,
						ec2Client:   ec2Client,
						spec:        cluster.Spec,
						tlsAssets:   tlsAssets,
						clusterName: cluster.Name,
						prefix:      prefixWorker,
					}); err != nil {
						s.logger.Log("error", microerror.MaskAny(err))
						return
					}

					s.logger.Log("info", fmt.Sprintf("cluster '%s' processed", cluster.Name))
				},
				DeleteFunc: func(obj interface{}) {
					cluster := obj.(*awstpr.CustomObject)
					s.logger.Log("info", fmt.Sprintf("cluster '%s' deleted", cluster.Name))

					if err := s.deleteClusterNamespace(cluster.Spec.Cluster); err != nil {
						s.logger.Log("error", "could not delete cluster namespace:", err)
					}
				},
			},
		)

		s.logger.Log("info", "starting watch")

		// Cluster informer lifecycle can be interrupted by putting a value into a "stop channel".
		// We aren't currently using that functionality, so we are passing a nil here.
		clusterInformer.Run(nil)
	})
}

type runMachinesInput struct {
	awsSession  *awssession.Session
	ec2Client   *ec2.EC2
	spec        awstpr.Spec
	tlsAssets   *cloudconfig.CompactTLSAssets
	clusterName string
	prefix      string
}

func (s *Service) runMachines(input runMachinesInput) error {
	var (
		machines    []node.Node
		awsMachines []awsinfo.Node
	)

	switch input.prefix {
	case prefixMaster:
		machines = input.spec.Cluster.Masters
		awsMachines = input.spec.AWS.Masters
	case prefixWorker:
		machines = input.spec.Cluster.Workers
		awsMachines = input.spec.AWS.Workers
	}

	// TODO(nhlfr): Create a separate module for validating specs and execute on the earlier stages.
	if len(machines) != len(awsMachines) {
		return microerror.MaskAny(fmt.Errorf("mismatched number of %s machines in the 'spec' and 'aws' sections: %d != %d",
			input.prefix,
			len(machines),
			len(awsMachines)))
	}

	for i := 0; i < len(machines); i++ {
		name := fmt.Sprintf(instanceNameFormat, input.clusterName, input.prefix, i)
		if err := s.runMachine(runMachineInput{
			awsSession:  input.awsSession,
			ec2Client:   input.ec2Client,
			spec:        input.spec,
			machine:     machines[i],
			awsNode:     awsMachines[i],
			tlsAssets:   input.tlsAssets,
			clusterName: input.clusterName,
			name:        name,
			prefix:      input.prefix,
		}); err != nil {
			return microerror.MaskAny(err)
		}
	}
	return nil
}

func allExistingInstancesMatch(instances *ec2.DescribeInstancesOutput, state EC2StateCode) bool {
	// If the instance doesn't exist, then the Reservation field should be nil.
	// Otherwise, it will contain a slice of instances (which is going to contain our one instance we queried for).
	// TODO(nhlfr): Check whether the instance has correct parameters. That will be most probably done when we
	// will introduce the interface for creating, deleting and updating resources.
	if instances.Reservations != nil {
		for _, r := range instances.Reservations {
			for _, i := range r.Instances {
				if *i.State.Code != int64(state) {
					return false
				}
			}
		}
	}
	return true
}

type runMachineInput struct {
	awsSession  *awssession.Session
	ec2Client   *ec2.EC2
	spec        awstpr.Spec
	machine     node.Node
	awsNode     awsinfo.Node
	tlsAssets   *cloudconfig.CompactTLSAssets
	clusterName string
	name        string
	prefix      string
}

func (s *Service) runMachine(input runMachineInput) error {
	instances, err := input.ec2Client.DescribeInstances(&ec2.DescribeInstancesInput{
		Filters: []*ec2.Filter{
			&ec2.Filter{
				Name: aws.String(fmt.Sprintf("tag:%s", tagKeyName)),
				Values: []*string{
					aws.String(input.name),
				},
			},
			&ec2.Filter{
				Name: aws.String(fmt.Sprintf("tag:%s", tagKeyCluster)),
				Values: []*string{
					aws.String(input.clusterName),
				},
			},
		},
	})
	if err != nil {
		return microerror.MaskAny(err)
	}

	cloudConfigParams := cloudconfig.CloudConfigTemplateParams{
		Cluster:   input.spec.Cluster,
		Node:      input.machine,
		TLSAssets: *input.tlsAssets,
	}

	cloudConfig, err := s.cloudConfig(input.prefix, cloudConfigParams)
	if err != nil {
		return err
	}

	if !allExistingInstancesMatch(instances, EC2TerminatedState) {
		s.logger.Log("info", fmt.Sprintf("instance '%s' already exists", input.name))
		return nil
	}

	reservation, err := input.ec2Client.RunInstances(&ec2.RunInstancesInput{
		ImageId:      aws.String(input.awsNode.ImageID),
		InstanceType: aws.String(input.awsNode.InstanceType),
		MinCount:     aws.Int64(int64(1)),
		MaxCount:     aws.Int64(int64(1)),
		UserData:     aws.String(cloudConfig),
	})
	if err != nil {
		return microerror.MaskAny(err)
	}

	s.logger.Log("info", fmt.Sprintf("instance '%s' reserved", input.name))

	if _, err := input.ec2Client.CreateTags(&ec2.CreateTagsInput{
		Resources: []*string{reservation.Instances[0].InstanceId},
		Tags: []*ec2.Tag{
			{
				Key:   aws.String(tagKeyName),
				Value: aws.String(input.name),
			},
			{
				Key:   aws.String(tagKeyCluster),
				Value: aws.String(input.clusterName),
			},
		},
	}); err != nil {
		return microerror.MaskAny(err)
	}

	s.logger.Log("info", fmt.Sprintf("instance '%s' tagged", input.name))

	return nil
}
<commit_msg>create: pass instance profile to new instances (#34)<commit_after>package create

import (
	"encoding/json"
	"fmt"
	"sync"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	awssession "github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/aws/aws-sdk-go/service/kms"
	"github.com/giantswarm/awstpr"
	awsinfo "github.com/giantswarm/awstpr/aws"
	"github.com/giantswarm/clustertpr/node"
	"github.com/giantswarm/k8scloudconfig"
	microerror "github.com/giantswarm/microkit/error"
	micrologger "github.com/giantswarm/microkit/logger"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/pkg/api"
	"k8s.io/client-go/pkg/runtime"
	"k8s.io/client-go/pkg/watch"
	"k8s.io/client-go/tools/cache"

	awsutil "github.com/giantswarm/aws-operator/client/aws"
	k8sutil "github.com/giantswarm/aws-operator/client/k8s"
)

const (
	ClusterListAPIEndpoint  string = "/apis/cluster.giantswarm.io/v1/awses"
	ClusterWatchAPIEndpoint string = "/apis/cluster.giantswarm.io/v1/watch/awses"
	// The format of instance's name is "[name of cluster]-[prefix ('master' or 'worker')]-[number]".
	instanceNameFormat string = "%s-%s-%d"
	// Period or re-synchronizing the list of objects in k8s watcher. 0 means that re-sync will be
	// delayed as long as possible, until the watch will be closed or timed out.
	resyncPeriod time.Duration = 0
	// Prefixes used for machine names.
	prefixMaster string = "master"
	prefixWorker string = "worker"
	// EC2 instance tag keys.
	tagKeyName    string = "Name"
	tagKeyCluster string = "Cluster"
)

type EC2StateCode int

const (
	// http://docs.aws.amazon.com/sdk-for-go/api/service/ec2/#InstanceState
	EC2PendingState      EC2StateCode = 0
	EC2RunningState      EC2StateCode = 16
	EC2ShuttingDownState EC2StateCode = 32
	EC2TerminatedState   EC2StateCode = 48
	EC2StoppingState     EC2StateCode = 64
	EC2StoppedState      EC2StateCode = 80
)

// Config represents the configuration used to create a version service.
type Config struct {
	// Dependencies.
	AwsConfig awsutil.Config
	K8sClient kubernetes.Interface
	Logger    micrologger.Logger
	CertsDir  string
}

// DefaultConfig provides a default configuration to create a new version service
// by best effort.
func DefaultConfig() Config {
	return Config{
		// Dependencies.
		K8sClient: nil,
		Logger:    nil,
		CertsDir:  "",
	}
}

// New creates a new configured version service.
func New(config Config) (*Service, error) {
	// Dependencies.
	if config.Logger == nil {
		return nil, microerror.MaskAnyf(invalidConfigError, "logger must not be empty")
	}

	newService := &Service{
		// Dependencies.
		awsConfig: config.AwsConfig,
		k8sClient: config.K8sClient,
		logger:    config.Logger,

		// AWS certificates options.
		certsDir: config.CertsDir,

		// Internals
		bootOnce: sync.Once{},
	}

	return newService, nil
}

// Service implements the version service interface.
type Service struct {
	// Dependencies.
	awsConfig awsutil.Config
	k8sClient kubernetes.Interface
	logger    micrologger.Logger

	// AWS certificates options.
	certsDir string

	// Internals.
	bootOnce sync.Once
}

type Event struct {
	Type   string
	Object *awstpr.CustomObject
}

func (s *Service) newClusterListWatch() *cache.ListWatch {
	client := s.k8sClient.Core().RESTClient()

	listWatch := &cache.ListWatch{
		ListFunc: func(options api.ListOptions) (runtime.Object, error) {
			req := client.Get().AbsPath(ClusterListAPIEndpoint)
			b, err := req.DoRaw()
			if err != nil {
				return nil, err
			}

			var c awstpr.List
			if err := json.Unmarshal(b, &c); err != nil {
				return nil, err
			}

			return &c, nil
		},

		WatchFunc: func(options api.ListOptions) (watch.Interface, error) {
			req := client.Get().AbsPath(ClusterWatchAPIEndpoint)
			stream, err := req.Stream()
			if err != nil {
				return nil, err
			}

			watcher := watch.NewStreamWatcher(&k8sutil.ClusterDecoder{
				Stream: stream,
			})

			return watcher, nil
		},
	}

	return listWatch
}

func (s *Service) Boot() {
	s.bootOnce.Do(func() {
		if err := s.createTPR(); err != nil {
			panic(err)
		}
		s.logger.Log("info", "successfully created third-party resource")

		_, clusterInformer := cache.NewInformer(
			s.newClusterListWatch(),
			&awstpr.CustomObject{},
			resyncPeriod,
			cache.ResourceEventHandlerFuncs{
				AddFunc: func(obj interface{}) {
					cluster := obj.(*awstpr.CustomObject)
					s.logger.Log("info", fmt.Sprintf("creating cluster '%s'", cluster.Name))

					if err := s.createClusterNamespace(cluster.Spec.Cluster); err != nil {
						s.logger.Log("error", fmt.Sprintf("could not create cluster namespace: %s", err))
						return
					}

					// Create AWS client
					s.awsConfig.Region = cluster.Spec.AWS.Region
					awsSession, ec2Client := awsutil.NewClient(s.awsConfig)

					// Create KMS key
					kmsSvc := kms.New(awsSession)
					key, err := kmsSvc.CreateKey(&kms.CreateKeyInput{})
					if err != nil {
						s.logger.Log("error", fmt.Sprintf("could not create KMS service client: %s", err))
						return
					}

					// Encode TLS assets
					tlsAssets, err := s.encodeTLSAssets(awsSession, *key.KeyMetadata.Arn)
					if err != nil {
						s.logger.Log("error", fmt.Sprintf("could not encode TLS assets: %s", err))
						return
					}

					// Run masters
					if err := s.runMachines(runMachinesInput{
						awsSession:  awsSession,
						ec2Client:   ec2Client,
						spec:        cluster.Spec,
						tlsAssets:   tlsAssets,
						clusterName: cluster.Name,
						prefix:      prefixMaster,
					}); err != nil {
						s.logger.Log("error", microerror.MaskAny(err))
						return
					}

					// Run workers
					if err := s.runMachines(runMachinesInput{
						awsSession:  awsSession,
						ec2Client:   ec2Client,
						spec:        cluster.Spec,
						tlsAssets:   tlsAssets,
						clusterName: cluster.Name,
						prefix:      prefixWorker,
					}); err != nil {
						s.logger.Log("error", microerror.MaskAny(err))
						return
					}

					s.logger.Log("info", fmt.Sprintf("cluster '%s' processed", cluster.Name))
				},
				DeleteFunc: func(obj interface{}) {
					cluster := obj.(*awstpr.CustomObject)
					s.logger.Log("info", fmt.Sprintf("cluster '%s' deleted", cluster.Name))

					if err := s.deleteClusterNamespace(cluster.Spec.Cluster); err != nil {
						s.logger.Log("error", "could not delete cluster namespace:", err)
					}
				},
			},
		)

		s.logger.Log("info", "starting watch")

		// Cluster informer lifecycle can be interrupted by putting a value into a "stop channel".
		// We aren't currently using that functionality, so we are passing a nil here.
		clusterInformer.Run(nil)
	})
}

type runMachinesInput struct {
	awsSession  *awssession.Session
	ec2Client   *ec2.EC2
	spec        awstpr.Spec
	tlsAssets   *cloudconfig.CompactTLSAssets
	clusterName string
	prefix      string
}

func (s *Service) runMachines(input runMachinesInput) error {
	var (
		machines    []node.Node
		awsMachines []awsinfo.Node
	)

	switch input.prefix {
	case prefixMaster:
		machines = input.spec.Cluster.Masters
		awsMachines = input.spec.AWS.Masters
	case prefixWorker:
		machines = input.spec.Cluster.Workers
		awsMachines = input.spec.AWS.Workers
	}

	// TODO(nhlfr): Create a separate module for validating specs and execute on the earlier stages.
	if len(machines) != len(awsMachines) {
		return microerror.MaskAny(fmt.Errorf("mismatched number of %s machines in the 'spec' and 'aws' sections: %d != %d",
			input.prefix,
			len(machines),
			len(awsMachines)))
	}

	for i := 0; i < len(machines); i++ {
		name := fmt.Sprintf(instanceNameFormat, input.clusterName, input.prefix, i)
		if err := s.runMachine(runMachineInput{
			awsSession:  input.awsSession,
			ec2Client:   input.ec2Client,
			spec:        input.spec,
			machine:     machines[i],
			awsNode:     awsMachines[i],
			tlsAssets:   input.tlsAssets,
			clusterName: input.clusterName,
			name:        name,
			prefix:      input.prefix,
		}); err != nil {
			return microerror.MaskAny(err)
		}
	}
	return nil
}

func allExistingInstancesMatch(instances *ec2.DescribeInstancesOutput, state EC2StateCode) bool {
	// If the instance doesn't exist, then the Reservation field should be nil.
	// Otherwise, it will contain a slice of instances (which is going to contain our one instance we queried for).
	// TODO(nhlfr): Check whether the instance has correct parameters. That will be most probably done when we
	// will introduce the interface for creating, deleting and updating resources.
	if instances.Reservations != nil {
		for _, r := range instances.Reservations {
			for _, i := range r.Instances {
				if *i.State.Code != int64(state) {
					return false
				}
			}
		}
	}
	return true
}

type runMachineInput struct {
	awsSession  *awssession.Session
	ec2Client   *ec2.EC2
	spec        awstpr.Spec
	machine     node.Node
	awsNode     awsinfo.Node
	tlsAssets   *cloudconfig.CompactTLSAssets
	clusterName string
	name        string
	prefix      string
}

func (s *Service) runMachine(input runMachineInput) error {
	instances, err := input.ec2Client.DescribeInstances(&ec2.DescribeInstancesInput{
		Filters: []*ec2.Filter{
			&ec2.Filter{
				Name: aws.String(fmt.Sprintf("tag:%s", tagKeyName)),
				Values: []*string{
					aws.String(input.name),
				},
			},
			&ec2.Filter{
				Name: aws.String(fmt.Sprintf("tag:%s", tagKeyCluster)),
				Values: []*string{
					aws.String(input.clusterName),
				},
			},
		},
	})
	if err != nil {
		return microerror.MaskAny(err)
	}

	cloudConfigParams := cloudconfig.CloudConfigTemplateParams{
		Cluster:   input.spec.Cluster,
		Node:      input.machine,
		TLSAssets: *input.tlsAssets,
	}

	cloudConfig, err := s.cloudConfig(input.prefix, cloudConfigParams)
	if err != nil {
		return err
	}

	if !allExistingInstancesMatch(instances, EC2TerminatedState) {
		s.logger.Log("info", fmt.Sprintf("instance '%s' already exists", input.name))
		return nil
	}

	reservation, err := input.ec2Client.RunInstances(&ec2.RunInstancesInput{
		ImageId:      aws.String(input.awsNode.ImageID),
		InstanceType: aws.String(input.awsNode.InstanceType),
		MinCount:     aws.Int64(int64(1)),
		MaxCount:     aws.Int64(int64(1)),
		UserData:     aws.String(cloudConfig),
		IamInstanceProfile: &ec2.IamInstanceProfileSpecification{
			Name: aws.String(ProfileName),
		},
	})
	if err != nil {
		return microerror.MaskAny(err)
	}

	s.logger.Log("info", fmt.Sprintf("instance '%s' reserved", input.name))

	if _, err := input.ec2Client.CreateTags(&ec2.CreateTagsInput{
		Resources: []*string{reservation.Instances[0].InstanceId},
		Tags: []*ec2.Tag{
			{
				Key:   aws.String(tagKeyName),
				Value: aws.String(input.name),
			},
			{
				Key:   aws.String(tagKeyCluster),
				Value: aws.String(input.clusterName),
			},
		},
	}); err != nil {
		return microerror.MaskAny(err)
	}

	s.logger.Log("info", fmt.Sprintf("instance '%s' tagged", input.name))

	return nil
}
<|endoftext|>


<commit_before>package smf

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"time"

	"github.com/gomidi/midi/internal/midilib"

	"github.com/gomidi/midi"
)

var (
	_ midi.Writer = Writer(nil)
	_ midi.Reader = Reader(nil)
)

// Writer writes midi messages to a standard midi file (SMF)
// Writer is also a midi.Writer
type Writer interface {

	// Header returns the header
	Header() Header

	// WriteHeader writes the midi header
	// If WriteHeader was not called before the first run of Write,
	// it will implicitly be called when calling Write.
	WriteHeader() (int, error)

	// Write writes a midi message to the SMF file.
	//
	// Due to the nature of SMF files there is some maybe surprising behavior.
	// - If the header has not been written yet, it will be written before writing the first message.
	// - The first message will be written to track 0 which will be implicitly created.
	// - All messages of a track will be buffered inside the track and only be written if an EndOfTrack
	//   message is written.
	// - The number of tracks that are written will never execeed the NumTracks that have been defined when creating the writer.
	//   If the last track has been written, io.EOF will be returned. (Also for any further attempt to write).
	// - It is the responsibility of the caller to make sure the provided NumTracks (which defaults to 1) is not
	//   larger as the number of tracks in the file.
	// Keep the above in mind when examinating the written nbytes that are returned. They reflect the number of bytes
	// that have been physically written at that point in time.
	// any error stops the writing, is tracked and prohibits further writing.
	// this last error is returned from Error()
	Write(midi.Message) (nBytes int, err error)

	// SetDelta sets a time distance between the last written and the following message in ticks.
	// The meaning of a tick depends on the time format that is set in the header of the SMF file.
	SetDelta(ticks uint32)
}

// Reader reads midi messages from a standard midi file (SMF)
// Reader is also a midi.Reader
type Reader interface {

	// ReadHeader reads the header of the SMF file. If Header is called before ReadHeader, it will panic.
	// ReadHeader is also implicitly called with the first call of Read() (if it has not been run before)
	ReadHeader() error

	// Read reads a MIDI message from a SMF file.
	// any error will be tracked and stops reading and prevents any other attempt to read.
	// this first and last error is returned from Error()
	Read() (midi.Message, error)

	// Header returns the header of SMF file
	// if the header is not yet read, it will be read before
	// if any error occurred during reading of header, it can be found with Error()
	Header() Header

	// Delta returns the time distance between the last read midi message and the message before in ticks.
	// The meaning of a tick depends on the time format that is set in the header of the SMF file.
	Delta() (ticks uint32)

	// Track returns the number of the track of the last read midi message (starting with 0)
	// It returns -1 if no message has been read yet.
	Track() int16
}

// Header represents the header of a SMF file.
type Header struct {

	// Format is the SMF file format: SMF0, SMF1 or SMF2
	Format

	// NumTracks is the number of tracks (always > 0)
	NumTracks uint16

	// TimeFormat is the time format (either MetricTicks or TimeCode)
	TimeFormat
}

func (h Header) String() string {
	return fmt.Sprintf("<Format: %v, NumTracks: %v, TimeFormat: %v>", h.Format, h.NumTracks, h.TimeFormat)
}

const (
	// SMF0 represents the singletrack SMF format (0)
	SMF0 = format(0)

	// SMF1 represents the multitrack SMF format (1)
	SMF1 = format(1)

	// SMF2 represents the sequential track SMF format (2)
	SMF2 = format(2)
)

type Chunk struct {
	typ  []byte // must always be 4 bytes long, to avoid conversions everytime, we take []byte here instead of [4]byte
	data []byte
}

// Chunk returns the length of the chunk body
func (c *Chunk) Len() int {
	return len(c.data)
}

// SetType sets the type of the chunk
func (c *Chunk) SetType(typ [4]byte) {
	c.typ = make([]byte, 4)
	c.typ[0] = typ[0]
	c.typ[1] = typ[1]
	c.typ[2] = typ[2]
	c.typ[3] = typ[3]
}

func (c *Chunk) Type() string {
	var bf bytes.Buffer
	bf.Write(c.typ)
	return bf.String()
}

// Clear removes all data but keeps the typ
func (c *Chunk) Clear() {
	c.data = nil
}

// WriteTo writes the content of the chunk to the given writer
func (c *Chunk) WriteTo(wr io.Writer) (int64, error) {
	if len(c.typ) != 4 {
		return 0, fmt.Errorf("chunk header not set properly")
	}

	var bf bytes.Buffer
	bf.Write(c.typ)
	binary.Write(&bf, binary.BigEndian, int32(c.Len()))
	bf.Write(c.data)
	i, err := wr.Write(bf.Bytes())
	return int64(i), err
}

// ReadHeader reads the header from the given reader
// returns the length of the following body
// for errors, length of 0 is returned
func (c *Chunk) ReadHeader(rd io.Reader) (length uint32, err error) {
	c.typ, err = midilib.ReadNBytes(4, rd)

	if err != nil {
		c.typ = nil
		return
	}

	return midilib.ReadUint32(rd)
}

// Write writes the given bytes to the body of the chunk
func (c *Chunk) Write(b []byte) (int, error) {
	c.data = append(c.data, b...)
	return len(b), nil
}

var (
	_ TimeFormat = MetricTicks(0)
	_ TimeFormat = TimeCode{}
	_ Format     = SMF0
)

// TimeCode is the SMPTE time format.
// It can be comfortable created with the SMPTE* functions.
type TimeCode struct {
	FramesPerSecond uint8
	SubFrames       uint8
}

// String represents the TimeCode as a string.
func (t TimeCode) String() string {

	switch t.FramesPerSecond {
	case 29:
		return fmt.Sprintf("SMPTE30DropFrame %v subframes", t.SubFrames)
	default:
		return fmt.Sprintf("SMPTE%v %v subframes", t.FramesPerSecond, t.SubFrames)
	}

}

func (t TimeCode) timeformat() {}

// SMPTE24 returns a SMPTE24 TimeCode with the given subframes
func SMPTE24(subframes uint8) TimeCode {
	return TimeCode{24, subframes}
}

// SMPTE25 returns a SMPTE25 TimeCode with the given subframes
func SMPTE25(subframes uint8) TimeCode {
	return TimeCode{25, subframes}
}

// SMPTE30DropFrame returns a SMPTE30 drop frame TimeCode with the given subframes
func SMPTE30DropFrame(subframes uint8) TimeCode {
	return TimeCode{29, subframes}
}

// SMPTE30 returns a SMPTE30 TimeCode with the given subframes
func SMPTE30(subframes uint8) TimeCode {
	return TimeCode{30, subframes}
}

// MetricTicks represents the "ticks per quarter note" (metric) time format
// It defaults to 960 (i.e. 0 is treated as if it where 960 ticks per quarter note)
type MetricTicks uint16

// Duration returns the time.Duration for a number of ticks at a certain tempo (in BPM)
func (q MetricTicks) Duration(tempoBPM uint32, deltaTicks uint32) time.Duration {
	// (60000 / T) * (d / R) = D[ms]
	durQnMilli := 60000 / float64(tempoBPM)
	_4thticks := float64(deltaTicks) / float64(uint16(q))
	return time.Duration(roundFloat(durQnMilli*_4thticks, 0)) * time.Millisecond
}

// Ticks returns the ticks for a given time.Duration at a certain tempo (in BPM)
func (q MetricTicks) Ticks(tempoBPM uint32, d time.Duration) (ticks uint32) {
	// d = (D[ms] * R * T) / 60000
	ticks = uint32(roundFloat((float64(d.Nanoseconds())/1000000*float64(uint16(q))*float64(tempoBPM))/60000, 0))
	return ticks
}

func (q MetricTicks) div(d float64) uint32 {
	return uint32(roundFloat(float64(q.Ticks4th())/d, 0))
}

// Number returns the number of the metric ticks (ticks for a quarter note, defaults to 960)
func (q MetricTicks) Number() uint16 {
	return q.Ticks4th()
}

// Ticks4th returns the ticks for a quarter note
func (q MetricTicks) Ticks4th() uint16 {
	if uint16(q) == 0 {
		return 960 // default
	}
	return uint16(q)
}

// Ticks8th returns the ticks for a quaver note
func (q MetricTicks) Ticks8th() uint32 {
	return q.div(2)
}

// Ticks16th returns the ticks for a 16th note
func (q MetricTicks) Ticks16th() uint32 {
	return q.div(4)
}

// Ticks32th returns the ticks for a 32th note
func (q MetricTicks) Ticks32th() uint32 {
	return q.div(8)
}

// Ticks64th returns the ticks for a 64th note
func (q MetricTicks) Ticks64th() uint32 {
	return q.div(16)
}

// Ticks128th returns the ticks for a 128th note
func (q MetricTicks) Ticks128th() uint32 {
	return q.div(32)
}

// Ticks256th returns the ticks for a 256th note
func (q MetricTicks) Ticks256th() uint32 {
	return q.div(64)
}

// Ticks512th returns the ticks for a 512th note
func (q MetricTicks) Ticks512th() uint32 {
	return q.div(128)
}

// Ticks1024th returns the ticks for a 1024th note
func (q MetricTicks) Ticks1024th() uint32 {
	return q.div(256)
}

// String returns the string representation of the quarter note resolution
func (q MetricTicks) String() string {
	return fmt.Sprintf("%v MetricTicks", q.Ticks4th())
}

func (q MetricTicks) timeformat() {}

// Format is the common interface of all SMF file formats
type Format interface {

	// String returns the string representation of the SMF format.
	String() string

	// Type returns the type of the SMF file: 0 for SMF0, 1 for SMF1 and 2 for SMF2
	Type() uint16

	smfformat() // make the implementation exclusive to this package
}

// TimeFormat is the common interface of all SMF time formats
type TimeFormat interface {
	String() string
	timeformat() // make the implementation exclusive to this package
}

// format is an implementation of Format
type format uint16

func (f format) Type() uint16 {
	return uint16(f)
}

func (f format) smfformat() {}

func (f format) String() string {
	switch f {
	case SMF0:
		return "SMF0 (singletrack)"
	case SMF1:
		return "SMF1 (multitrack)"
	case SMF2:
		return "SMF2 (sequential tracks)"
	}
	panic("unreachable")
}
<commit_msg>make Ticks4th consistent with other Ticks* functions (fixes #10)<commit_after>package smf

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"time"

	"github.com/gomidi/midi/internal/midilib"

	"github.com/gomidi/midi"
)

var (
	_ midi.Writer = Writer(nil)
	_ midi.Reader = Reader(nil)
)

// Writer writes midi messages to a standard midi file (SMF)
// Writer is also a midi.Writer
type Writer interface {

	// Header returns the header
	Header() Header

	// WriteHeader writes the midi header
	// If WriteHeader was not called before the first run of Write,
	// it will implicitly be called when calling Write.
	WriteHeader() (int, error)

	// Write writes a midi message to the SMF file.
	//
	// Due to the nature of SMF files there is some maybe surprising behavior.
	// - If the header has not been written yet, it will be written before writing the first message.
	// - The first message will be written to track 0 which will be implicitly created.
	// - All messages of a track will be buffered inside the track and only be written if an EndOfTrack
	//   message is written.
	// - The number of tracks that are written will never execeed the NumTracks that have been defined when creating the writer.
	//   If the last track has been written, io.EOF will be returned. (Also for any further attempt to write).
	// - It is the responsibility of the caller to make sure the provided NumTracks (which defaults to 1) is not
	//   larger as the number of tracks in the file.
	// Keep the above in mind when examinating the written nbytes that are returned. They reflect the number of bytes
	// that have been physically written at that point in time.
	// any error stops the writing, is tracked and prohibits further writing.
	// this last error is returned from Error()
	Write(midi.Message) (nBytes int, err error)

	// SetDelta sets a time distance between the last written and the following message in ticks.
	// The meaning of a tick depends on the time format that is set in the header of the SMF file.
	SetDelta(ticks uint32)
}

// Reader reads midi messages from a standard midi file (SMF)
// Reader is also a midi.Reader
type Reader interface {

	// ReadHeader reads the header of the SMF file. If Header is called before ReadHeader, it will panic.
	// ReadHeader is also implicitly called with the first call of Read() (if it has not been run before)
	ReadHeader() error

	// Read reads a MIDI message from a SMF file.
	// any error will be tracked and stops reading and prevents any other attempt to read.
	// this first and last error is returned from Error()
	Read() (midi.Message, error)

	// Header returns the header of SMF file
	// if the header is not yet read, it will be read before
	// if any error occurred during reading of header, it can be found with Error()
	Header() Header

	// Delta returns the time distance between the last read midi message and the message before in ticks.
	// The meaning of a tick depends on the time format that is set in the header of the SMF file.
	Delta() (ticks uint32)

	// Track returns the number of the track of the last read midi message (starting with 0)
	// It returns -1 if no message has been read yet.
	Track() int16
}

// Header represents the header of a SMF file.
type Header struct {

	// Format is the SMF file format: SMF0, SMF1 or SMF2
	Format

	// NumTracks is the number of tracks (always > 0)
	NumTracks uint16

	// TimeFormat is the time format (either MetricTicks or TimeCode)
	TimeFormat
}

func (h Header) String() string {
	return fmt.Sprintf("<Format: %v, NumTracks: %v, TimeFormat: %v>", h.Format, h.NumTracks, h.TimeFormat)
}

const (
	// SMF0 represents the singletrack SMF format (0)
	SMF0 = format(0)

	// SMF1 represents the multitrack SMF format (1)
	SMF1 = format(1)

	// SMF2 represents the sequential track SMF format (2)
	SMF2 = format(2)
)

type Chunk struct {
	typ  []byte // must always be 4 bytes long, to avoid conversions everytime, we take []byte here instead of [4]byte
	data []byte
}

// Chunk returns the length of the chunk body
func (c *Chunk) Len() int {
	return len(c.data)
}

// SetType sets the type of the chunk
func (c *Chunk) SetType(typ [4]byte) {
	c.typ = make([]byte, 4)
	c.typ[0] = typ[0]
	c.typ[1] = typ[1]
	c.typ[2] = typ[2]
	c.typ[3] = typ[3]
}

func (c *Chunk) Type() string {
	var bf bytes.Buffer
	bf.Write(c.typ)
	return bf.String()
}

// Clear removes all data but keeps the typ
func (c *Chunk) Clear() {
	c.data = nil
}

// WriteTo writes the content of the chunk to the given writer
func (c *Chunk) WriteTo(wr io.Writer) (int64, error) {
	if len(c.typ) != 4 {
		return 0, fmt.Errorf("chunk header not set properly")
	}

	var bf bytes.Buffer
	bf.Write(c.typ)
	binary.Write(&bf, binary.BigEndian, int32(c.Len()))
	bf.Write(c.data)
	i, err := wr.Write(bf.Bytes())
	return int64(i), err
}

// ReadHeader reads the header from the given reader
// returns the length of the following body
// for errors, length of 0 is returned
func (c *Chunk) ReadHeader(rd io.Reader) (length uint32, err error) {
	c.typ, err = midilib.ReadNBytes(4, rd)

	if err != nil {
		c.typ = nil
		return
	}

	return midilib.ReadUint32(rd)
}

// Write writes the given bytes to the body of the chunk
func (c *Chunk) Write(b []byte) (int, error) {
	c.data = append(c.data, b...)
	return len(b), nil
}

var (
	_ TimeFormat = MetricTicks(0)
	_ TimeFormat = TimeCode{}
	_ Format     = SMF0
)

// TimeCode is the SMPTE time format.
// It can be comfortable created with the SMPTE* functions.
type TimeCode struct {
	FramesPerSecond uint8
	SubFrames       uint8
}

// String represents the TimeCode as a string.
func (t TimeCode) String() string {

	switch t.FramesPerSecond {
	case 29:
		return fmt.Sprintf("SMPTE30DropFrame %v subframes", t.SubFrames)
	default:
		return fmt.Sprintf("SMPTE%v %v subframes", t.FramesPerSecond, t.SubFrames)
	}

}

func (t TimeCode) timeformat() {}

// SMPTE24 returns a SMPTE24 TimeCode with the given subframes
func SMPTE24(subframes uint8) TimeCode {
	return TimeCode{24, subframes}
}

// SMPTE25 returns a SMPTE25 TimeCode with the given subframes
func SMPTE25(subframes uint8) TimeCode {
	return TimeCode{25, subframes}
}

// SMPTE30DropFrame returns a SMPTE30 drop frame TimeCode with the given subframes
func SMPTE30DropFrame(subframes uint8) TimeCode {
	return TimeCode{29, subframes}
}

// SMPTE30 returns a SMPTE30 TimeCode with the given subframes
func SMPTE30(subframes uint8) TimeCode {
	return TimeCode{30, subframes}
}

// MetricTicks represents the "ticks per quarter note" (metric) time format
// It defaults to 960 (i.e. 0 is treated as if it where 960 ticks per quarter note)
type MetricTicks uint16

// Duration returns the time.Duration for a number of ticks at a certain tempo (in BPM)
func (q MetricTicks) Duration(tempoBPM uint32, deltaTicks uint32) time.Duration {
	// (60000 / T) * (d / R) = D[ms]
	durQnMilli := 60000 / float64(tempoBPM)
	_4thticks := float64(deltaTicks) / float64(uint16(q))
	return time.Duration(roundFloat(durQnMilli*_4thticks, 0)) * time.Millisecond
}

// Ticks returns the ticks for a given time.Duration at a certain tempo (in BPM)
func (q MetricTicks) Ticks(tempoBPM uint32, d time.Duration) (ticks uint32) {
	// d = (D[ms] * R * T) / 60000
	ticks = uint32(roundFloat((float64(d.Nanoseconds())/1000000*float64(uint16(q))*float64(tempoBPM))/60000, 0))
	return ticks
}

func (q MetricTicks) div(d float64) uint32 {
	return uint32(roundFloat(float64(q.Number())/d, 0))
}

// Number returns the number of the metric ticks (ticks for a quarter note, defaults to 960)
func (q MetricTicks) Number() uint16 {
	if q == 0 {
		return 960 // default
	}
	return uint16(q)
}

// Ticks4th returns the ticks for a quarter note
func (q MetricTicks) Ticks4th() uint32 {
	return uint32(q.Number())
}

// Ticks8th returns the ticks for a quaver note
func (q MetricTicks) Ticks8th() uint32 {
	return q.div(2)
}

// Ticks16th returns the ticks for a 16th note
func (q MetricTicks) Ticks16th() uint32 {
	return q.div(4)
}

// Ticks32th returns the ticks for a 32th note
func (q MetricTicks) Ticks32th() uint32 {
	return q.div(8)
}

// Ticks64th returns the ticks for a 64th note
func (q MetricTicks) Ticks64th() uint32 {
	return q.div(16)
}

// Ticks128th returns the ticks for a 128th note
func (q MetricTicks) Ticks128th() uint32 {
	return q.div(32)
}

// Ticks256th returns the ticks for a 256th note
func (q MetricTicks) Ticks256th() uint32 {
	return q.div(64)
}

// Ticks512th returns the ticks for a 512th note
func (q MetricTicks) Ticks512th() uint32 {
	return q.div(128)
}

// Ticks1024th returns the ticks for a 1024th note
func (q MetricTicks) Ticks1024th() uint32 {
	return q.div(256)
}

// String returns the string representation of the quarter note resolution
func (q MetricTicks) String() string {
	return fmt.Sprintf("%v MetricTicks", q.Ticks4th())
}

func (q MetricTicks) timeformat() {}

// Format is the common interface of all SMF file formats
type Format interface {

	// String returns the string representation of the SMF format.
	String() string

	// Type returns the type of the SMF file: 0 for SMF0, 1 for SMF1 and 2 for SMF2
	Type() uint16

	smfformat() // make the implementation exclusive to this package
}

// TimeFormat is the common interface of all SMF time formats
type TimeFormat interface {
	String() string
	timeformat() // make the implementation exclusive to this package
}

// format is an implementation of Format
type format uint16

func (f format) Type() uint16 {
	return uint16(f)
}

func (f format) smfformat() {}

func (f format) String() string {
	switch f {
	case SMF0:
		return "SMF0 (singletrack)"
	case SMF1:
		return "SMF1 (multitrack)"
	case SMF2:
		return "SMF2 (sequential tracks)"
	}
	panic("unreachable")
}
<|endoftext|>


<commit_before>package smtpapi

import (
	"bytes"
	"encoding/json"
	"fmt"
	"unicode/utf16"
)

const Version = "0.4.2"

// SMTPAPIHeader will be used to set up X-SMTPAPI params
type SMTPAPIHeader struct {
	To         []string            `json:"to,omitempty"`
	Sub        map[string][]string `json:"sub,omitempty"`
	Section    map[string]string   `json:"section,omitempty"`
	Category   []string            `json:"category,omitempty"`
	UniqueArgs map[string]string   `json:"unique_args,omitempty"`
	Filters    map[string]Filter   `json:"filters,omitempty"`
	ASMGroupID int                 `json:"asm_group_id,omitempty"`
	SendAt     int64               `json:"send_at,omitempty"`
	SendEachAt []int64             `json:"send_each_at,omitempty"`
	IpPool     string              `json:"ip_pool,omitempty"`
	BatchID    string              `json:"batch_id,omitempty"`
}

// Filter represents an App/Filter and its settings
type Filter struct {
	Settings map[string]interface{} `json:"settings,omitempty"`
}

// NewSMTPAPIHeader creates a new header struct
func NewSMTPAPIHeader() *SMTPAPIHeader {
	return &SMTPAPIHeader{}
}

// AddTo appends a single email to the To header
func (h *SMTPAPIHeader) AddTo(email string) {
	h.To = append(h.To, email)
}

// AddTos appends multiple emails to the To header
func (h *SMTPAPIHeader) AddTos(emails []string) {
	for i := 0; i < len(emails); i++ {
		h.AddTo(emails[i])
	}
}

// SetTos sets the value of the To header
func (h *SMTPAPIHeader) SetTos(emails []string) {
	h.To = emails
}

// AddSubstitution adds a new substitution to a specific key
func (h *SMTPAPIHeader) AddSubstitution(key, sub string) {
	if h.Sub == nil {
		h.Sub = make(map[string][]string)
	}
	h.Sub[key] = append(h.Sub[key], sub)
}

// AddSubstitutions adds a multiple substitutions to a specific key
func (h *SMTPAPIHeader) AddSubstitutions(key string, subs []string) {
	for i := 0; i < len(subs); i++ {
		h.AddSubstitution(key, subs[i])
	}
}

// SetSubstitutions sets the value of the substitutions on the Sub header
func (h *SMTPAPIHeader) SetSubstitutions(sub map[string][]string) {
	h.Sub = sub
}

// AddSection sets the value for a specific section
func (h *SMTPAPIHeader) AddSection(section, value string) {
	if h.Section == nil {
		h.Section = make(map[string]string)
	}
	h.Section[section] = value
}

// SetSections sets the value for the Section header
func (h *SMTPAPIHeader) SetSections(sections map[string]string) {
	h.Section = sections
}

// AddCategory adds a new category to the Category header
func (h *SMTPAPIHeader) AddCategory(category string) {
	h.Category = append(h.Category, category)
}

// AddCategories adds multiple categories to the Category header
func (h *SMTPAPIHeader) AddCategories(categories []string) {
	for i := 0; i < len(categories); i++ {
		h.AddCategory(categories[i])
	}
}

// SetCategories will set the value of the Categories field
func (h *SMTPAPIHeader) SetCategories(categories []string) {
	h.Category = categories
}

// SetASMGroupID will set the value of the ASMGroupID field
func (h *SMTPAPIHeader) SetASMGroupID(groupID int) {
	h.ASMGroupID = groupID
}

// AddUniqueArg will set the value of a specific argument
func (h *SMTPAPIHeader) AddUniqueArg(arg, value string) {
	if h.UniqueArgs == nil {
		h.UniqueArgs = make(map[string]string)
	}
	h.UniqueArgs[arg] = value
}

// SetUniqueArgs will set the value of the Unique_args header
func (h *SMTPAPIHeader) SetUniqueArgs(args map[string]string) {
	h.UniqueArgs = args
}

// AddFilter will set the specific setting for a filter
func (h *SMTPAPIHeader) AddFilter(filter, setting string, value interface{}) {
	if h.Filters == nil {
		h.Filters = make(map[string]Filter)
	}
	if _, ok := h.Filters[filter]; !ok {
		h.Filters[filter] = Filter{
			Settings: make(map[string]interface{}),
		}
	}
	h.Filters[filter].Settings[setting] = value
}

// SetFilter takes in a Filter struct with predetermined settings and sets it for such Filter key
func (h *SMTPAPIHeader) SetFilter(filter string, value *Filter) {
	if h.Filters == nil {
		h.Filters = make(map[string]Filter)
	}
	h.Filters[filter] = *value
}

// SetSendAt takes in a timestamp which determines when the email will be sent
func (h *SMTPAPIHeader) SetSendAt(sendAt int64) {
	h.SendAt = sendAt
}

// AddSendEachAt takes in a timestamp and pushes it into a list Must match length of To emails
func (h *SMTPAPIHeader) AddSendEachAt(sendEachAt int64) {
	h.SendEachAt = append(h.SendEachAt, sendEachAt)
}

// SetSendEachAt takes an array of timestamps. Must match length of To emails
func (h *SMTPAPIHeader) SetSendEachAt(sendEachAt []int64) {
	h.SendEachAt = sendEachAt
}

// SetIpPool takes a strings and sets the IpPool field
func (h *SMTPAPIHeader) SetIpPool(ipPool string) {
	h.IpPool = ipPool
}

// Unicode escape
func escapeUnicode(input string) string {
	//var buffer bytes.Buffer
	buffer := bytes.NewBufferString("")
	for _, r := range input {
		if r > 65535 {
			// surrogate pair
			var r1, r2 = utf16.EncodeRune(r)
			var s = fmt.Sprintf("\\u%x\\u%x", r1, r2)
			buffer.WriteString(s)
		} else if r > 127 {
			var s = fmt.Sprintf("\\u%04x", r)
			buffer.WriteString(s)
		} else {
			var s = fmt.Sprintf("%c", r)
			buffer.WriteString(s)
		}
	}
	return buffer.String()
}

// JSONString returns the representation of the Header
func (h *SMTPAPIHeader) JSONString() (string, error) {
	headers, e := json.Marshal(h)
	return escapeUnicode(string(headers)), e
}

// Load allows you to load a pre-formed x-smtpapi header
func (h *SMTPAPIHeader) Load(b []byte) error {
	return json.Unmarshal(b, h)
}
<commit_msg>QEA-1700: support asm_groups_to_display<commit_after>package smtpapi

import (
	"bytes"
	"encoding/json"
	"fmt"
	"unicode/utf16"
)

const Version = "0.4.2"

// SMTPAPIHeader will be used to set up X-SMTPAPI params
type SMTPAPIHeader struct {
	To         []string            `json:"to,omitempty"`
	Sub        map[string][]string `json:"sub,omitempty"`
	Section    map[string]string   `json:"section,omitempty"`
	Category   []string            `json:"category,omitempty"`
	UniqueArgs map[string]string   `json:"unique_args,omitempty"`
	Filters    map[string]Filter   `json:"filters,omitempty"`
	ASMGroupID int                 `json:"asm_group_id,omitempty"`
	ASMGroups  []int               `json:"asm_groups_to_display,omitempty"`
	SendAt     int64               `json:"send_at,omitempty"`
	SendEachAt []int64             `json:"send_each_at,omitempty"`
	IpPool     string              `json:"ip_pool,omitempty"`
	BatchID    string              `json:"batch_id,omitempty"`
}

// Filter represents an App/Filter and its settings
type Filter struct {
	Settings map[string]interface{} `json:"settings,omitempty"`
}

// NewSMTPAPIHeader creates a new header struct
func NewSMTPAPIHeader() *SMTPAPIHeader {
	return &SMTPAPIHeader{}
}

// AddTo appends a single email to the To header
func (h *SMTPAPIHeader) AddTo(email string) {
	h.To = append(h.To, email)
}

// AddTos appends multiple emails to the To header
func (h *SMTPAPIHeader) AddTos(emails []string) {
	for i := 0; i < len(emails); i++ {
		h.AddTo(emails[i])
	}
}

// SetTos sets the value of the To header
func (h *SMTPAPIHeader) SetTos(emails []string) {
	h.To = emails
}

// AddSubstitution adds a new substitution to a specific key
func (h *SMTPAPIHeader) AddSubstitution(key, sub string) {
	if h.Sub == nil {
		h.Sub = make(map[string][]string)
	}
	h.Sub[key] = append(h.Sub[key], sub)
}

// AddSubstitutions adds a multiple substitutions to a specific key
func (h *SMTPAPIHeader) AddSubstitutions(key string, subs []string) {
	for i := 0; i < len(subs); i++ {
		h.AddSubstitution(key, subs[i])
	}
}

// SetSubstitutions sets the value of the substitutions on the Sub header
func (h *SMTPAPIHeader) SetSubstitutions(sub map[string][]string) {
	h.Sub = sub
}

// AddSection sets the value for a specific section
func (h *SMTPAPIHeader) AddSection(section, value string) {
	if h.Section == nil {
		h.Section = make(map[string]string)
	}
	h.Section[section] = value
}

// SetSections sets the value for the Section header
func (h *SMTPAPIHeader) SetSections(sections map[string]string) {
	h.Section = sections
}

// AddCategory adds a new category to the Category header
func (h *SMTPAPIHeader) AddCategory(category string) {
	h.Category = append(h.Category, category)
}

// AddCategories adds multiple categories to the Category header
func (h *SMTPAPIHeader) AddCategories(categories []string) {
	for i := 0; i < len(categories); i++ {
		h.AddCategory(categories[i])
	}
}

// SetCategories will set the value of the Categories field
func (h *SMTPAPIHeader) SetCategories(categories []string) {
	h.Category = categories
}

// SetASMGroupID will set the value of the ASMGroupID field
func (h *SMTPAPIHeader) SetASMGroupID(groupID int) {
	h.ASMGroupID = groupID
}

// AddASMGroupToDisplay adds a new ASM group ID to be displayed
func (h *SMTPAPIHeader) AddASMGroupToDisplay(groupID int) {
	h.ASMGroups = append(h.ASMGroups, groupID)
}

// AddASMGroupsToDisplay adds multiple ASM group IDs to be displayed
func (h *SMTPAPIHeader) AddASMGroupsToDisplay(groupIDs []int) {
	for i := 0; i < len(groupIDs); i++ {
		h.AddASMGroupToDisplay(groupIDs[i])
	}
}

// SetASMGroupsToDisplay will set the value of the ASMGroups field
func (h *SMTPAPIHeader) SetASMGroupsToDisplay(groups []int) {
	h.ASMGroups = groups
}

// AddUniqueArg will set the value of a specific argument
func (h *SMTPAPIHeader) AddUniqueArg(arg, value string) {
	if h.UniqueArgs == nil {
		h.UniqueArgs = make(map[string]string)
	}
	h.UniqueArgs[arg] = value
}

// SetUniqueArgs will set the value of the Unique_args header
func (h *SMTPAPIHeader) SetUniqueArgs(args map[string]string) {
	h.UniqueArgs = args
}

// AddFilter will set the specific setting for a filter
func (h *SMTPAPIHeader) AddFilter(filter, setting string, value interface{}) {
	if h.Filters == nil {
		h.Filters = make(map[string]Filter)
	}
	if _, ok := h.Filters[filter]; !ok {
		h.Filters[filter] = Filter{
			Settings: make(map[string]interface{}),
		}
	}
	h.Filters[filter].Settings[setting] = value
}

// SetFilter takes in a Filter struct with predetermined settings and sets it for such Filter key
func (h *SMTPAPIHeader) SetFilter(filter string, value *Filter) {
	if h.Filters == nil {
		h.Filters = make(map[string]Filter)
	}
	h.Filters[filter] = *value
}

// SetSendAt takes in a timestamp which determines when the email will be sent
func (h *SMTPAPIHeader) SetSendAt(sendAt int64) {
	h.SendAt = sendAt
}

// AddSendEachAt takes in a timestamp and pushes it into a list Must match length of To emails
func (h *SMTPAPIHeader) AddSendEachAt(sendEachAt int64) {
	h.SendEachAt = append(h.SendEachAt, sendEachAt)
}

// SetSendEachAt takes an array of timestamps. Must match length of To emails
func (h *SMTPAPIHeader) SetSendEachAt(sendEachAt []int64) {
	h.SendEachAt = sendEachAt
}

// SetIpPool takes a strings and sets the IpPool field
func (h *SMTPAPIHeader) SetIpPool(ipPool string) {
	h.IpPool = ipPool
}

// Unicode escape
func escapeUnicode(input string) string {
	//var buffer bytes.Buffer
	buffer := bytes.NewBufferString("")
	for _, r := range input {
		if r > 65535 {
			// surrogate pair
			var r1, r2 = utf16.EncodeRune(r)
			var s = fmt.Sprintf("\\u%x\\u%x", r1, r2)
			buffer.WriteString(s)
		} else if r > 127 {
			var s = fmt.Sprintf("\\u%04x", r)
			buffer.WriteString(s)
		} else {
			var s = fmt.Sprintf("%c", r)
			buffer.WriteString(s)
		}
	}
	return buffer.String()
}

// JSONString returns the representation of the Header
func (h *SMTPAPIHeader) JSONString() (string, error) {
	headers, e := json.Marshal(h)
	return escapeUnicode(string(headers)), e
}

// Load allows you to load a pre-formed x-smtpapi header
func (h *SMTPAPIHeader) Load(b []byte) error {
	return json.Unmarshal(b, h)
}
<|endoftext|>


<commit_before>/*
Soultion for Project Euler Problem #17 - https://projecteuler.net/problem=17

(c) 2016 dpetker
*/
package main

import (
	"fmt"
)

var baseDict = map[int]int{
	1:    len("one"),
	2:    len("two"),
	3:    len("three"),
	4:    len("four"),
	5:    len("five"),
	6:    len("six"),
	7:    len("seven"),
	8:    len("eight"),
	9:    len("nine"),
	10:   len("ten"),
	11:   len("eleven"),
	12:   len("twelve"),
	13:   len("thirteen"),
	14:   len("fourteen"),
	15:   len("fifteen"),
	16:   len("sixteen"),
	17:   len("seventeen"),
	18:   len("eighteen"),
	19:   len("nineteen"),
	20:   len("twenty"),
	30:   len("thirty"),
	40:   len("fourty"),
	50:   len("fifty"),
	60:   len("sixty"),
	70:   len("seventy"),
	80:   len("eighty"),
	90:   len("ninety"),
	100:  len("hundred"),
	1000: len("thousand"),
}

func main() {
	sum := 0

	for i := 1; i < 1001; i++ {
		sum += findCharCount(i)
	}

	fmt.Println(sum)
}

func findCharCount(num int) int {
	if num <= 20 {
		return baseDict[num]
	}

	return 0
}
<commit_msg>Solution for problem 17<commit_after>/*
Soultion for Project Euler Problem #17 - https://projecteuler.net/problem=17

(c) 2016 dpetker
*/
package main

import (
	"fmt"
)

var baseDict = map[int]int{
	0:    0,
	1:    len("one"),
	2:    len("two"),
	3:    len("three"),
	4:    len("four"),
	5:    len("five"),
	6:    len("six"),
	7:    len("seven"),
	8:    len("eight"),
	9:    len("nine"),
	10:   len("ten"),
	11:   len("eleven"),
	12:   len("twelve"),
	13:   len("thirteen"),
	14:   len("fourteen"),
	15:   len("fifteen"),
	16:   len("sixteen"),
	17:   len("seventeen"),
	18:   len("eighteen"),
	19:   len("nineteen"),
	20:   len("twenty"),
	30:   len("thirty"),
	40:   len("forty"),
	50:   len("fifty"),
	60:   len("sixty"),
	70:   len("seventy"),
	80:   len("eighty"),
	90:   len("ninety"),
	100:  len("hundred"),
	1000: len("onethousand"),
}

func main() {
	sum := 0

	for i := 1; i < 1001; i++ {
		sum += findCharCount(i)
	}

	fmt.Println(sum)
}

func findCharCount(num int) int {
	switch {
	case num <= 20 || num == 1000:
		return baseDict[num]
	case num > 20 && num < 100:
		tens := (num / 10) * 10
		ones := num % 10
		return baseDict[tens] + baseDict[ones]
	case num >= 100 && num < 1000:
		hundreds := num / 100
		remainder := num % 100

		// +3 letters for "and" as in, "one hundred and forty two"
		tempSum := baseDict[hundreds] + baseDict[100] + 3 + findCharCount(remainder)

		if remainder == 0 {
			// No "and" when we're right on a hundred
			tempSum -= 3
		}

		return tempSum
	}

	return 0
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/papplampe/euler/sieve"
	"strconv"
)

var (
	primes []bool
)

const (
	N = 1000000
)

func rotateBytes(s []byte) {
	last := s[len(s)-1]
	for i := len(s) - 1; i > 0; i-- {
		s[i] = s[i-1]
	}
	s[0] = last
}

func isCircularPrime(n uint64) bool {
	if !primes[n] {
		return false
	}
	t := []byte(fmt.Sprint(n))
	if len(t) == 1 {
		return true
	}
	for i := 0; i < len(t); i++ {
		rotateBytes(t)
		x, _ := strconv.Atoi(string(t))
		if !primes[x] {
			return false
		}
	}
	return true
}

func main() {
	var i, s uint64
	primes = sieve.Sieve(N)
	for i = 1; i < N; i++ {
		if isCircularPrime(i) {
			fmt.Println(i)
			s++
		}
	}
	fmt.Println(s)
}
<commit_msg>35 blah<commit_after>package main

import (
	"fmt"
	"github.com/papplampe/euler/sieve"
	"strconv"
)

var (
	primes []bool
)

const (
	N = 1000000
)

func rotateBytes(s []byte) {
	last := s[len(s)-1]
	for i := len(s) - 1; i > 0; i-- {
		s[i] = s[i-1]
	}
	s[0] = last
}

func isCircularPrime(n uint64) bool {
	if !primes[n] {
		return false
	}
	t := []byte(fmt.Sprint(n))
	if len(t) == 1 {
		return true
	}
	for i := 0; i < len(t); i++ {
		rotateBytes(t)
		x, _ := strconv.Atoi(string(t))
		if !primes[x] {
			return false
		}
	}
	return true
}

func main() {
	var i, s uint64
	primes = sieve.Sieve(N)
	for i = 1; i < N; i++ {
		if isCircularPrime(i) {
			s++
		}
	}
	fmt.Println(s)
}
<|endoftext|>


<commit_before>package libytcd

import (
	"libGFC"
)

type Server struct {
	ports       []Port
	transaction chan TransactionError
	blocks      chan BlockError
	state       *libGFC.GFCChain
}

func NewServer(ports []Port) (s *Server) {
	s = new(Server)
	s.ports = ports
	s.state = libGFC.NewGFCChain()

	s.blocks = make(chan BlockError)
	s.transaction = make(chan TransactionError)

	go s.handleChannels()

	for _, p := range ports {
		p.AddTransactionChannel(s.transaction)
		p.AddBlockChannel(s.blocks)
	}

	return
}

func (s *Server) handleChannels() {
	for {
		select {
		case c := <-s.transaction:
			update := c.BlockMessage
			err := update.Verify(s.state)
			if err != nil {
				c.error <- err
			} else {
				for _, p := range s.ports {
					if p != c.Source {
						p.AddTransaction(update)
					}
				}
			}
		case block := <-s.blocks:
			for _, v := range block.BlockMessage {
				err := v.Verify(s.state)
				if err != nil {
					block.error <- err
					break
				}
				v.Apply(s.state)
			}

			for _, p := range s.ports {
				if p != block.Source {
					p.AddBlock(block.BlockMessage)
				}
			}
		}
	}
}
<commit_msg>Fix blocking bug in ytcd<commit_after>package libytcd

import (
	"libGFC"
)

type Server struct {
	ports       []Port
	transaction chan TransactionError
	blocks      chan BlockError
	state       *libGFC.GFCChain
}

func NewServer(ports []Port) (s *Server) {
	s = new(Server)
	s.ports = ports
	s.state = libGFC.NewGFCChain()

	s.blocks = make(chan BlockError)
	s.transaction = make(chan TransactionError)

	go s.handleChannels()

	for _, p := range ports {
		p.AddTransactionChannel(s.transaction)
		p.AddBlockChannel(s.blocks)
	}

	return
}

func (s *Server) handleChannels() {
	for {
		select {
		case c := <-s.transaction:
			update := c.BlockMessage
			err := update.Verify(s.state)
			if err != nil {
				c.error <- err
			} else {
				c.error <- nil
				for _, p := range s.ports {
					if p != c.Source {
						p.AddTransaction(update)
					}
				}
			}
		case block := <-s.blocks:
			var err error = nil
			for _, v := range block.BlockMessage {
				err = v.Verify(s.state)
				if err != nil {
					break
				}
				v.Apply(s.state)
			}
			block.error <- err

			for _, p := range s.ports {
				if p != block.Source {
					p.AddBlock(block.BlockMessage)
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2010 Brad Fitzpatrick. All rights reserved. See LICENSE file.
//
// This program mirrors FotoBilder galleries & pictures.  FotoBilder
// is the software the runs LiveJournal's photo galleries, and
// previously ran picpix.com, which is now apparently shutting down.
//
// This tool fetches public galleries & photos only when passwords
// aren't handy.
//
// If you have the passwords, use fotoup.pl and its --backup mode,
// which uses the authenticated API.

package main

import (
	"flag"
	"fmt"
	"http"
	"io/ioutil"
	"log"
	"os"
	"regexp"
	"sync"
	"time"
	"xml"
)

import _ "http/pprof"

var flagBase *string = flag.String("base", "",
	"e.g. http://www.picpix.com/username (no trailing slash)")

var flagDest *string = flag.String("dest", "", "Destination backup root")

var flagSloppy *bool = flag.Bool("sloppy", false, "Continue on errors")
var flagMaxNetwork *int = flag.Int("concurrency", 20, "Max concurrent requests")

var flagProfile *string = flag.String("profile", "",
	"the listen address to run a webserver for profiling; empty to leave disabled")

var galleryMutex sync.Mutex
var galleryMap map[string]*Gallery = make(map[string]*Gallery)

var picMutex sync.Mutex
var picMap map[string]*MediaSetItem = make(map[string]*MediaSetItem)

var networkOpGate chan bool

// Consult ulimit -n; you may have to up your
// /etc/security/limits.conf's nofile.
var localOpGate chan bool = make(chan bool, 10000)

var opsMutex sync.Mutex
var opsInFlight int

var errorMutex sync.Mutex
var errors []string = make([]string, 0)

var galleryPattern *regexp.Regexp = regexp.MustCompile(
	"/gallery/([0-9a-z][0-9a-z][0-9a-z][0-9a-z][0-9a-z][0-9a-z][0-9a-z][0-9a-z])")
var picPattern *regexp.Regexp = regexp.MustCompile(
	"/pic/([0-9a-z][0-9a-z][0-9a-z][0-9a-z][0-9a-z][0-9a-z][0-9a-z][0-9a-z])")

func addError(msg string) {
	errorMutex.Lock()
	defer errorMutex.Unlock()
	errors = append(errors, msg)
	if *flagSloppy {
		log.Printf("ERROR: %s", msg)
	} else {
		log.Exitf("ERROR: %s", msg)
	}
}

type Operation interface {
	Done()
}

type NetworkOperation int
type LocalOperation int

func NewLocalOperation() Operation {
	opsMutex.Lock()
	opsInFlight++
	opsMutex.Unlock()
	localOpGate <- true // buffer-limited, may/should block
	return LocalOperation(0)
}

func NewNetworkOperation() Operation {
	opsMutex.Lock()
	opsInFlight++
	opsMutex.Unlock()
	networkOpGate <- true
	return NetworkOperation(0)
}

func (o LocalOperation) Done() {
	<-localOpGate
	opsMutex.Lock()
	defer opsMutex.Unlock()
	opsInFlight--
}

func (o NetworkOperation) Done() {
	<-networkOpGate
	opsMutex.Lock()
	defer opsMutex.Unlock()
	opsInFlight--
}

func OperationsInFlight() int {
	opsMutex.Lock()
	defer opsMutex.Unlock()
	return opsInFlight
}

func fetchUrlToFile(url, filename string, expectedSize int64) bool {
	fi, statErr := os.Stat(filename)
	if statErr == nil &&
		(expectedSize == -1 && fi.Size > 0 ||
			expectedSize == fi.Size) {
		// TODO: re-fetch mode?
		return true
	}

	netop := NewNetworkOperation()
	defer netop.Done()

	res, _, err := http.Get(url)
	if err != nil {
		addError(fmt.Sprintf("Error fetching %s: %v", url, err))
		return false
	}
	defer res.Body.Close()

	fileBytes, err := ioutil.ReadAll(res.Body)
	if err != nil {
		addError(fmt.Sprintf("Error reading XML from %s: %v", url, err))
		return false
	}

	err = ioutil.WriteFile(filename, fileBytes, 0600)
	if err != nil {
		addError(fmt.Sprintf("Error writing file %s: %v", filename, err))
		return false
	}
	return true
}

type Gallery struct {
	key string
}

func (g *Gallery) XmlUrl() string {
	return fmt.Sprintf("%s/gallery/%s.xml", *flagBase, g.key)
}

func (g *Gallery) Fetch(op Operation) {
	defer op.Done()

	galXmlFilename := fmt.Sprintf("%s/gallery-%s.xml", *flagDest, g.key)
	if fetchUrlToFile(g.XmlUrl(), galXmlFilename, -1) {
		go fetchPhotosInGallery(galXmlFilename, NewLocalOperation())
	}
}

type DigestInfo struct {
	XMLName xml.Name "digest"
	Type    string   "attr"
	Value   string   "chardata"
}

type MediaFile struct {
	XMLName xml.Name "file"
	Digest  DigestInfo
	Mime    string
	Width   int
	Height  int
	Bytes   int64
	Url     string // the raw URL
}

type MediaSetItem struct {
	XMLName     xml.Name "mediaSetItem"
	Title       string
	Description string
	InfoURL     string // the xml URL
	File        MediaFile

	// Not in the XML:
	key string // the 8 chars
}

func (p *MediaSetItem) XmlUrl() string {
	return fmt.Sprintf("%s/pic/%s.xml", *flagBase, p.key)
}

func (p *MediaSetItem) BlobUrl() string {
	return fmt.Sprintf("%s/pic/%s", *flagBase, p.key)
}

func (p *MediaSetItem) XmlBackupFilename() string {
	return fmt.Sprintf("%s/pic-%s.xml", *flagDest, p.key)
}

func (p *MediaSetItem) BlobBackupFilename() string {
	var ext string
	switch p.File.Mime {
	case "image/jpeg":
		ext = "jpg"
	case "image/png":
		ext = "png"
	case "image/gif":
		ext = "gif"
	}
	return fmt.Sprintf("%s/pic-%s.%s", *flagDest, p.key, ext)
}

func (p *MediaSetItem) Fetch(op Operation) {
	defer op.Done()
	if !fetchUrlToFile(p.XmlUrl(), p.XmlBackupFilename(), -1) {
		return
	}

	if p.File.Bytes <= 0 {
		panic("expected file to have some known file size")
	}
	fetchUrlToFile(p.BlobUrl(), p.BlobBackupFilename(), p.File.Bytes)
}

type MediaSetItemsWrapper struct {
	XMLName      xml.Name "mediaSetItems"
	MediaSetItem []MediaSetItem
}

type LinkedFromSet struct {
	XMLName xml.Name "linkedFrom"
	InfoURL []string // xml gallery URLs of 'parent' galleries (not a DAG)
}

type LinkedToSet struct {
	XMLName xml.Name "linkedTo"
	InfoURL []string // xml gallery URLs of 'children' galleries (not a DAG)
}

type MediaSet struct {
	XMLName       xml.Name "mediaSet"
	MediaSetItems MediaSetItemsWrapper
	LinkedFrom    LinkedFromSet
	LinkedTo      LinkedToSet
}

func fetchPhotosInGallery(filename string, op Operation) {
	defer op.Done()

	f, err := os.Open(filename, os.O_RDONLY, 0)
	if err != nil {
		addError(fmt.Sprintf("Failed to open %s: %v", filename, err))
		return
	}
	defer f.Close()
	mediaSet := new(MediaSet)
	err = xml.Unmarshal(f, mediaSet)
	if err != nil {
		addError(fmt.Sprintf("Failed to unmarshal %s: %v", filename, err))
		return
	}

	// Learn about new galleries, potentially?
	for _, url := range mediaSet.LinkedFrom.InfoURL {
		noteGallery(url)
	}
	for _, url := range mediaSet.LinkedTo.InfoURL {
		noteGallery(url)
	}

	//log.Printf("Parse of %s is: %q", filename, mediaSet)
	for _, item := range mediaSet.MediaSetItems.MediaSetItem {
		item.key = findKey(item.InfoURL, picPattern)
		//log.Printf("   pic: %s", item.InfoURL)
		notePhoto(&item)
	}
}

func knownGalleries() int {
	galleryMutex.Lock()
	defer galleryMutex.Unlock()
	return len(galleryMap)
}

func findKey(keyOrUrl string, pattern *regexp.Regexp) string {
	if len(keyOrUrl) == 8 {
		return keyOrUrl
	}

	matches := pattern.FindStringSubmatch(keyOrUrl)
	if matches == nil {
		panic("Failed to parse: " + keyOrUrl)
	}
	if len(matches[1]) != 8 {
		panic("Expected match of 8 chars in " + keyOrUrl)
	}
	return matches[1]
}

func noteGallery(keyOrUrl string) {
	key := findKey(keyOrUrl, galleryPattern)
	galleryMutex.Lock()
	defer galleryMutex.Unlock()
	if _, known := galleryMap[key]; known {
		return
	}
	gallery := &Gallery{key}
	galleryMap[key] = gallery
	log.Printf("Gallery: %s", gallery.XmlUrl())
	go gallery.Fetch(NewLocalOperation())
}

func notePhoto(pic *MediaSetItem) {
	picMutex.Lock()
	defer picMutex.Unlock()
	if _, known := picMap[pic.key]; known {
		return
	}
	picMap[pic.key] = pic
	log.Printf("Photo: %s", pic.XmlUrl())
	go pic.Fetch(NewLocalOperation())
}

func fetchGalleryPage(page int) {
	log.Printf("Fetching gallery page %d", page)
	res, finalUrl, err := http.Get(fmt.Sprintf("%s/?sort=alpha&page=%d",
		*flagBase, page))
	if err != nil {
		log.Exitf("Error fetching gallery page %d: %v", page, err)
	}
	log.Printf("Fetched page %d: %s", page, finalUrl)
	htmlBytes, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Exitf("Error reading gallery page %d's HTML: %v", page, err)
	}
	res.Body.Close()

	html := string(htmlBytes)
	log.Printf("read %d bytes", len(html))

	matches := galleryPattern.FindAllStringSubmatch(html, -1)
	for _, match := range matches {
		noteGallery(match[1])
	}
}

func main() {
	flag.Parse()

	if *flagDest == "" {
		log.Exitf("No --dest given.")
	}
	if *flagBase == "" {
		log.Exitf("No --base URL given.")
	}

	networkOpGate = make(chan bool, *flagMaxNetwork)

	log.Printf("Starting.")

	if *flagProfile != "" {
		log.Printf("Listening on http://%s", *flagProfile)
		go http.ListenAndServe(*flagProfile, nil)
	}

	page := 1
	for {
		countBefore := knownGalleries()
		fetchGalleryPage(page)
		countAfter := knownGalleries()
		log.Printf("Galleries known: %d", countAfter)
		if countAfter == countBefore {
			log.Printf("No new galleries, stopping.")
			break
		}
		page++
	}

	for {
		n := OperationsInFlight()
		if n == 0 {
			break
		}
		log.Printf("%d Operations in-flight.  Waiting.", n)
		time.Sleep(5 * 1e9)
	}
	log.Printf("Done.")
}
<commit_msg>remove stuttering<commit_after>// Copyright 2010 Brad Fitzpatrick. All rights reserved. See LICENSE file.
//
// This program mirrors FotoBilder galleries & pictures.  FotoBilder
// is the software the runs LiveJournal's photo galleries, and
// previously ran picpix.com, which is now apparently shutting down.
//
// This tool fetches public galleries & photos only when passwords
// aren't handy.
//
// If you have the passwords, use fotoup.pl and its --backup mode,
// which uses the authenticated API.

package main

import (
	"flag"
	"fmt"
	"http"
	"io/ioutil"
	"log"
	"os"
	"regexp"
	"sync"
	"time"
	"xml"
)

import _ "http/pprof"

var flagBase = flag.String("base", "",
	"e.g. http://www.picpix.com/username (no trailing slash)")

var flagDest = flag.String("dest", "", "Destination backup root")

var flagSloppy = flag.Bool("sloppy", false, "Continue on errors")
var flagMaxNetwork = flag.Int("concurrency", 20, "Max concurrent requests")

var flagProfile = flag.String("profile", "",
	"the listen address to run a webserver for profiling; empty to leave disabled")

var galleryMutex sync.Mutex
var galleryMap = make(map[string]*Gallery)

var picMutex sync.Mutex
var picMap = make(map[string]*MediaSetItem)

var networkOpGate chan bool

// Consult ulimit -n; you may have to up your
// /etc/security/limits.conf's nofile.
var localOpGate = make(chan bool, 10000)

var opsMutex sync.Mutex
var opsInFlight int

var errorMutex sync.Mutex
var errors = make([]string, 0)

var galleryPattern = regexp.MustCompile(
	"/gallery/([0-9a-z][0-9a-z][0-9a-z][0-9a-z][0-9a-z][0-9a-z][0-9a-z][0-9a-z])")
var picPattern = regexp.MustCompile(
	"/pic/([0-9a-z][0-9a-z][0-9a-z][0-9a-z][0-9a-z][0-9a-z][0-9a-z][0-9a-z])")

func addError(msg string) {
	errorMutex.Lock()
	defer errorMutex.Unlock()
	errors = append(errors, msg)
	if *flagSloppy {
		log.Printf("ERROR: %s", msg)
	} else {
		log.Exitf("ERROR: %s", msg)
	}
}

type Operation interface {
	Done()
}

type NetworkOperation int
type LocalOperation int

func NewLocalOperation() Operation {
	opsMutex.Lock()
	opsInFlight++
	opsMutex.Unlock()
	localOpGate <- true // buffer-limited, may/should block
	return LocalOperation(0)
}

func NewNetworkOperation() Operation {
	opsMutex.Lock()
	opsInFlight++
	opsMutex.Unlock()
	networkOpGate <- true
	return NetworkOperation(0)
}

func (o LocalOperation) Done() {
	<-localOpGate
	opsMutex.Lock()
	defer opsMutex.Unlock()
	opsInFlight--
}

func (o NetworkOperation) Done() {
	<-networkOpGate
	opsMutex.Lock()
	defer opsMutex.Unlock()
	opsInFlight--
}

func OperationsInFlight() int {
	opsMutex.Lock()
	defer opsMutex.Unlock()
	return opsInFlight
}

func fetchUrlToFile(url, filename string, expectedSize int64) bool {
	fi, statErr := os.Stat(filename)
	if statErr == nil &&
		(expectedSize == -1 && fi.Size > 0 ||
			expectedSize == fi.Size) {
		// TODO: re-fetch mode?
		return true
	}

	netop := NewNetworkOperation()
	defer netop.Done()

	res, _, err := http.Get(url)
	if err != nil {
		addError(fmt.Sprintf("Error fetching %s: %v", url, err))
		return false
	}
	defer res.Body.Close()

	fileBytes, err := ioutil.ReadAll(res.Body)
	if err != nil {
		addError(fmt.Sprintf("Error reading XML from %s: %v", url, err))
		return false
	}

	err = ioutil.WriteFile(filename, fileBytes, 0600)
	if err != nil {
		addError(fmt.Sprintf("Error writing file %s: %v", filename, err))
		return false
	}
	return true
}

type Gallery struct {
	key string
}

func (g *Gallery) XmlUrl() string {
	return fmt.Sprintf("%s/gallery/%s.xml", *flagBase, g.key)
}

func (g *Gallery) Fetch(op Operation) {
	defer op.Done()

	galXmlFilename := fmt.Sprintf("%s/gallery-%s.xml", *flagDest, g.key)
	if fetchUrlToFile(g.XmlUrl(), galXmlFilename, -1) {
		go fetchPhotosInGallery(galXmlFilename, NewLocalOperation())
	}
}

type DigestInfo struct {
	XMLName xml.Name "digest"
	Type    string   "attr"
	Value   string   "chardata"
}

type MediaFile struct {
	XMLName xml.Name "file"
	Digest  DigestInfo
	Mime    string
	Width   int
	Height  int
	Bytes   int64
	Url     string // the raw URL
}

type MediaSetItem struct {
	XMLName     xml.Name "mediaSetItem"
	Title       string
	Description string
	InfoURL     string // the xml URL
	File        MediaFile

	// Not in the XML:
	key string // the 8 chars
}

func (p *MediaSetItem) XmlUrl() string {
	return fmt.Sprintf("%s/pic/%s.xml", *flagBase, p.key)
}

func (p *MediaSetItem) BlobUrl() string {
	return fmt.Sprintf("%s/pic/%s", *flagBase, p.key)
}

func (p *MediaSetItem) XmlBackupFilename() string {
	return fmt.Sprintf("%s/pic-%s.xml", *flagDest, p.key)
}

func (p *MediaSetItem) BlobBackupFilename() string {
	var ext string
	switch p.File.Mime {
	case "image/jpeg":
		ext = "jpg"
	case "image/png":
		ext = "png"
	case "image/gif":
		ext = "gif"
	}
	return fmt.Sprintf("%s/pic-%s.%s", *flagDest, p.key, ext)
}

func (p *MediaSetItem) Fetch(op Operation) {
	defer op.Done()
	if !fetchUrlToFile(p.XmlUrl(), p.XmlBackupFilename(), -1) {
		return
	}

	if p.File.Bytes <= 0 {
		panic("expected file to have some known file size")
	}
	fetchUrlToFile(p.BlobUrl(), p.BlobBackupFilename(), p.File.Bytes)
}

type MediaSetItemsWrapper struct {
	XMLName      xml.Name "mediaSetItems"
	MediaSetItem []MediaSetItem
}

type LinkedFromSet struct {
	XMLName xml.Name "linkedFrom"
	InfoURL []string // xml gallery URLs of 'parent' galleries (not a DAG)
}

type LinkedToSet struct {
	XMLName xml.Name "linkedTo"
	InfoURL []string // xml gallery URLs of 'children' galleries (not a DAG)
}

type MediaSet struct {
	XMLName       xml.Name "mediaSet"
	MediaSetItems MediaSetItemsWrapper
	LinkedFrom    LinkedFromSet
	LinkedTo      LinkedToSet
}

func fetchPhotosInGallery(filename string, op Operation) {
	defer op.Done()

	f, err := os.Open(filename, os.O_RDONLY, 0)
	if err != nil {
		addError(fmt.Sprintf("Failed to open %s: %v", filename, err))
		return
	}
	defer f.Close()
	mediaSet := new(MediaSet)
	err = xml.Unmarshal(f, mediaSet)
	if err != nil {
		addError(fmt.Sprintf("Failed to unmarshal %s: %v", filename, err))
		return
	}

	// Learn about new galleries, potentially?
	for _, url := range mediaSet.LinkedFrom.InfoURL {
		noteGallery(url)
	}
	for _, url := range mediaSet.LinkedTo.InfoURL {
		noteGallery(url)
	}

	//log.Printf("Parse of %s is: %q", filename, mediaSet)
	for _, item := range mediaSet.MediaSetItems.MediaSetItem {
		item.key = findKey(item.InfoURL, picPattern)
		//log.Printf("   pic: %s", item.InfoURL)
		notePhoto(&item)
	}
}

func knownGalleries() int {
	galleryMutex.Lock()
	defer galleryMutex.Unlock()
	return len(galleryMap)
}

func findKey(keyOrUrl string, pattern *regexp.Regexp) string {
	if len(keyOrUrl) == 8 {
		return keyOrUrl
	}

	matches := pattern.FindStringSubmatch(keyOrUrl)
	if matches == nil {
		panic("Failed to parse: " + keyOrUrl)
	}
	if len(matches[1]) != 8 {
		panic("Expected match of 8 chars in " + keyOrUrl)
	}
	return matches[1]
}

func noteGallery(keyOrUrl string) {
	key := findKey(keyOrUrl, galleryPattern)
	galleryMutex.Lock()
	defer galleryMutex.Unlock()
	if _, known := galleryMap[key]; known {
		return
	}
	gallery := &Gallery{key}
	galleryMap[key] = gallery
	log.Printf("Gallery: %s", gallery.XmlUrl())
	go gallery.Fetch(NewLocalOperation())
}

func notePhoto(pic *MediaSetItem) {
	picMutex.Lock()
	defer picMutex.Unlock()
	if _, known := picMap[pic.key]; known {
		return
	}
	picMap[pic.key] = pic
	log.Printf("Photo: %s", pic.XmlUrl())
	go pic.Fetch(NewLocalOperation())
}

func fetchGalleryPage(page int) {
	log.Printf("Fetching gallery page %d", page)
	res, finalUrl, err := http.Get(fmt.Sprintf("%s/?sort=alpha&page=%d",
		*flagBase, page))
	if err != nil {
		log.Exitf("Error fetching gallery page %d: %v", page, err)
	}
	log.Printf("Fetched page %d: %s", page, finalUrl)
	htmlBytes, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Exitf("Error reading gallery page %d's HTML: %v", page, err)
	}
	res.Body.Close()

	html := string(htmlBytes)
	log.Printf("read %d bytes", len(html))

	matches := galleryPattern.FindAllStringSubmatch(html, -1)
	for _, match := range matches {
		noteGallery(match[1])
	}
}

func main() {
	flag.Parse()

	if *flagDest == "" {
		log.Exitf("No --dest given.")
	}
	if *flagBase == "" {
		log.Exitf("No --base URL given.")
	}

	networkOpGate = make(chan bool, *flagMaxNetwork)

	log.Printf("Starting.")

	if *flagProfile != "" {
		log.Printf("Listening on http://%s", *flagProfile)
		go http.ListenAndServe(*flagProfile, nil)
	}

	page := 1
	for {
		countBefore := knownGalleries()
		fetchGalleryPage(page)
		countAfter := knownGalleries()
		log.Printf("Galleries known: %d", countAfter)
		if countAfter == countBefore {
			log.Printf("No new galleries, stopping.")
			break
		}
		page++
	}

	for {
		n := OperationsInFlight()
		if n == 0 {
			break
		}
		log.Printf("%d Operations in-flight.  Waiting.", n)
		time.Sleep(5 * 1e9)
	}
	log.Printf("Done.")
}
<|endoftext|>