<commit_before>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package atom

import (
	"context"

	"github.com/google/gapid/core/data/protoconv"
	"github.com/google/gapid/core/log"
	"github.com/google/gapid/gapis/atom/atom_pb"
	"github.com/google/gapid/gapis/gfxapi/core/core_pb"
)

// ProtoToAtom returns a function that converts all the storage atoms it is
// handed, passing the generated live atoms to the handler.
// You must call this with a nil to flush the final atom.
func ProtoToAtom(handler func(a Atom)) func(context.Context, atom_pb.Atom) error {
	var (
		last         Atom
		observations *Observations
		invoked      bool
		count        int
	)
	var threadID uint64
	return func(ctx context.Context, in atom_pb.Atom) error {
		count++
		if in == nil {
			if last != nil {
				handler(last)
			}
			last = nil
			return nil
		}

		if in, ok := in.(*core_pb.SwitchThread); ok {
			threadID = in.ThreadID
		}

		out, err := protoconv.ToObject(ctx, in)
		if err != nil {
			return nil
		}
		switch out := out.(type) {
		case Atom:
			if last != nil {
				handler(last)
			}
			last = out
			invoked = false
			observations = nil
			out.SetThread(threadID)

		case Observation:
			if observations == nil {
				observations = &Observations{}
				e := last.Extras()
				if e == nil {
					return log.Errf(ctx, nil, "Not allowed extras %T:%v", last, last)
				}
				*e = append(*e, observations)
			}
			if !invoked {
				observations.Reads = append(observations.Reads, out)
			} else {
				observations.Writes = append(observations.Writes, out)
			}
		case invokeMarker:
			invoked = true
		case Extra:
			e := last.Extras()
			if e == nil {
				return log.Errf(ctx, nil, "Not allowed extras %T:%v", last, last)
			}
			*e = append(*e, out)
		default:
			return log.Errf(ctx, nil, "Unhandled type during conversion %T:%v", out, out)
		}
		return nil
	}
}

// AtomToProto returns a function that converts all the atoms it is handed,
// passing the generated proto atoms to the handler.
func AtomToProto(handler func(a atom_pb.Atom)) func(context.Context, Atom) error {
	return func(ctx context.Context, in Atom) error {
		out, err := protoconv.ToProto(ctx, in)
		if err != nil {
			return err
		}
		handler(out)

		for _, e := range in.Extras().All() {
			switch e := e.(type) {
			case Observations:
				for _, o := range e.Reads {
					p, err := protoconv.ToProto(ctx, o)
					if err != nil {
						return nil
					}
					handler(p)
				}
				handler(atom_pb.InvokeMarker)
				for _, o := range e.Writes {
					p, err := protoconv.ToProto(ctx, o)
					if err != nil {
						return nil
					}
					handler(p)
				}
			default:
				p, err := protoconv.ToProto(ctx, e)
				if err != nil {
					return nil
				}
				handler(p)
			}
		}

		return nil
	}
}

type invokeMarker struct{}

func init() {
	protoconv.Register(
		func(ctx context.Context, a *invokeMarker) (*atom_pb.Invoke, error) {
			return &atom_pb.Invoke{}, nil
		},
		func(ctx context.Context, a *atom_pb.Invoke) (*invokeMarker, error) {
			return &invokeMarker{}, nil
		},
	)
}
<commit_msg>gapis/atom: Fix invokeMarker type mismatch<commit_after>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package atom

import (
	"context"

	"github.com/google/gapid/core/data/protoconv"
	"github.com/google/gapid/core/log"
	"github.com/google/gapid/gapis/atom/atom_pb"
	"github.com/google/gapid/gapis/gfxapi/core/core_pb"
)

// ProtoToAtom returns a function that converts all the storage atoms it is
// handed, passing the generated live atoms to the handler.
// You must call this with a nil to flush the final atom.
func ProtoToAtom(handler func(a Atom)) func(context.Context, atom_pb.Atom) error {
	var (
		last         Atom
		observations *Observations
		invoked      bool
		count        int
	)
	var threadID uint64
	return func(ctx context.Context, in atom_pb.Atom) error {
		count++
		if in == nil {
			if last != nil {
				handler(last)
			}
			last = nil
			return nil
		}

		if in, ok := in.(*core_pb.SwitchThread); ok {
			threadID = in.ThreadID
		}

		out, err := protoconv.ToObject(ctx, in)
		if err != nil {
			return nil
		}
		switch out := out.(type) {
		case Atom:
			if last != nil {
				handler(last)
			}
			last = out
			invoked = false
			observations = nil
			out.SetThread(threadID)

		case Observation:
			if observations == nil {
				observations = &Observations{}
				e := last.Extras()
				if e == nil {
					return log.Errf(ctx, nil, "Not allowed extras %T:%v", last, last)
				}
				*e = append(*e, observations)
			}
			if !invoked {
				observations.Reads = append(observations.Reads, out)
			} else {
				observations.Writes = append(observations.Writes, out)
			}
		case *invokeMarker:
			invoked = true
		case Extra:
			e := last.Extras()
			if e == nil {
				return log.Errf(ctx, nil, "Not allowed extras %T:%v", last, last)
			}
			*e = append(*e, out)
		default:
			return log.Errf(ctx, nil, "Unhandled type during conversion %T:%v", out, out)
		}
		return nil
	}
}

// AtomToProto returns a function that converts all the atoms it is handed,
// passing the generated proto atoms to the handler.
func AtomToProto(handler func(a atom_pb.Atom)) func(context.Context, Atom) error {
	return func(ctx context.Context, in Atom) error {
		out, err := protoconv.ToProto(ctx, in)
		if err != nil {
			return err
		}
		handler(out)

		for _, e := range in.Extras().All() {
			switch e := e.(type) {
			case Observations:
				for _, o := range e.Reads {
					p, err := protoconv.ToProto(ctx, o)
					if err != nil {
						return nil
					}
					handler(p)
				}
				handler(atom_pb.InvokeMarker)
				for _, o := range e.Writes {
					p, err := protoconv.ToProto(ctx, o)
					if err != nil {
						return nil
					}
					handler(p)
				}
			default:
				p, err := protoconv.ToProto(ctx, e)
				if err != nil {
					return nil
				}
				handler(p)
			}
		}

		return nil
	}
}

type invokeMarker struct{}

func init() {
	protoconv.Register(
		func(ctx context.Context, a *invokeMarker) (*atom_pb.Invoke, error) {
			return &atom_pb.Invoke{}, nil
		},
		func(ctx context.Context, a *atom_pb.Invoke) (*invokeMarker, error) {
			return &invokeMarker{}, nil
		},
	)
}
<|endoftext|>


<commit_before>// Copyright 2018 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package main contains an agent for connecting to a Swarming server.
package main

import (
	"bytes"
	"context"
	"os"
	"text/template"

	"cloud.google.com/go/compute/metadata"
	"github.com/maruel/subcommands"

	"go.chromium.org/luci/auth"
	"go.chromium.org/luci/auth/client/authcli"
	"go.chromium.org/luci/common/cli"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/logging/gologger"
	"go.chromium.org/luci/grpc/prpc"
	"go.chromium.org/luci/hardcoded/chromeinfra"

	"go.chromium.org/luci/gce/api/instances/v1"
	"go.chromium.org/luci/gce/vmtoken/client"
)

// substitute performs substitutions in a template string.
func substitute(c context.Context, s string, subs interface{}) (string, error) {
	t, err := template.New("tmpl").Parse(s)
	if err != nil {
		return "", err
	}
	buf := bytes.Buffer{}
	if err = t.Execute(&buf, subs); err != nil {
		return "", nil
	}
	return buf.String(), nil
}

// metaKey is the key to a *metadata.Client in the context.
var metaKey = "meta"

// withMetadata returns a new context with the given *metadata.Client installed.
func withMetadata(c context.Context, cli *metadata.Client) context.Context {
	return context.WithValue(c, &metaKey, cli)
}

// getMetadata returns the *metadata.Client installed in the current context.
func getMetadata(c context.Context) *metadata.Client {
	return c.Value(&metaKey).(*metadata.Client)
}

// newInstances returns a new instances.InstancesClient.
func newInstances(c context.Context, acc, host string) instances.InstancesClient {
	return instances.NewInstancesPRPCClient(&prpc.Client{
		C:    client.NewClient(getMetadata(c), acc),
		Host: host,
	})
}

// cmdRunBase is the base struct all subcommands should embed.
// Implements cli.ContextModificator.
type cmdRunBase struct {
	subcommands.CommandRunBase
	authFlags      authcli.Flags
	serviceAccount string
}

// Initialize registers common flags.
func (b *cmdRunBase) Initialize() {
	opts := chromeinfra.DefaultAuthOptions()
	b.authFlags.Register(b.GetFlags(), opts)
}

// ModifyContext returns a new context to be used by all commands. Implements
// cli.ContextModificator.
func (b *cmdRunBase) ModifyContext(c context.Context) context.Context {
	c = logging.SetLevel(gologger.StdConfig.Use(c), logging.Debug)
	opts, err := b.authFlags.Options()
	if err != nil {
		logging.Errorf(c, "%s", err.Error())
		panic("failed to get auth options")
	}
	b.serviceAccount = opts.GCEAccountName
	http, err := auth.NewAuthenticator(c, auth.OptionalLogin, opts).Client()
	if err != nil {
		logging.Errorf(c, "%s", err.Error())
		panic("failed to get authenticator")
	}
	meta := metadata.NewClient(http)
	swr := &SwarmingClient{
		Client:           http,
		PlatformStrategy: newStrategy(),
	}
	return withSwarming(withMetadata(c, meta), swr)
}

// New returns a new agent application.
func New() *cli.Application {
	return &cli.Application{
		Name:  "agent",
		Title: "GCE agent",
		Commands: []*subcommands.Command{
			subcommands.CmdHelp,
			newConnectCmd(),
		},
	}
}

func main() {
	os.Exit(subcommands.Run(New(), os.Args[1:]))
}
<commit_msg>gce-agent: use non-default user client.<commit_after>// Copyright 2018 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package main contains an agent for connecting to a Swarming server.
package main

import (
	"bytes"
	"context"
	"os"
	"text/template"

	"cloud.google.com/go/compute/metadata"
	"github.com/maruel/subcommands"

	"go.chromium.org/luci/auth"
	"go.chromium.org/luci/auth/client/authcli"
	"go.chromium.org/luci/common/cli"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/logging/gologger"
	"go.chromium.org/luci/grpc/prpc"
	"go.chromium.org/luci/hardcoded/chromeinfra"

	"go.chromium.org/luci/gce/api/instances/v1"
	"go.chromium.org/luci/gce/vmtoken/client"
)

// substitute performs substitutions in a template string.
func substitute(c context.Context, s string, subs interface{}) (string, error) {
	t, err := template.New("tmpl").Parse(s)
	if err != nil {
		return "", err
	}
	buf := bytes.Buffer{}
	if err = t.Execute(&buf, subs); err != nil {
		return "", nil
	}
	return buf.String(), nil
}

// metaKey is the key to a *metadata.Client in the context.
var metaKey = "meta"

// withMetadata returns a new context with the given *metadata.Client installed.
func withMetadata(c context.Context, cli *metadata.Client) context.Context {
	return context.WithValue(c, &metaKey, cli)
}

// getMetadata returns the *metadata.Client installed in the current context.
func getMetadata(c context.Context) *metadata.Client {
	return c.Value(&metaKey).(*metadata.Client)
}

// newInstances returns a new instances.InstancesClient.
func newInstances(c context.Context, acc, host string) instances.InstancesClient {
	options := prpc.DefaultOptions()
	// TODO(tandrii): report CIPD package version.
	options.UserAgent = "gce-agent, v1"
	return instances.NewInstancesPRPCClient(&prpc.Client{
		C:       client.NewClient(getMetadata(c), acc),
		Host:    host,
		Options: options,
	})
}

// cmdRunBase is the base struct all subcommands should embed.
// Implements cli.ContextModificator.
type cmdRunBase struct {
	subcommands.CommandRunBase
	authFlags      authcli.Flags
	serviceAccount string
}

// Initialize registers common flags.
func (b *cmdRunBase) Initialize() {
	opts := chromeinfra.DefaultAuthOptions()
	b.authFlags.Register(b.GetFlags(), opts)
}

// ModifyContext returns a new context to be used by all commands. Implements
// cli.ContextModificator.
func (b *cmdRunBase) ModifyContext(c context.Context) context.Context {
	c = logging.SetLevel(gologger.StdConfig.Use(c), logging.Debug)
	opts, err := b.authFlags.Options()
	if err != nil {
		logging.Errorf(c, "%s", err.Error())
		panic("failed to get auth options")
	}
	b.serviceAccount = opts.GCEAccountName
	http, err := auth.NewAuthenticator(c, auth.OptionalLogin, opts).Client()
	if err != nil {
		logging.Errorf(c, "%s", err.Error())
		panic("failed to get authenticator")
	}
	meta := metadata.NewClient(http)
	swr := &SwarmingClient{
		Client:           http,
		PlatformStrategy: newStrategy(),
	}
	return withSwarming(withMetadata(c, meta), swr)
}

// New returns a new agent application.
func New() *cli.Application {
	return &cli.Application{
		Name:  "agent",
		Title: "GCE agent",
		Commands: []*subcommands.Command{
			subcommands.CmdHelp,
			newConnectCmd(),
		},
	}
}

func main() {
	os.Exit(subcommands.Run(New(), os.Args[1:]))
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/xconstruct/go-pushbullet"
	"log"
	"os"
)

type Config struct {
	ApiKey  string   `json:"api_key"`
	Devices []Device `json:"devices"`
}

type Device struct {
	Id   int    `json:"id"`
	Name string `json:"name"`
}

func getArg(i int, fallback string) string {
	if len(os.Args) <= i {
		return ""
	}
	return os.Args[i]
}

func main() {
	cmd := getArg(1, "")

	switch cmd {
	case "login":
		login()
	case "note":
		pushNote()
	default:
		printHelp()
	}
}

func login() {
	key := getArg(2, "")
	var cfg Config

	cfg.ApiKey = key
	cfg.Devices = make([]Device, 0)

	if key == "" {
		writeConfig(cfg)
		return
	}

	pb := pushbullet.New(key)
	devs, err := pb.Devices()
	if err != nil {
		log.Fatalln(err)
	}
	for _, dev := range devs {
		name := dev.Extras.Nickname
		if name == "" {
			name = dev.Extras.Model
		}
		cfg.Devices = append(cfg.Devices, Device{
			Id:   dev.Id,
			Name: name,
		})
	}
	writeConfig(cfg)
}

func readConfig() (Config, error) {
	path := os.Getenv("XDG_CONFIG_HOME") + "/pushb"
	f, err := os.Open(path + "/config.json")
	if err != nil {
		return Config{}, err
	}
	defer func() {
		f.Close()
	}()

	var cfg Config
	dec := json.NewDecoder(f)
	if err = dec.Decode(&cfg); err != nil {
		return Config{}, err
	}
	return cfg, nil
}

func writeConfig(cfg Config) {
	path := os.Getenv("XDG_CONFIG_HOME") + "/pushb"
	f, err := os.OpenFile(path+"/config.json", os.O_WRONLY|os.O_CREATE, 0600)
	if err != nil {
		log.Fatalln(err)
	}
	defer func() {
		f.Close()
	}()

	enc := json.NewEncoder(f)
	if err = enc.Encode(cfg); err != nil {
		log.Fatalln(err)
	}
}

func pushNote() {
	cfg, err := readConfig()
	if err != nil {
		log.Fatalln(err)
	}

	title := getArg(2, "")
	body := getArg(3, "")
	pb := pushbullet.New(cfg.ApiKey)
	err = pb.PushNote(cfg.Devices[0].Id, title, body)
	if err != nil {
		log.Fatalln(err)
	}
}

func printHelp() {
	topic := getArg(2, "")

	switch topic {
	default:
		fmt.Println(`Pushb is a simple client for PushBullet.

Usage:
    pushb command [flags] [arguments]

Commands:
    login      Saves the api key in the config
    devices    Shows a list of registered devices
	help       Shows this help

    address    Pushes an address to a device
    link       Pushes a link to a device
    list       Pushes a list to a device
    note       Pushes a note to a device

Use "pushb help [topic] for more information about that topic.`)
	}
}
<commit_msg>pushb: fix formatting of help message<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/xconstruct/go-pushbullet"
	"log"
	"os"
)

type Config struct {
	ApiKey  string   `json:"api_key"`
	Devices []Device `json:"devices"`
}

type Device struct {
	Id   int    `json:"id"`
	Name string `json:"name"`
}

func getArg(i int, fallback string) string {
	if len(os.Args) <= i {
		return ""
	}
	return os.Args[i]
}

func main() {
	cmd := getArg(1, "")

	switch cmd {
	case "login":
		login()
	case "note":
		pushNote()
	default:
		printHelp()
	}
}

func login() {
	key := getArg(2, "")
	var cfg Config

	cfg.ApiKey = key
	cfg.Devices = make([]Device, 0)

	if key == "" {
		writeConfig(cfg)
		return
	}

	pb := pushbullet.New(key)
	devs, err := pb.Devices()
	if err != nil {
		log.Fatalln(err)
	}
	for _, dev := range devs {
		name := dev.Extras.Nickname
		if name == "" {
			name = dev.Extras.Model
		}
		cfg.Devices = append(cfg.Devices, Device{
			Id:   dev.Id,
			Name: name,
		})
	}
	writeConfig(cfg)
}

func readConfig() (Config, error) {
	path := os.Getenv("XDG_CONFIG_HOME") + "/pushb"
	f, err := os.Open(path + "/config.json")
	if err != nil {
		return Config{}, err
	}
	defer func() {
		f.Close()
	}()

	var cfg Config
	dec := json.NewDecoder(f)
	if err = dec.Decode(&cfg); err != nil {
		return Config{}, err
	}
	return cfg, nil
}

func writeConfig(cfg Config) {
	path := os.Getenv("XDG_CONFIG_HOME") + "/pushb"
	f, err := os.OpenFile(path+"/config.json", os.O_WRONLY|os.O_CREATE, 0600)
	if err != nil {
		log.Fatalln(err)
	}
	defer func() {
		f.Close()
	}()

	enc := json.NewEncoder(f)
	if err = enc.Encode(cfg); err != nil {
		log.Fatalln(err)
	}
}

func pushNote() {
	cfg, err := readConfig()
	if err != nil {
		log.Fatalln(err)
	}

	title := getArg(2, "")
	body := getArg(3, "")
	pb := pushbullet.New(cfg.ApiKey)
	err = pb.PushNote(cfg.Devices[0].Id, title, body)
	if err != nil {
		log.Fatalln(err)
	}
}

func printHelp() {
	topic := getArg(2, "")

	switch topic {
	default:
		fmt.Println(`Pushb is a simple client for PushBullet.

Usage:
    pushb command [flags] [arguments]

Commands:
    login      Saves the api key in the config
    devices    Shows a list of registered devices
    help       Shows this help

    address    Pushes an address to a device
    link       Pushes a link to a device
    list       Pushes a list to a device
    note       Pushes a note to a device

Use "pushb help [topic] for more information about that topic.`)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"../geolib"
	"bufio"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"
)

type GeoMetaData struct {
	TimeStamps      []string         `json:"timestamps"`
	FileNameFields map[string]string `json:"filename_fields"`
	Polygon        json.RawMessage   `json:"polygon"`
	RasterCount  int       `json:"raster_count"`
	Type           string            `json:"array_type"`
	XSize          int               `json:"x_size"`
	YSize          int               `json:"y_size"`
	ProjWKT        string            `json:"proj_wkt"`
	GeoTransform   []float64         `json:"geotransform"`
}

var parserStrings map[string]string = map[string]string{"landsat": `LC(?P<mission>\d)(?P<path>\d\d\d)(?P<row>\d\d\d)(?P<year>\d\d\d\d)(?P<julian_day>\d\d\d)(?P<processing_level>[a-zA-Z0-9]+)_(?P<band>[a-zA-Z0-9]+)`,
				  "modis1": `M(?P<satellite>[OD|YD])(?P<product>[0-9]+_[A-Z0-9]+).A[0-9]+.[0-9]+.(?P<collection_version>\d\d\d).(?P<year>\d\d\d\d)(?P<julian_day>\d\d\d)(?P<hour>\d\d)(?P<minute>\d\d)(?P<second>\d\d)`,
				  "modis2": `MCD43A4.A[0-9]+.(?P<horizontal>h\d\d)(?P<vertical>v\d\d).(?P<resolution>\d\d\d).(?P<year>\d\d\d\d)(?P<julian_day>\d\d\d)(?P<hour>\d\d)(?P<minute>\d\d)(?P<second>\d\d)`,
				  "agdc_landsat1": `LS(?P<mission>\d)_(?P<sensor>[A-Z]+)_(?P<correction>[A-Z]+)_(?P<epsg>\d+)_(?P<x_coord>-?\d+)_(?P<y_coord>-?\d+)_(?P<year>\d\d\d\d).`,}

var parsers map[string]*regexp.Regexp = map[string]*regexp.Regexp{}
//var timeExtractors map[string]func(map[string] string) time.Time = map[string]func(map[string] string) time.Time{"landsat":landsatTime, "modis1": modisTime, "modis2": modisTime}

func init() {
	for key, value := range(parserStrings) {
		parsers[key] = regexp.MustCompile(value)
	}
}

func parseName(filePath string) (map[string]string, time.Time) {

	for _, r := range(parsers) {
		_, fileName := filepath.Split(filePath)

		if (r.MatchString(fileName)) {
			match := r.FindStringSubmatch(fileName)

			result := make(map[string]string)
			for i, name := range r.SubexpNames() {
				if i != 0 {
					result[name] = match[i]
				}
			}

			return result, parseTime(result)
		}
	}
	return nil, time.Time{}
}

func parseTime(nameFields map[string]string) time.Time {
	if _, ok := nameFields["year"]; ok {
		year, _ := strconv.Atoi(nameFields["year"])
		t := time.Date(year, 0, 0, 0, 0, 0, 0, time.UTC)
		if _, ok := nameFields["julian_day"]; ok {
			julianDay, _ := strconv.Atoi(nameFields["julian_day"])
			t = t.Add(time.Hour * 24 * time.Duration(julianDay))
		}
		if _, ok := nameFields["hour"]; ok {
			hour, _ := strconv.Atoi(nameFields["hour"])
			t = t.Add(time.Hour * time.Duration(hour))
		}
		if _, ok := nameFields["minute"]; ok {
			minute, _ := strconv.Atoi(nameFields["minute"])
			t = t.Add(time.Minute * time.Duration(minute))
		}
		if _, ok := nameFields["second"]; ok {
			second, _ := strconv.Atoi(nameFields["second"])
			t = t.Add(time.Second * time.Duration(second))
		}

		return t
	}
	return time.Time{}
}

func main() {
	s := bufio.NewScanner(os.Stdin)
	for s.Scan() {
		parts := strings.Split(s.Text(), "\t")
		if len(parts) != 2 {
			fmt.Printf("Input not recognised: %s\n", s.Text())
		}

		gdalFile := geolib.GDALFile{}
		err := json.Unmarshal([]byte(parts[1]), &gdalFile)
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}

		nameFields, timeStamp := parseName(parts[0])

		for _, ds := range gdalFile.DataSets {
			if ds.ProjWKT != "" {
				poly := geolib.GetPolygon(ds.ProjWKT, ds.GeoTransform, ds.XSize, ds.YSize)
				polyWGS84 := poly.ReprojectToWGS84()

				var times []string
				if nc_times, ok := ds.Extras["nc_times"]; ok {
					times = nc_times
				} else {
					times = []string{timeStamp.Format("2006-01-02T15:04:05Z")}
				}

				fileMetaData := GeoMetaData{TimeStamps: times, FileNameFields: nameFields, Polygon: json.RawMessage(polyWGS84.ToGeoJSON()),
					RasterCount: ds.RasterCount, Type: ds.Type, XSize: ds.XSize, YSize: ds.YSize, ProjWKT: ds.ProjWKT, GeoTransform: ds.GeoTransform}

				out, err := json.Marshal(&fileMetaData)
				if err != nil {
					fmt.Println(err)
					os.Exit(1)
				}
				fmt.Printf("%s\t%s\n", ds.DataSetName, string(out))
			}
		}
	}
}
<commit_msg>parser datasets aggregated<commit_after>package main

import (
	"../geolib"
	"bufio"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"
)

type GeoMetaData struct {
	DataSetName  string    `json:"ds_name"`
	TimeStamps      []string         `json:"timestamps"`
	FileNameFields map[string]string `json:"filename_fields"`
	Polygon        json.RawMessage   `json:"polygon"`
	RasterCount  int       `json:"raster_count"`
	Type           string            `json:"array_type"`
	XSize          int               `json:"x_size"`
	YSize          int               `json:"y_size"`
	ProjWKT        string            `json:"proj_wkt"`
	GeoTransform   []float64         `json:"geotransform"`
}

type GeoFile struct {
	Driver   string        `json:"file_type"`
	DataSets []GeoMetaData `json:"geo_metadata"`
}

var parserStrings map[string]string = map[string]string{"landsat": `LC(?P<mission>\d)(?P<path>\d\d\d)(?P<row>\d\d\d)(?P<year>\d\d\d\d)(?P<julian_day>\d\d\d)(?P<processing_level>[a-zA-Z0-9]+)_(?P<band>[a-zA-Z0-9]+)`,
				  "modis1": `M(?P<satellite>[OD|YD])(?P<product>[0-9]+_[A-Z0-9]+).A[0-9]+.[0-9]+.(?P<collection_version>\d\d\d).(?P<year>\d\d\d\d)(?P<julian_day>\d\d\d)(?P<hour>\d\d)(?P<minute>\d\d)(?P<second>\d\d)`,
				  "modis2": `MCD43A4.A[0-9]+.(?P<horizontal>h\d\d)(?P<vertical>v\d\d).(?P<resolution>\d\d\d).(?P<year>\d\d\d\d)(?P<julian_day>\d\d\d)(?P<hour>\d\d)(?P<minute>\d\d)(?P<second>\d\d)`,
				  "agdc_landsat1": `LS(?P<mission>\d)_(?P<sensor>[A-Z]+)_(?P<correction>[A-Z]+)_(?P<epsg>\d+)_(?P<x_coord>-?\d+)_(?P<y_coord>-?\d+)_(?P<year>\d\d\d\d).`,}

var parsers map[string]*regexp.Regexp = map[string]*regexp.Regexp{}
//var timeExtractors map[string]func(map[string] string) time.Time = map[string]func(map[string] string) time.Time{"landsat":landsatTime, "modis1": modisTime, "modis2": modisTime}

func init() {
	for key, value := range(parserStrings) {
		parsers[key] = regexp.MustCompile(value)
	}
}

func parseName(filePath string) (map[string]string, time.Time) {

	for _, r := range(parsers) {
		_, fileName := filepath.Split(filePath)

		if (r.MatchString(fileName)) {
			match := r.FindStringSubmatch(fileName)

			result := make(map[string]string)
			for i, name := range r.SubexpNames() {
				if i != 0 {
					result[name] = match[i]
				}
			}

			return result, parseTime(result)
		}
	}
	return nil, time.Time{}
}

func parseTime(nameFields map[string]string) time.Time {
	if _, ok := nameFields["year"]; ok {
		year, _ := strconv.Atoi(nameFields["year"])
		t := time.Date(year, 0, 0, 0, 0, 0, 0, time.UTC)
		if _, ok := nameFields["julian_day"]; ok {
			julianDay, _ := strconv.Atoi(nameFields["julian_day"])
			t = t.Add(time.Hour * 24 * time.Duration(julianDay))
		}
		if _, ok := nameFields["hour"]; ok {
			hour, _ := strconv.Atoi(nameFields["hour"])
			t = t.Add(time.Hour * time.Duration(hour))
		}
		if _, ok := nameFields["minute"]; ok {
			minute, _ := strconv.Atoi(nameFields["minute"])
			t = t.Add(time.Minute * time.Duration(minute))
		}
		if _, ok := nameFields["second"]; ok {
			second, _ := strconv.Atoi(nameFields["second"])
			t = t.Add(time.Second * time.Duration(second))
		}

		return t
	}
	return time.Time{}
}

func main() {
	s := bufio.NewScanner(os.Stdin)
	for s.Scan() {
		parts := strings.Split(s.Text(), "\t")
		if len(parts) != 2 {
			fmt.Printf("Input not recognised: %s\n", s.Text())
		}

		gdalFile := geolib.GDALFile{}
		err := json.Unmarshal([]byte(parts[1]), &gdalFile)
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}

		geoFile := geolib.GeoFile{Driver: gdalFile.Driver}

		nameFields, timeStamp := parseName(parts[0])

		for _, ds := range gdalFile.DataSets {
			if ds.ProjWKT != "" {
				poly := geolib.GetPolygon(ds.ProjWKT, ds.GeoTransform, ds.XSize, ds.YSize)
				polyWGS84 := poly.ReprojectToWGS84()

				var times []string
				if nc_times, ok := ds.Extras["nc_times"]; ok {
					times = nc_times
				} else {
					times = []string{timeStamp.Format("2006-01-02T15:04:05Z")}
				}

				geoFile.DataSets = append(geoFile.DataSets, GeoMetaData{DataSetName: ds.DataSetName, TimeStamps: times, FileNameFields: nameFields, Polygon: json.RawMessage(polyWGS84.ToGeoJSON()), RasterCount: ds.RasterCount, Type: ds.Type, XSize: ds.XSize, YSize: ds.YSize, ProjWKT: ds.ProjWKT, GeoTransform: ds.GeoTransform})

			}
		}
		out, err := json.Marshal(&geoFile)
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
		fmt.Printf("%s\tgdal\t%s\n", parts[0], string(out))

	}
}
<|endoftext|>


<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Binary dam_reset to reset the storage of a DAM
package main

import (
	"context"
	"flag"
	"strings"

	glog "github.com/golang/glog" /* copybara-comment */
	"github.com/GoogleCloudPlatform/healthcare-federated-access-services/lib/dam" /* copybara-comment: dam */
	"github.com/GoogleCloudPlatform/healthcare-federated-access-services/lib/dsstore" /* copybara-comment: dsstore */
	"github.com/GoogleCloudPlatform/healthcare-federated-access-services/lib/saw" /* copybara-comment: saw */
	"github.com/GoogleCloudPlatform/healthcare-federated-access-services/lib/storage" /* copybara-comment: storage */
)

func main() {
	pre := flag.String("account_prefix", "", "when a wipe is requested, accounts matching this prefix override will be removed")
	path := flag.String("path", "deploy/config", "specifies the relative or absolute path to the config file root")
	wipe := flag.String("wipe", "", "specify 'unsafe_wipe_in_non_production' to remove all data for the service from the storage layer first (DO NOT USE IN PRODUCTION)")

	flag.Parse()
	args := flag.Args()

	if len(args) != 2 {
		glog.Exitf("Usage: dam_import -wipe=... -path=<config_root> -account_prefix=<service_account_prefix_to_delete> <project> <environment>")
	}
	project := args[0]
	env := args[1]
	envPrefix := ""
	service := "dam"
	if len(env) > 0 {
		envPrefix = "-" + env
		service += envPrefix
	}
	accountPrefix := "ic" + envPrefix + "-dot-"
	if *pre != "" {
		accountPrefix = *pre
	}
	ctx := context.Background()
	store := dsstore.NewDatastoreStorage(context.Background(), project, service, *path)
	wh := saw.MustNew(ctx, store)
	vars := map[string]string{
		"${YOUR_PROJECT_ID}":  project,
		"${YOUR_ENVIRONMENT}": envPrefix,
	}
	if *wipe != "" {
		if *wipe != "unsafe_wipe_in_non_production" {
			glog.Exitf("attempted wipe failed: only works if specific safety value set. See -h for help.")
		}
		glog.Infof("WIPE STORAGE FOR SERVICE %q...", service)
		if err := store.Wipe(storage.AllRealms); err != nil {
			glog.Exitf("error wiping storage for service %q: %v", service, err)
		}
		glog.Infof("Wipe complete")
	}

	if err := dam.ImportConfig(store, service, wh, vars); err != nil {
		glog.Exitf("error importing files: %v", err)
	}

	if *wipe != "" {
		cleanupServiceAccounts(ctx, accountPrefix, project, store)
	}

	glog.Infof("SUCCESS resetting DAM service %q", service)
}

func cleanupServiceAccounts(ctx context.Context, accountPrefix, project string, store *dsstore.DatastoreStorage) {
	wh := saw.MustNew(ctx, store)
	var (
		removed, skipped, errors int
		emails                   []string
	)
	maxErrors := 20
	aborted := ""

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()
	accounts, err := wh.GetServiceAccounts(ctx, project)
	if err != nil {
		glog.Errorf("fetching service accounts from project %q failed: %v", project, err)
		return
	}

	for a := range accounts {
		// DAM adds service account DisplayName of the form: subject|service_full_path
		// so pull out the service_full_path and match on the accountPrefix provided.
		parts := strings.SplitN(a.DisplayName, "|", 2)
		if len(parts) < 2 || !strings.HasPrefix(parts[1], accountPrefix) {
			skipped++
			continue
		}
		emails = append(emails, a.ID)
	}

	for _, email := range emails {
		if err := wh.RemoveServiceAccount(ctx, project, email); err != nil {
			if errors < 3 {
				glog.Errorf("deleting service account %q on project %q failed: %v", email, project, err)
			}
			errors++
			if errors >= maxErrors {
				aborted = "+ (aborted early)"
				break
			}
		} else {
			removed++
		}
	}
	glog.Infof("status of removing service accounts: project %q, prefix %q, matched %d, removed %d, skipped %d, errors %d%s", project, accountPrefix, len(emails), removed, skipped, errors, aborted)
}
<commit_msg>don't error if service account is already not there<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Binary dam_reset to reset the storage of a DAM
package main

import (
	"context"
	"flag"
	"strings"

	glog "github.com/golang/glog" /* copybara-comment */
	"google.golang.org/grpc/codes" /* copybara-comment */
	"google.golang.org/grpc/status" /* copybara-comment */
	"github.com/GoogleCloudPlatform/healthcare-federated-access-services/lib/dam" /* copybara-comment: dam */
	"github.com/GoogleCloudPlatform/healthcare-federated-access-services/lib/dsstore" /* copybara-comment: dsstore */
	"github.com/GoogleCloudPlatform/healthcare-federated-access-services/lib/saw" /* copybara-comment: saw */
	"github.com/GoogleCloudPlatform/healthcare-federated-access-services/lib/storage" /* copybara-comment: storage */
)

func main() {
	pre := flag.String("account_prefix", "", "when a wipe is requested, accounts matching this prefix override will be removed")
	path := flag.String("path", "deploy/config", "specifies the relative or absolute path to the config file root")
	wipe := flag.String("wipe", "", "specify 'unsafe_wipe_in_non_production' to remove all data for the service from the storage layer first (DO NOT USE IN PRODUCTION)")

	flag.Parse()
	args := flag.Args()

	if len(args) != 2 {
		glog.Exitf("Usage: dam_import -wipe=... -path=<config_root> -account_prefix=<service_account_prefix_to_delete> <project> <environment>")
	}
	project := args[0]
	env := args[1]
	envPrefix := ""
	service := "dam"
	if len(env) > 0 {
		envPrefix = "-" + env
		service += envPrefix
	}
	accountPrefix := "ic" + envPrefix + "-dot-"
	if *pre != "" {
		accountPrefix = *pre
	}
	ctx := context.Background()
	store := dsstore.NewDatastoreStorage(context.Background(), project, service, *path)
	wh := saw.MustNew(ctx, store)
	vars := map[string]string{
		"${YOUR_PROJECT_ID}":  project,
		"${YOUR_ENVIRONMENT}": envPrefix,
	}
	if *wipe != "" {
		if *wipe != "unsafe_wipe_in_non_production" {
			glog.Exitf("attempted wipe failed: only works if specific safety value set. See -h for help.")
		}
		glog.Infof("WIPE STORAGE FOR SERVICE %q...", service)
		if err := store.Wipe(storage.AllRealms); err != nil {
			glog.Exitf("error wiping storage for service %q: %v", service, err)
		}
		glog.Infof("Wipe complete")
	}

	if err := dam.ImportConfig(store, service, wh, vars); err != nil {
		glog.Exitf("error importing files: %v", err)
	}

	if *wipe != "" {
		cleanupServiceAccounts(ctx, accountPrefix, project, store)
	}

	glog.Infof("SUCCESS resetting DAM service %q", service)
}

func cleanupServiceAccounts(ctx context.Context, accountPrefix, project string, store *dsstore.DatastoreStorage) {
	wh := saw.MustNew(ctx, store)
	var (
		removed, skipped, errors int
		emails                   []string
	)
	maxErrors := 20
	aborted := ""

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()
	accounts, err := wh.GetServiceAccounts(ctx, project)
	if err != nil {
		glog.Errorf("fetching service accounts from project %q failed: %v", project, err)
		return
	}

	for a := range accounts {
		// DAM adds service account DisplayName of the form: subject|service_full_path
		// so pull out the service_full_path and match on the accountPrefix provided.
		parts := strings.SplitN(a.DisplayName, "|", 2)
		if len(parts) < 2 || !strings.HasPrefix(parts[1], accountPrefix) {
			skipped++
			continue
		}
		emails = append(emails, a.ID)
	}

	for _, email := range emails {
		err := wh.RemoveServiceAccount(ctx, project, email)
		switch status.Code(err) {
		case codes.OK:
			removed++

		case codes.NotFound:
			glog.Infof("deleting service account %q on project %q: acccount does not exist.", email, project)

		default:
			errors++
			if errors >= maxErrors {
				aborted = "+ (aborted early)"
				break
			}
			if errors < 3 {
				glog.Errorf("deleting service account %q on project %q failed: %v", email, project, err)
			}
		}
	}
	glog.Infof("status of removing service accounts: project %q, prefix %q, matched %d, removed %d, skipped %d, errors %d%s", project, accountPrefix, len(emails), removed, skipped, errors, aborted)
}
<|endoftext|>


<commit_before>package docker

import (
	"testing"
	"github.com/docker/engine-api/types/container"
	"github.com/docking-tools/register/api"
	"strings"
)

func TestServiceMetadataSSS(t *testing.T) {
	config := container.Config {
		Labels : make(map[string]string),
		Env : make([]string, 1),
	}

	config.Labels["SERVICE.NAME"]="test1"
	config.Labels["service_8080_name"]="ok-port"
	config.Labels["SERVICE_ignore"]="true"

	config.Labels["service.8A_test"]="ko"

	config.Labels["service_test"]="ok"
	config.Labels["test_service_test"]="ko"

	config.Env[0]="SERVICE.TEST=ok"

	metadata, metaFromPort :=  serviceMetaData(&config, "8080")

	t.Log("%v", metadata)
	t.Log("%v", metaFromPort)

	ignore := mapDefault(metadata,"ignore","")
	t.Log("%#v", ignore)

	if len(metadata)!=4 {
		t.Fatal("Number of result MetaData is not 4")
	}
	if !metaFromPort["name"] {
		t.Fatal("mettaFromPort for key name can be true")
	}
	if !strings.EqualFold(metadata["ignore"],"true") {
		t.Fatal("mettadata for key 'ignore' can be true")
	}

	if len(metaFromPort) !=1 {
		t.Fatal("Number of result MetaFromPort is not 1")
	}
}

func TestGraphMetaData(t *testing.T) {
	config := container.Config {
		Labels : make(map[string]string),
		Env : make([]string, 1),
	}

	config.Labels["cron.test.titi"]="ok"
	config.Labels["cron.test"]="KO"

	config.Labels["cron.test.tutu"]="ok"
	config.Labels["cron.8080.test"]="ok-port"

	config.Labels["crone.8A.test"]="ko"
	config.Labels["cron_test.toto.tata"]="ok"
	config.Env[0]="test_cron=ok"
	config.Env[1]="sans_valeur"

	result  :=  graphMetaData(&config)
	t.Logf("%v", result)

	if len(result)!=3 {
		t.Fatal("Number of result MetaData is not 3 %v", result)
	}

	if len(result["cron"].(api.Recmap)) !=2 {
		t.Fatal("Number of result MEtaData is not 1 %v", result["cron"].(api.Recmap))
	}
	if result["test"].(api.Recmap)["cron"]!="ok" {
		t.Fatal("cron.test not equals to ok", result["cron"].(api.Recmap)["test"])
	}
	if result["cron"].(api.Recmap)["test"].(api.Recmap)["tutu"]!="ok" {
		t.Fatal("cron.test.tutu not equals to ok", result["cron"].(api.Recmap)["test"])
	}
}
<commit_msg>correction test util<commit_after>package docker

import (
	"testing"
	"github.com/docker/engine-api/types/container"
	"github.com/docking-tools/register/api"
	"strings"
)

func TestServiceMetadataSSS(t *testing.T) {
	config := container.Config {
		Labels : make(map[string]string),
		Env : make([]string, 1),
	}

	config.Labels["SERVICE.NAME"]="test1"
	config.Labels["service_8080_name"]="ok-port"
	config.Labels["SERVICE_ignore"]="true"

	config.Labels["service.8A_test"]="ko"

	config.Labels["service_test"]="ok"
	config.Labels["test_service_test"]="ko"

	config.Env[0]="SERVICE.TEST=ok"

	metadata, metaFromPort :=  serviceMetaData(&config, "8080")

	t.Log("%v", metadata)
	t.Log("%v", metaFromPort)

	ignore := mapDefault(metadata,"ignore","")
	t.Log("%#v", ignore)

	if len(metadata)!=4 {
		t.Fatal("Number of result MetaData is not 4")
	}
	if !metaFromPort["name"] {
		t.Fatal("mettaFromPort for key name can be true")
	}
	if !strings.EqualFold(metadata["ignore"],"true") {
		t.Fatal("mettadata for key 'ignore' can be true")
	}

	if len(metaFromPort) !=1 {
		t.Fatal("Number of result MetaFromPort is not 1")
	}
}

func TestGraphMetaData(t *testing.T) {
	config := container.Config {
		Labels : make(map[string]string),
		Env : make([]string, 2),
	}

	config.Labels["cron.test.titi"]="ok"
	config.Labels["cron.test"]="KO"

	config.Labels["cron.test.tutu"]="ok"
	config.Labels["cron.8080.test"]="ok-port"

	config.Labels["crone.8A.test"]="ko"
	config.Labels["cron_test.toto.tata"]="ok"
	config.Env[0]="test_cron=ok"
	config.Env[1]="sans_valeur"

	result  :=  graphMetaData(&config)
	t.Logf("%v", result)

	if len(result)!=3 {
		t.Fatal("Number of result MetaData is not 3 %v", result)
	}

	if len(result["cron"].(api.Recmap)) !=2 {
		t.Fatal("Number of result MEtaData is not 1 %v", result["cron"].(api.Recmap))
	}
	if result["test"].(api.Recmap)["cron"]!="ok" {
		t.Fatal("cron.test not equals to ok", result["cron"].(api.Recmap)["test"])
	}
	if result["cron"].(api.Recmap)["test"].(api.Recmap)["tutu"]!="ok" {
		t.Fatal("cron.test.tutu not equals to ok", result["cron"].(api.Recmap)["test"])
	}
}
<|endoftext|>


<commit_before>package docker

import (
	"context"
	"github.com/docker/docker/api/types"
	"github.com/gorilla/websocket"
	"io"
	"log"
	"net/http"
	"regexp"
)

var logWebsocketRequest = regexp.MustCompile(`/containers/([^/]+)/logs`)
var hostCheck = regexp.MustCompile(`vibioh\.fr$`)

var upgrader = websocket.Upgrader{
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
	CheckOrigin: func(r *http.Request) bool {
		return hostCheck.MatchString(r.Host)
	},
}

func logsContainerWebsocketHandler(w http.ResponseWriter, r *http.Request, containerID []byte) {
	logs, err := docker.ContainerLogs(context.Background(), string(containerID), types.ContainerLogsOptions{ShowStdout: true, ShowStderr: true, Follow: true})
	if err != nil {
		log.Print(err)
		return
	}

	defer logs.Close()

	ws, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Print(err)
		return
	}

	defer ws.Close()

	for {
		wsWriter, err := ws.NextWriter(websocket.BinaryMessage)
		if err != nil {
			log.Print(err)
			return
		}

		if _, err = io.Copy(wsWriter, logs); err != nil {
			log.Print(err)
			return
		}
	}
}

func handleWebsocket(w http.ResponseWriter, r *http.Request) {
	urlPath := []byte(r.URL.Path)

	if logWebsocketRequest.Match(urlPath) {
		logsContainerWebsocketHandler(w, r, logWebsocketRequest.FindSubmatch(urlPath)[1])
	}
}
<commit_msg>Update websocket.go<commit_after>package docker

import (
	"context"
	"github.com/docker/docker/api/types"
	"github.com/gorilla/websocket"
	"io"
	"log"
	"net/http"
	"regexp"
)

var logWebsocketRequest = regexp.MustCompile(`/containers/([^/]+)/logs`)
var hostCheck = regexp.MustCompile(`vibioh\.fr$`)

var upgrader = websocket.Upgrader{
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
	CheckOrigin: func(r *http.Request) bool {
		return hostCheck.MatchString(r.Host)
	},
}

func logsContainerWebsocketHandler(w http.ResponseWriter, r *http.Request, containerID []byte) {
	logs, err := docker.ContainerLogs(context.Background(), string(containerID), types.ContainerLogsOptions{ShowStdout: true, ShowStderr: true, Follow: false})
	if err != nil {
		log.Print(err)
		return
	}

	defer logs.Close()

	ws, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Print(err)
		return
	}

	defer ws.Close()

	for {
		wsWriter, err := ws.NextWriter(websocket.BinaryMessage)
		if err != nil {
			log.Print(err)
			return
		}

		if _, err = io.Copy(wsWriter, logs); err != nil {
			log.Print(err)
			return
		}
	}
}

func handleWebsocket(w http.ResponseWriter, r *http.Request) {
	urlPath := []byte(r.URL.Path)

	if logWebsocketRequest.Match(urlPath) {
		logsContainerWebsocketHandler(w, r, logWebsocketRequest.FindSubmatch(urlPath)[1])
	}
}
<|endoftext|>


<commit_before>package maptiles

import (
	"log"
	"net/http"
	"regexp"
	"strconv"
)

// TODO serve list of registered layers per HTTP (preferably leafletjs-compatible js-array)

// Handles HTTP requests for map tiles, caching any produced tiles
// in an MBtiles 1.2 compatible sqlite db.
type TileServer struct {
	m         *TileDb
	lmp       *LayerMultiplex
	TmsSchema bool
}

func NewTileServer(cacheFile string) *TileServer {
	t := TileServer{}
	t.lmp = NewLayerMultiplex()
	t.m = NewTileDb(cacheFile)

	return &t
}

func (t *TileServer) AddMapnikLayer(layerName string, stylesheet string) {
	t.lmp.AddRenderer(layerName, stylesheet)
}

var pathRegex = regexp.MustCompile(`/([A-Za-z0-9]+)/([0-9]+)/([0-9]+)/([0-9]+)\.png`)

func (t *TileServer) ServeTileRequest(w http.ResponseWriter, r *http.Request, tc TileCoord) {
	log.Println(r.RemoteAddr, tc)
	ch := make(chan TileFetchResult)

	tr := TileFetchRequest{tc, ch}
	t.m.RequestQueue() <- tr

	result := <-ch
	needsInsert := false

	if result.BlobPNG == nil {
		// Tile was not provided by DB, so submit the tile request to the renderer
		t.lmp.SubmitRequest(tr)
		result = <-ch
		if result.BlobPNG == nil {
			// The tile could not be rendered, now we need to bail out.
			http.NotFound(w, r)
			return
		}
		needsInsert = true
	}

	w.Header().Set("Content-Type", "image/png")
	_, err := w.Write(result.BlobPNG)
	if err != nil {
		log.Println(err)
	}
	if needsInsert {
		t.m.InsertQueue() <- result // insert newly rendered tile into cache db
	}
}

func (t *TileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	path := pathRegex.FindStringSubmatch(r.URL.Path)

	if path == nil {
		http.NotFound(w, r)
		return
	}

	l := path[1]
	z, _ := strconv.ParseUint(path[2], 10, 64)
	x, _ := strconv.ParseUint(path[3], 10, 64)
	y, _ := strconv.ParseUint(path[4], 10, 64)

	t.ServeTileRequest(w, r, TileCoord{x, y, z, t.TmsSchema, l})
}
<commit_msg>tile server logging<commit_after>package maptiles

import (
	"log"
	"net/http"
	"regexp"
	"strconv"
)

// TODO serve list of registered layers per HTTP (preferably leafletjs-compatible js-array)

// Handles HTTP requests for map tiles, caching any produced tiles
// in an MBtiles 1.2 compatible sqlite db.
type TileServer struct {
	m         *TileDb
	lmp       *LayerMultiplex
	TmsSchema bool
}

func NewTileServer(cacheFile string) *TileServer {
	t := TileServer{}
	t.lmp = NewLayerMultiplex()
	t.m = NewTileDb(cacheFile)

	return &t
}

func (t *TileServer) AddMapnikLayer(layerName string, stylesheet string) {
	t.lmp.AddRenderer(layerName, stylesheet)
}

var pathRegex = regexp.MustCompile(`/([A-Za-z0-9]+)/([0-9]+)/([0-9]+)/([0-9]+)\.png`)

func (t *TileServer) ServeTileRequest(w http.ResponseWriter, r *http.Request, tc TileCoord) {
	log.Println(r.RemoteAddr, tc)
	ch := make(chan TileFetchResult)

	tr := TileFetchRequest{tc, ch}
	t.m.RequestQueue() <- tr

	result := <-ch
	needsInsert := false

	if result.BlobPNG == nil {
		// Tile was not provided by DB, so submit the tile request to the renderer
		t.lmp.SubmitRequest(tr)
		result = <-ch
		if result.BlobPNG == nil {
			// The tile could not be rendered, now we need to bail out.
			http.NotFound(w, r)
			return
		}
		needsInsert = true
	}

	w.Header().Set("Content-Type", "image/png")
	_, err := w.Write(result.BlobPNG)
	if err != nil {
		log.Println(err)
	}
	if needsInsert {
		t.m.InsertQueue() <- result // insert newly rendered tile into cache db
	}
}

func (t *TileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {

	if string.Contains(r.URL.Path, "metadata") {
		t.MetaDataHandler(w, r)
		return;
	}

	path := pathRegex.FindStringSubmatch(r.URL.Path)

	if path == nil {
		http.NotFound(w, r)
		return
	}

	l := path[1]
	z, _ := strconv.ParseUint(path[2], 10, 64)
	x, _ := strconv.ParseUint(path[3], 10, 64)
	y, _ := strconv.ParseUint(path[4], 10, 64)

	t.ServeTileRequest(w, r, TileCoord{x, y, z, t.TmsSchema, l})
}

func (t *TileServer) MetaDataHandler(w http.ResponseWriter, r *http.Request) {
	// Set headers
	w.Header().Set("Content-Type", "application/json")
	w.Header().Set("Access-Control-Allow-Origin", "*")

	// Get params
	vars := mux.Vars(r)
	dbname := vars["db"]

	// check for file
	if _, err := os.Stat(dbname+".mbtiles"); os.IsNotExist(err) {
		fmt.Println("File not found [" + dbname + ".mbtiles]")
		http.Error(w, err.Error(), http.StatusNotFound)
		return
	}

	// Open database
	db, _ := sql.Open("sqlite3", "./"+dbname+".mbtiles")
	rows, _ := db.Query("SELECT * FROM metadata")

	metadata :=  make(map[string]string)

	for rows.Next() {

		var name string
		var value string
		rows.Scan(&name, &value)

		metadata[name] = value
	}

	db.Close()

	response_wrapper := make(map[string]interface{})
	response_wrapper["status"] = "success"
	response_wrapper["data"] = metadata

	js, err := json.Marshal(response_wrapper)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Write(js)

}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package maas

import (
	"fmt"
	"net/url"
	"strings"

	"github.com/juju/errors"
	"github.com/juju/gomaasapi"
	"github.com/juju/utils/set"

	"github.com/juju/juju/constraints"
	"github.com/juju/juju/network"
)

var unsupportedConstraints = []string{
	constraints.CpuPower,
	constraints.InstanceType,
	constraints.VirtType,
}

// ConstraintsValidator is defined on the Environs interface.
func (environ *maasEnviron) ConstraintsValidator() (constraints.Validator, error) {
	validator := constraints.NewValidator()
	validator.RegisterUnsupported(unsupportedConstraints)
	supportedArches, err := environ.SupportedArchitectures()
	if err != nil {
		return nil, err
	}
	validator.RegisterVocabulary(constraints.Arch, supportedArches)
	return validator, nil
}

// convertConstraints converts the given constraints into an url.Values object
// suitable to pass to MAAS when acquiring a node. CpuPower is ignored because
// it cannot be translated into something meaningful for MAAS right now.
func convertConstraints(cons constraints.Value) url.Values {
	params := url.Values{}
	if cons.Arch != nil {
		// Note: Juju and MAAS use the same architecture names.
		// MAAS also accepts a subarchitecture (e.g. "highbank"
		// for ARM), which defaults to "generic" if unspecified.
		params.Add("arch", *cons.Arch)
	}
	if cons.CpuCores != nil {
		params.Add("cpu_count", fmt.Sprintf("%d", *cons.CpuCores))
	}
	if cons.Mem != nil {
		params.Add("mem", fmt.Sprintf("%d", *cons.Mem))
	}
	convertTagsToParams(params, cons.Tags)
	if cons.CpuPower != nil {
		logger.Warningf("ignoring unsupported constraint 'cpu-power'")
	}
	return params
}

// convertConstraints2 converts the given constraints into a
// gomaasapi.AllocateMachineArgs for paasing to MAAS 2.
func convertConstraints2(cons constraints.Value) gomaasapi.AllocateMachineArgs {
	params := gomaasapi.AllocateMachineArgs{}
	if cons.Arch != nil {
		params.Architecture = *cons.Arch
	}
	if cons.CpuCores != nil {
		params.MinCPUCount = int(*cons.CpuCores)
	}
	if cons.Mem != nil {
		params.MinMemory = int(*cons.Mem)
	}
	if cons.Tags != nil {
		positives, negatives := parseDelimitedValues(*cons.Tags)
		if len(positives) > 0 {
			params.Tags = positives
		}
		if len(negatives) > 0 {
			params.NotTags = negatives
		}
	}
	if cons.CpuPower != nil {
		logger.Warningf("ignoring unsupported constraint 'cpu-power'")
	}
	return params
}

// convertTagsToParams converts a list of positive/negative tags from
// constraints into two comma-delimited lists of values, which can then be
// passed to MAAS using the "tags" and "not_tags" arguments to acquire. If
// either list of tags is empty, the respective argument is not added to params.
func convertTagsToParams(params url.Values, tags *[]string) {
	if tags == nil || len(*tags) == 0 {
		return
	}
	positives, negatives := parseDelimitedValues(*tags)
	if len(positives) > 0 {
		params.Add("tags", strings.Join(positives, ","))
	}
	if len(negatives) > 0 {
		params.Add("not_tags", strings.Join(negatives, ","))
	}
}

// convertSpacesFromConstraints extracts spaces from constraints and converts
// them to two lists of positive and negative spaces.
func convertSpacesFromConstraints(spaces *[]string) ([]string, []string) {
	if spaces == nil || len(*spaces) == 0 {
		return nil, nil
	}
	return parseDelimitedValues(*spaces)
}

// parseDelimitedValues parses a slice of raw values coming from constraints
// (Tags or Spaces). The result is split into two slices - positives and
// negatives (prefixed with "^"). Empty values are ignored.
func parseDelimitedValues(rawValues []string) (positives, negatives []string) {
	for _, value := range rawValues {
		if value == "" || value == "^" {
			// Neither of these cases should happen in practise, as constraints
			// are validated before setting them and empty names for spaces or
			// tags are not allowed.
			continue
		}
		if strings.HasPrefix(value, "^") {
			negatives = append(negatives, strings.TrimPrefix(value, "^"))
		} else {
			positives = append(positives, value)
		}
	}
	return positives, negatives
}

// interfaceBinding defines a requirement that a node interface must satisfy in
// order for that node to get selected and started, based on deploy-time
// bindings of a service.
//
// TODO(dimitern): Once the services have bindings defined in state, a version
// of this should go to the network package (needs to be non-MAAS-specifc
// first). Also, we need to transform Juju space names from constraints into
// MAAS space provider IDs.
type interfaceBinding struct {
	Name            string
	SpaceProviderId string

	// add more as needed.
}

// numericLabelLimit is a sentinel value used in addInterfaces to limit the
// number of disabmiguation inner loop iterations in case named labels clash
// with numeric labels for spaces coming from constraints. It's defined here to
// facilitate testing this behavior.
var numericLabelLimit uint = 0xffff

// addInterfaces converts a slice of interface bindings, postiveSpaces and
// negativeSpaces coming from constraints to the format MAAS expects for the
// "interfaces" and "not_networks" arguments to acquire node. Returns an error
// satisfying errors.IsNotValid() if the bindings contains duplicates, empty
// Name/SpaceProviderId, or if negative spaces clash with specified bindings.
// Duplicates between specified bindings and positiveSpaces are silently
// skipped.
func addInterfaces(
	params url.Values,
	bindings []interfaceBinding,
	positiveSpaces, negativeSpaces []network.SpaceInfo,
) error {
	combinedBindings, negatives, err := getBindings(bindings, positiveSpaces, negativeSpaces)
	if err != nil {
		return errors.Trace(err)
	}
	if len(combinedBindings) > 0 {
		combinedBindingsString := make([]string, len(combinedBindings))
		for i, binding := range combinedBindings {
			combinedBindingsString[i] = fmt.Sprintf("%s:space=%s", binding.Name, binding.SpaceProviderId)
		}
		params.Add("interfaces", strings.Join(combinedBindingsString, ";"))
	}
	if len(negatives) > 0 {
		negativesString := make([]string, len(negatives))
		for i, binding := range negatives {
			negativesString[i] = fmt.Sprintf("space:%s", binding.SpaceProviderId)
		}
		params.Add("not_networks", strings.Join(negativesString, ","))
	}
	return nil
}

func getBindings(
	bindings []interfaceBinding,
	positiveSpaces, negativeSpaces []network.SpaceInfo,
) ([]interfaceBinding, []interfaceBinding, error) {
	var (
		index            uint
		combinedBindings []interfaceBinding
	)
	namesSet := set.NewStrings()
	spacesSet := set.NewStrings()
	for _, binding := range bindings {
		switch {
		case binding.Name == "":
			return nil, nil, errors.NewNotValid(nil, "interface bindings cannot have empty names")
		case binding.SpaceProviderId == "":
			return nil, nil, errors.NewNotValid(nil, fmt.Sprintf(
				"invalid interface binding %q: space provider ID is required",
				binding.Name,
			))
		case namesSet.Contains(binding.Name):
			return nil, nil, errors.NewNotValid(nil, fmt.Sprintf(
				"duplicated interface binding %q",
				binding.Name,
			))
		}
		namesSet.Add(binding.Name)
		spacesSet.Add(binding.SpaceProviderId)

		combinedBindings = append(combinedBindings, binding)
	}

	createLabel := func(index uint, namesSet set.Strings) (string, uint, error) {
		var label string
		for {
			label = fmt.Sprintf("%v", index)
			if !namesSet.Contains(label) {
				break
			}
			if index > numericLabelLimit { // ...just to make sure we won't loop forever.
				return "", index, errors.Errorf("too many conflicting numeric labels, giving up.")
			}
			index++
		}
		namesSet.Add(label)
		return label, index, nil
	}
	for _, space := range positiveSpaces {
		if spacesSet.Contains(string(space.ProviderId)) {
			// Skip duplicates in positiveSpaces.
			continue
		}
		spacesSet.Add(string(space.ProviderId))

		var label string
		var err error
		label, index, err = createLabel(index, namesSet)
		if err != nil {
			return nil, nil, errors.Errorf("too many conflicting numeric labels, giving up.")
		}
		// Make sure we pick a label that doesn't clash with possible bindings.
		combinedBindings = append(combinedBindings, interfaceBinding{label, string(space.ProviderId)})
	}

	var negatives []interfaceBinding
	for _, space := range negativeSpaces {
		if spacesSet.Contains(string(space.ProviderId)) {
			return nil, nil, errors.NewNotValid(nil, fmt.Sprintf(
				"negative space %q from constraints clashes with interface bindings",
				space.Name,
			))
		}
		var label string
		var err error
		label, index, err = createLabel(index, namesSet)
		if err != nil {
			return nil, nil, errors.Errorf("too many conflicting numeric labels, giving up.")
		}
		negatives = append(negatives, interfaceBinding{label, string(space.ProviderId)})
		index++
	}
	return combinedBindings, negatives, nil
}

func addInterfaces2(
	params *gomaasapi.AllocateMachineArgs,
	bindings []interfaceBinding,
	positiveSpaces, negativeSpaces []network.SpaceInfo,
) error {
	combinedBindings, negatives, err := getBindings(bindings, positiveSpaces, negativeSpaces)
	if err != nil {
		return errors.Trace(err)
	}

	if len(combinedBindings) > 0 {
		interfaceSpecs := make([]gomaasapi.InterfaceSpec, len(combinedBindings))
		for i, space := range combinedBindings {
			interfaceSpecs[i] = gomaasapi.InterfaceSpec{space.Name, space.SpaceProviderId}
		}
		params.Interfaces = interfaceSpecs
	}
	if len(negatives) > 0 {
		negativeStrings := make([]string, len(negatives))
		for i, space := range negatives {
			negativeStrings[i] = space.SpaceProviderId
		}
		params.NotSpace = negativeStrings
	}
	return nil
}

// addStorage converts volume information into url.Values object suitable to
// pass to MAAS when acquiring a node.
func addStorage(params url.Values, volumes []volumeInfo) {
	if len(volumes) == 0 {
		return
	}
	// Requests for specific values are passed to the acquire URL
	// as a storage URL parameter of the form:
	// [volume-name:]sizeinGB[tag,...]
	// See http://maas.ubuntu.com/docs/api.html#nodes

	// eg storage=root:0(ssd),data:20(magnetic,5400rpm),45
	makeVolumeParams := func(v volumeInfo) string {
		var params string
		if v.name != "" {
			params = v.name + ":"
		}
		params += fmt.Sprintf("%d", v.sizeInGB)
		if len(v.tags) > 0 {
			params += fmt.Sprintf("(%s)", strings.Join(v.tags, ","))
		}
		return params
	}
	var volParms []string
	for _, v := range volumes {
		params := makeVolumeParams(v)
		volParms = append(volParms, params)
	}
	params.Add("storage", strings.Join(volParms, ","))
}

// addStorage2 adds volume information onto a gomaasapi.AllocateMachineArgs
// object suitable to pass to MAAS 2 when acquiring a node.
func addStorage2(params *gomaasapi.AllocateMachineArgs, volumes []volumeInfo) {
	if len(volumes) == 0 {
		return
	}
	var volParams []gomaasapi.StorageSpec
	for _, v := range volumes {
		volSpec := gomaasapi.StorageSpec{
			Label: v.name,
			Size:  int(v.sizeInGB),
			Tags:  v.tags,
		}
		volParams = append(volParams, volSpec)
	}
	params.Storage = volParams
}
<commit_msg>Remove spurious line<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package maas

import (
	"fmt"
	"net/url"
	"strings"

	"github.com/juju/errors"
	"github.com/juju/gomaasapi"
	"github.com/juju/utils/set"

	"github.com/juju/juju/constraints"
	"github.com/juju/juju/network"
)

var unsupportedConstraints = []string{
	constraints.CpuPower,
	constraints.InstanceType,
	constraints.VirtType,
}

// ConstraintsValidator is defined on the Environs interface.
func (environ *maasEnviron) ConstraintsValidator() (constraints.Validator, error) {
	validator := constraints.NewValidator()
	validator.RegisterUnsupported(unsupportedConstraints)
	supportedArches, err := environ.SupportedArchitectures()
	if err != nil {
		return nil, err
	}
	validator.RegisterVocabulary(constraints.Arch, supportedArches)
	return validator, nil
}

// convertConstraints converts the given constraints into an url.Values object
// suitable to pass to MAAS when acquiring a node. CpuPower is ignored because
// it cannot be translated into something meaningful for MAAS right now.
func convertConstraints(cons constraints.Value) url.Values {
	params := url.Values{}
	if cons.Arch != nil {
		// Note: Juju and MAAS use the same architecture names.
		// MAAS also accepts a subarchitecture (e.g. "highbank"
		// for ARM), which defaults to "generic" if unspecified.
		params.Add("arch", *cons.Arch)
	}
	if cons.CpuCores != nil {
		params.Add("cpu_count", fmt.Sprintf("%d", *cons.CpuCores))
	}
	if cons.Mem != nil {
		params.Add("mem", fmt.Sprintf("%d", *cons.Mem))
	}
	convertTagsToParams(params, cons.Tags)
	if cons.CpuPower != nil {
		logger.Warningf("ignoring unsupported constraint 'cpu-power'")
	}
	return params
}

// convertConstraints2 converts the given constraints into a
// gomaasapi.AllocateMachineArgs for paasing to MAAS 2.
func convertConstraints2(cons constraints.Value) gomaasapi.AllocateMachineArgs {
	params := gomaasapi.AllocateMachineArgs{}
	if cons.Arch != nil {
		params.Architecture = *cons.Arch
	}
	if cons.CpuCores != nil {
		params.MinCPUCount = int(*cons.CpuCores)
	}
	if cons.Mem != nil {
		params.MinMemory = int(*cons.Mem)
	}
	if cons.Tags != nil {
		positives, negatives := parseDelimitedValues(*cons.Tags)
		if len(positives) > 0 {
			params.Tags = positives
		}
		if len(negatives) > 0 {
			params.NotTags = negatives
		}
	}
	if cons.CpuPower != nil {
		logger.Warningf("ignoring unsupported constraint 'cpu-power'")
	}
	return params
}

// convertTagsToParams converts a list of positive/negative tags from
// constraints into two comma-delimited lists of values, which can then be
// passed to MAAS using the "tags" and "not_tags" arguments to acquire. If
// either list of tags is empty, the respective argument is not added to params.
func convertTagsToParams(params url.Values, tags *[]string) {
	if tags == nil || len(*tags) == 0 {
		return
	}
	positives, negatives := parseDelimitedValues(*tags)
	if len(positives) > 0 {
		params.Add("tags", strings.Join(positives, ","))
	}
	if len(negatives) > 0 {
		params.Add("not_tags", strings.Join(negatives, ","))
	}
}

// convertSpacesFromConstraints extracts spaces from constraints and converts
// them to two lists of positive and negative spaces.
func convertSpacesFromConstraints(spaces *[]string) ([]string, []string) {
	if spaces == nil || len(*spaces) == 0 {
		return nil, nil
	}
	return parseDelimitedValues(*spaces)
}

// parseDelimitedValues parses a slice of raw values coming from constraints
// (Tags or Spaces). The result is split into two slices - positives and
// negatives (prefixed with "^"). Empty values are ignored.
func parseDelimitedValues(rawValues []string) (positives, negatives []string) {
	for _, value := range rawValues {
		if value == "" || value == "^" {
			// Neither of these cases should happen in practise, as constraints
			// are validated before setting them and empty names for spaces or
			// tags are not allowed.
			continue
		}
		if strings.HasPrefix(value, "^") {
			negatives = append(negatives, strings.TrimPrefix(value, "^"))
		} else {
			positives = append(positives, value)
		}
	}
	return positives, negatives
}

// interfaceBinding defines a requirement that a node interface must satisfy in
// order for that node to get selected and started, based on deploy-time
// bindings of a service.
//
// TODO(dimitern): Once the services have bindings defined in state, a version
// of this should go to the network package (needs to be non-MAAS-specifc
// first). Also, we need to transform Juju space names from constraints into
// MAAS space provider IDs.
type interfaceBinding struct {
	Name            string
	SpaceProviderId string

	// add more as needed.
}

// numericLabelLimit is a sentinel value used in addInterfaces to limit the
// number of disabmiguation inner loop iterations in case named labels clash
// with numeric labels for spaces coming from constraints. It's defined here to
// facilitate testing this behavior.
var numericLabelLimit uint = 0xffff

// addInterfaces converts a slice of interface bindings, postiveSpaces and
// negativeSpaces coming from constraints to the format MAAS expects for the
// "interfaces" and "not_networks" arguments to acquire node. Returns an error
// satisfying errors.IsNotValid() if the bindings contains duplicates, empty
// Name/SpaceProviderId, or if negative spaces clash with specified bindings.
// Duplicates between specified bindings and positiveSpaces are silently
// skipped.
func addInterfaces(
	params url.Values,
	bindings []interfaceBinding,
	positiveSpaces, negativeSpaces []network.SpaceInfo,
) error {
	combinedBindings, negatives, err := getBindings(bindings, positiveSpaces, negativeSpaces)
	if err != nil {
		return errors.Trace(err)
	}
	if len(combinedBindings) > 0 {
		combinedBindingsString := make([]string, len(combinedBindings))
		for i, binding := range combinedBindings {
			combinedBindingsString[i] = fmt.Sprintf("%s:space=%s", binding.Name, binding.SpaceProviderId)
		}
		params.Add("interfaces", strings.Join(combinedBindingsString, ";"))
	}
	if len(negatives) > 0 {
		negativesString := make([]string, len(negatives))
		for i, binding := range negatives {
			negativesString[i] = fmt.Sprintf("space:%s", binding.SpaceProviderId)
		}
		params.Add("not_networks", strings.Join(negativesString, ","))
	}
	return nil
}

func getBindings(
	bindings []interfaceBinding,
	positiveSpaces, negativeSpaces []network.SpaceInfo,
) ([]interfaceBinding, []interfaceBinding, error) {
	var (
		index            uint
		combinedBindings []interfaceBinding
	)
	namesSet := set.NewStrings()
	spacesSet := set.NewStrings()
	for _, binding := range bindings {
		switch {
		case binding.Name == "":
			return nil, nil, errors.NewNotValid(nil, "interface bindings cannot have empty names")
		case binding.SpaceProviderId == "":
			return nil, nil, errors.NewNotValid(nil, fmt.Sprintf(
				"invalid interface binding %q: space provider ID is required",
				binding.Name,
			))
		case namesSet.Contains(binding.Name):
			return nil, nil, errors.NewNotValid(nil, fmt.Sprintf(
				"duplicated interface binding %q",
				binding.Name,
			))
		}
		namesSet.Add(binding.Name)
		spacesSet.Add(binding.SpaceProviderId)

		combinedBindings = append(combinedBindings, binding)
	}

	createLabel := func(index uint, namesSet set.Strings) (string, uint, error) {
		var label string
		for {
			label = fmt.Sprintf("%v", index)
			if !namesSet.Contains(label) {
				break
			}
			if index > numericLabelLimit { // ...just to make sure we won't loop forever.
				return "", index, errors.Errorf("too many conflicting numeric labels, giving up.")
			}
			index++
		}
		namesSet.Add(label)
		return label, index, nil
	}
	for _, space := range positiveSpaces {
		if spacesSet.Contains(string(space.ProviderId)) {
			// Skip duplicates in positiveSpaces.
			continue
		}
		spacesSet.Add(string(space.ProviderId))

		var label string
		var err error
		label, index, err = createLabel(index, namesSet)
		if err != nil {
			return nil, nil, errors.Trace(err)
		}
		// Make sure we pick a label that doesn't clash with possible bindings.
		combinedBindings = append(combinedBindings, interfaceBinding{label, string(space.ProviderId)})
	}

	var negatives []interfaceBinding
	for _, space := range negativeSpaces {
		if spacesSet.Contains(string(space.ProviderId)) {
			return nil, nil, errors.NewNotValid(nil, fmt.Sprintf(
				"negative space %q from constraints clashes with interface bindings",
				space.Name,
			))
		}
		var label string
		var err error
		label, index, err = createLabel(index, namesSet)
		if err != nil {
			return nil, nil, errors.Trace(err)
		}
		negatives = append(negatives, interfaceBinding{label, string(space.ProviderId)})
	}
	return combinedBindings, negatives, nil
}

func addInterfaces2(
	params *gomaasapi.AllocateMachineArgs,
	bindings []interfaceBinding,
	positiveSpaces, negativeSpaces []network.SpaceInfo,
) error {
	combinedBindings, negatives, err := getBindings(bindings, positiveSpaces, negativeSpaces)
	if err != nil {
		return errors.Trace(err)
	}

	if len(combinedBindings) > 0 {
		interfaceSpecs := make([]gomaasapi.InterfaceSpec, len(combinedBindings))
		for i, space := range combinedBindings {
			interfaceSpecs[i] = gomaasapi.InterfaceSpec{space.Name, space.SpaceProviderId}
		}
		params.Interfaces = interfaceSpecs
	}
	if len(negatives) > 0 {
		negativeStrings := make([]string, len(negatives))
		for i, space := range negatives {
			negativeStrings[i] = space.SpaceProviderId
		}
		params.NotSpace = negativeStrings
	}
	return nil
}

// addStorage converts volume information into url.Values object suitable to
// pass to MAAS when acquiring a node.
func addStorage(params url.Values, volumes []volumeInfo) {
	if len(volumes) == 0 {
		return
	}
	// Requests for specific values are passed to the acquire URL
	// as a storage URL parameter of the form:
	// [volume-name:]sizeinGB[tag,...]
	// See http://maas.ubuntu.com/docs/api.html#nodes

	// eg storage=root:0(ssd),data:20(magnetic,5400rpm),45
	makeVolumeParams := func(v volumeInfo) string {
		var params string
		if v.name != "" {
			params = v.name + ":"
		}
		params += fmt.Sprintf("%d", v.sizeInGB)
		if len(v.tags) > 0 {
			params += fmt.Sprintf("(%s)", strings.Join(v.tags, ","))
		}
		return params
	}
	var volParms []string
	for _, v := range volumes {
		params := makeVolumeParams(v)
		volParms = append(volParms, params)
	}
	params.Add("storage", strings.Join(volParms, ","))
}

// addStorage2 adds volume information onto a gomaasapi.AllocateMachineArgs
// object suitable to pass to MAAS 2 when acquiring a node.
func addStorage2(params *gomaasapi.AllocateMachineArgs, volumes []volumeInfo) {
	if len(volumes) == 0 {
		return
	}
	var volParams []gomaasapi.StorageSpec
	for _, v := range volumes {
		volSpec := gomaasapi.StorageSpec{
			Label: v.name,
			Size:  int(v.sizeInGB),
			Tags:  v.tags,
		}
		volParams = append(volParams, volSpec)
	}
	params.Storage = volParams
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"code.google.com/p/go.net/html"
	_ "crypto/sha512"
	"errors"
	"golang.org/x/net/html/atom"
	"golang.org/x/net/html/charset"
	"golang.org/x/text/transform"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"
)

// how many URLs can the cache store
const cacheSize = 500

// how many hours an entry should be considered valid
const cacheValidHours = 24

// how many kilo bytes should be considered when looking for the title
// tag.
const httpReadKByte = 100

// don’t repost the same title within this period
const noRepostWithinSeconds = 30

// matches all whitespace and zero bytes. Additionally, all Unicode
// characters of class Cf (format chars, e.g. right-to-left) and Cc
// (control chars) are matched.
var whitespaceRegex = regexp.MustCompile(`[\s\0\p{Cf}\p{Cc}]+`)

var ignoreDomainsRegex = regexp.MustCompile(`^http://p\.nnev\.de`)

var twitterDomainRegex = regexp.MustCompile(`(?i)^https?://(?:[a-z0-9]\.)?twitter.com`)
var twitterPicsRegex = regexp.MustCompile(`(?i)(?:\b|^)pic\.twitter\.com/[a-z0-9]+(?:\b|$)`)

var noSpoilerRegex = regexp.MustCompile(`(?i)(don't|no|kein|nicht) spoiler`)

// blacklist pointless titles /////////////////////////////////////////
var pointlessTitles = []string{"",
	"imgur: the simple image sharer",
	"Fefes Blog",
	"Gmane Loom",
	"i3 - A better tiling and dynamic window manager",
	"i3 - improved tiling wm",
	"IT-News, c't, iX, Technology Review, Telepolis | heise online",
	"debian Pastezone",
	"Index of /docs/",
	"NoName e.V. pastebin",
	"Nopaste - powered by project-mindstorm IT Services",
	"Diff NoName e.V. pastebin",
	"pr0gramm.com",
	"Google"}

func runnerUrifind(parsed Message) {
	defer func() {
		if r := recover(); r != nil {
			log.Printf("MEGA-WTF:pkg: %v", r)
		}
	}()

	if parsed.Command != "PRIVMSG" {
		return
	}

	msg := parsed.Trailing

	if noSpoilerRegex.MatchString(msg) {
		log.Printf("not spoilering this line: %s", msg)
		return
	}

	urls := extract(msg)

	for _, url := range urls {
		if url == "" {
			continue
		}

		if cp := cacheGetByUrl(url); cp != nil {
			log.Printf("using cache for URL: %s", cp.url)
			ago := cacheGetTimeAgo(cp)
			postTitle(parsed, cp.title, "cached "+ago+" ago")
			// Hack: add title to the cache again so we can correctly check
			// for reposts, even if the original link has been cached quite
			// some time ago. Since the repost check searches by title, but
			// here we search by URL wie get the correct time when it was
			// cached while still preventing people from using frank to
			// multiply their spamming.
			cacheAdd("", cp.title)
			continue
		}

		go func(url string) {
			if ignoreDomainsRegex.MatchString(url) {
				log.Printf("ignoring this URL: %s", url)
				return
			}

			log.Printf("testing URL: %s", url)
			title, _, err := TitleGet(url)
			if err != nil {
				//postTitle(conn, line, err.Error(), "Error")
			} else if !IsIn(title, pointlessTitles) {
				postTitle(parsed, title, "")
				cacheAdd(url, title)
			}
		}(url)
	}
}

// regexing ////////////////////////////////////////////////////////////

func extract(msg string) []string {
	results := make([]string, 0)
	for idx := strings.Index(msg, "http"); idx > -1; idx = strings.Index(msg, "http") {
		url := msg[idx:]
		if !strings.HasPrefix(url, "http://") &&
			!strings.HasPrefix(url, "https://") {
			msg = msg[idx+len("http"):]
			continue
		}

		// End on commas, but only if they are followed by a space.
		// spiegel.de URLs have commas in them, that would be a
		// false positive otherwise.
		if end := strings.Index(url, ", "); end > -1 {
			url = url[:end]
		}

		// use special handling if the URL contains closing parens
		closingParen := strings.Index(url, ")")
		if closingParen > -1 {
			absPos := idx + closingParen + 1
			if len(msg) > absPos && msg[absPos] == ')' {
				// if an URL ends with double closing parens, assume that the
				// former one belongs to the URL
				url = url[:closingParen+1]
			} else if idx > 0 && msg[idx-1] == '(' {
				// if it ends on a single closing parens (follow by other chars)
				// only remove that closing parens if the URL is directly
				// preceded by one
				url = url[:closingParen]
			}
		}

		// Whitespace always ends a URL.
		if end := strings.IndexAny(url, " \t"); end > -1 {
			url = url[:end]
		}

		results = append(results, url)
		msg = msg[idx+len(url):]
	}
	return results
}

// http/html stuff /////////////////////////////////////////////////////

func TitleGet(url string) (string, string, error) {
	c := http.Client{Timeout: 10 * time.Second}

	r, err := c.Get(url)
	if err != nil {
		log.Printf("WTF: could not resolve %s: %s", url, err)
		return "", url, err
	}
	defer r.Body.Close()

	body, err := ioutil.ReadAll(io.LimitReader(r.Body, 1024*httpReadKByte))
	if err != nil {
		log.Printf("WTF: could not read body for %s: %s", url, err)
		body = []byte{}
	}

	contentType := r.Header.Get("Content-Type")
	encoding, encodingName, _ := charset.DetermineEncoding(body, contentType)
	if encodingName != "utf-8" {
		if *verbose {
			log.Printf("Encoding for URL %s: %s", url, encodingName)
		}

		fixed := transform.NewReader(bytes.NewReader(body), encoding.NewDecoder())
		body, err = ioutil.ReadAll(fixed)
	}

	title, tweet := titleParseHtml(body)
	lastUrl := r.Request.URL.String()

	if r.StatusCode != 200 {
		return "", lastUrl, errors.New("[" + strconv.Itoa(r.StatusCode) + "] " + title)
	}

	if tweet != "" && twitterDomainRegex.MatchString(lastUrl) {
		title = tweet
	}

	log.Printf("Title for URL %s: %s", url, title)

	return title, lastUrl, nil
}

// parses the incoming HTML fragment and tries to extract text from
// suitable tags. Currently this is the page’s title tag and tweets
// when the HTML-code is similar enough to twitter.com. Returns
// title and tweet.
func titleParseHtml(body []byte) (string, string) {
	doc, err := html.Parse(bytes.NewReader(body))
	if err != nil {
		log.Printf("WTF: html parser blew up: %s", err)
		return "", ""
	}

	title := ""
	tweetText := ""
	tweetUserName := ""
	tweetUserScreenName := ""
	tweetPicUrl := ""

	var f func(*html.Node)
	f = func(n *html.Node) {
		if title == "" && n.Type == html.ElementNode && n.DataAtom == atom.Title {
			title = extractText(n)
			return
		}

		if hasClass(n, "permalink-tweet") {
			tweetUserName = getAttr(n, "data-name")
			tweetUserScreenName = getAttr(n, "data-screen-name")
			// find next child “tweet-text”
			for c := n.FirstChild; c != nil; c = c.NextSibling {
				if hasClass(c, "tweet-text") {
					tweetText = extractText(c)
					break
				}
			}
		}

		isMedia := hasClass(n, "media") || hasClass(n, "media-thumbnail")
		if tweetPicUrl == "" && isMedia && !hasClass(n, "profile-picture") {
			attrVal := getAttr(n, "data-url")
			if attrVal != "" {
				tweetPicUrl = attrVal
				return
			}
		}

		// recurse down
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			f(c)
		}

	}
	f(doc)

	// cleanup
	tweet := ""
	tweetUser := ""
	if tweetText != "" {
		tweetText = twitterPicsRegex.ReplaceAllString(tweetText, "")
		tweetUser = tweetUserName + " (@" + tweetUserScreenName + "): "
		tweet = tweetUser + tweetText + " " + tweetPicUrl
		tweet = clean(tweet)
	}

	return strings.TrimSpace(title), strings.TrimSpace(tweet)
}

func extractText(n *html.Node) string {
	text := ""
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		if c.Type == html.TextNode {
			text += c.Data
		} else {
			text += extractText(c)
		}
	}
	return clean(text)
}

func hasClass(n *html.Node, class string) bool {
	if n.Type != html.ElementNode {
		return false
	}

	class = " " + strings.TrimSpace(class) + " "
	attr := strings.Replace(getAttr(n, "class"), "\n", " ", -1)
	if strings.Contains(" "+attr+" ", class) {
		return true
	}
	return false
}

func getAttr(n *html.Node, findAttr string) string {
	for _, attr := range n.Attr {
		if attr.Key == findAttr {
			return attr.Val
		}
	}
	return ""
}

// Cache ///////////////////////////////////////////////////////////////
type Cache struct {
	url   string
	title string
	date  time.Time
}

var cache = [cacheSize]Cache{}
var cacheIndex = 0

func cacheAdd(url string, title string) {
	if len(cache) == cacheIndex {
		cacheIndex = 0
	}
	cache[cacheIndex] = Cache{url, title, time.Now()}
	cacheIndex += 1
}

func cacheGetByUrl(url string) *Cache {
	for _, cc := range cache {
		if cc.url == url && time.Since(cc.date).Hours() <= cacheValidHours {
			return &cc
		}
	}
	return nil
}

func cacheGetTimeAgo(cc *Cache) string {
	ago := time.Since(cc.date).Minutes()
	if ago < 60 {
		return strconv.Itoa(int(ago)) + "m"
	} else {
		hours := strconv.Itoa(int(ago/60.0 + 0.5))
		return hours + "h"
	}
}

func cacheGetSecondsToLastPost(title string) int {
	var secondsAgo = int(^uint(0) >> 1)
	for _, cc := range cache {
		var a = int(time.Since(cc.date).Seconds())
		if cc.title == title && a < secondsAgo {
			secondsAgo = a
		}
	}
	return secondsAgo
}

// util ////////////////////////////////////////////////////////////////

func postTitle(parsed Message, title string, prefix string) {
	tgt := Target(parsed)

	secondsAgo := cacheGetSecondsToLastPost(title)
	if secondsAgo <= noRepostWithinSeconds {
		log.Printf("Skipping, because posted %d seconds ago (“%s”)", secondsAgo, title)
		return
	}

	if *verbose {
		log.Printf("Title was last posted: %#v (“%s”)", secondsAgo, title)
	}

	log.Printf("nick=%s, target=%s, title=%s", Nick(parsed), tgt, title)
	// if target is our current nick, it was a private message.
	// Answer the users in this case.
	if IsPrivateQuery(parsed) {
		tgt = Nick(parsed)
	}
	if prefix == "" {
		prefix = "Link Info"
	} else {
		prefix = clean(prefix)
	}
	title = clean(title)
	// the IRC spec states that notice should be used instead of msg
	// and that bots should not react to notice at all. However, no
	// real world bot adheres to this. Furthermore, people who can’t
	// configure their client to not highlight them on notices will
	// complain.
	Privmsg(tgt, "["+prefix+"] "+title)
}

func clean(text string) string {
	text = whitespaceRegex.ReplaceAllString(text, " ")
	return strings.TrimSpace(text)
}
<commit_msg>speed up title extraction by reading only as much as needed<commit_after>package main

import (
	"bytes"
	"code.google.com/p/go.net/html"
	_ "crypto/sha512"
	"errors"
	"golang.org/x/net/html/atom"
	"golang.org/x/net/html/charset"
	"golang.org/x/text/transform"
	"io"
	"log"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"
)

// how many URLs can the cache store
const cacheSize = 500

// how many hours an entry should be considered valid
const cacheValidHours = 24

// how many bytes should be considered when looking for the title
// tag.
const httpReadByte = 1024 * 100

// don’t repost the same title within this period
const noRepostWithinSeconds = 30

// matches all whitespace and zero bytes. Additionally, all Unicode
// characters of class Cf (format chars, e.g. right-to-left) and Cc
// (control chars) are matched.
var whitespaceRegex = regexp.MustCompile(`[\s\0\p{Cf}\p{Cc}]+`)

var ignoreDomainsRegex = regexp.MustCompile(`^http://p\.nnev\.de`)

var twitterDomainRegex = regexp.MustCompile(`(?i)^https?://(?:[a-z0-9]\.)?twitter.com`)
var twitterPicsRegex = regexp.MustCompile(`(?i)(?:\b|^)pic\.twitter\.com/[a-z0-9]+(?:\b|$)`)

var noSpoilerRegex = regexp.MustCompile(`(?i)(don't|no|kein|nicht) spoiler`)

// blacklist pointless titles /////////////////////////////////////////
var pointlessTitles = []string{"",
	"imgur: the simple image sharer",
	"Fefes Blog",
	"Gmane Loom",
	"i3 - A better tiling and dynamic window manager",
	"i3 - improved tiling wm",
	"IT-News, c't, iX, Technology Review, Telepolis | heise online",
	"debian Pastezone",
	"Index of /docs/",
	"NoName e.V. pastebin",
	"Nopaste - powered by project-mindstorm IT Services",
	"Diff NoName e.V. pastebin",
	"pr0gramm.com",
	"Google"}

func runnerUrifind(parsed Message) {
	defer func() {
		if r := recover(); r != nil {
			log.Printf("MEGA-WTF:pkg: %v", r)
		}
	}()

	if parsed.Command != "PRIVMSG" {
		return
	}

	msg := parsed.Trailing

	if noSpoilerRegex.MatchString(msg) {
		log.Printf("not spoilering this line: %s", msg)
		return
	}

	urls := extract(msg)

	for _, url := range urls {
		if url == "" {
			continue
		}

		if cp := cacheGetByUrl(url); cp != nil {
			log.Printf("using cache for URL: %s", cp.url)
			ago := cacheGetTimeAgo(cp)
			postTitle(parsed, cp.title, "cached "+ago+" ago")
			// Hack: add title to the cache again so we can correctly check
			// for reposts, even if the original link has been cached quite
			// some time ago. Since the repost check searches by title, but
			// here we search by URL wie get the correct time when it was
			// cached while still preventing people from using frank to
			// multiply their spamming.
			cacheAdd("", cp.title)
			continue
		}

		go func(url string) {
			if ignoreDomainsRegex.MatchString(url) {
				log.Printf("ignoring this URL: %s", url)
				return
			}

			log.Printf("testing URL: %s", url)
			title, _, err := TitleGet(url)
			if err != nil {
				//postTitle(conn, line, err.Error(), "Error")
			} else if !IsIn(title, pointlessTitles) {
				postTitle(parsed, title, "")
				cacheAdd(url, title)
			}
		}(url)
	}
}

// regexing ////////////////////////////////////////////////////////////

func extract(msg string) []string {
	results := make([]string, 0)
	for idx := strings.Index(msg, "http"); idx > -1; idx = strings.Index(msg, "http") {
		url := msg[idx:]
		if !strings.HasPrefix(url, "http://") &&
			!strings.HasPrefix(url, "https://") {
			msg = msg[idx+len("http"):]
			continue
		}

		// End on commas, but only if they are followed by a space.
		// spiegel.de URLs have commas in them, that would be a
		// false positive otherwise.
		if end := strings.Index(url, ", "); end > -1 {
			url = url[:end]
		}

		// use special handling if the URL contains closing parens
		closingParen := strings.Index(url, ")")
		if closingParen > -1 {
			absPos := idx + closingParen + 1
			if len(msg) > absPos && msg[absPos] == ')' {
				// if an URL ends with double closing parens, assume that the
				// former one belongs to the URL
				url = url[:closingParen+1]
			} else if idx > 0 && msg[idx-1] == '(' {
				// if it ends on a single closing parens (follow by other chars)
				// only remove that closing parens if the URL is directly
				// preceded by one
				url = url[:closingParen]
			}
		}

		// Whitespace always ends a URL.
		if end := strings.IndexAny(url, " \t"); end > -1 {
			url = url[:end]
		}

		results = append(results, url)
		msg = msg[idx+len(url):]
	}
	return results
}

// http/html stuff /////////////////////////////////////////////////////

func TitleGet(url string) (string, string, error) {
	c := http.Client{Timeout: 10 * time.Second}

	r, err := c.Get(url)
	if err != nil {
		log.Printf("WTF: could not resolve %s: %s", url, err)
		return "", url, err
	}
	defer r.Body.Close()

	lastUrl := r.Request.URL.String()
	isTweet := twitterDomainRegex.MatchString(lastUrl)

	body := io.LimitedReader{r.Body, httpReadByte}
	title := titleParseHtml(&body, isTweet)

	// check encoding using only already retrieved data
	contentType := r.Header.Get("Content-Type")
	bytesRead := httpReadByte - body.N
	if *verbose {
		log.Printf("bytes read to determine title: %d", bytesRead)
	}
	alreadyRead := make([]byte, bytesRead)
	body.R.Read(alreadyRead)
	encoding, encodingName, _ := charset.DetermineEncoding(alreadyRead, contentType)
	if encodingName != "utf-8" {
		if *verbose {
			log.Printf("Encoding for URL %s: %s (%s)", url, encodingName, encoding)
		}

		transTitle, _, transErr := transform.String(encoding.NewDecoder(), title)
		if transErr != nil {
			log.Printf("Encoding Transformation Failed: %s", transErr)
		} else {
			title = transTitle
		}
	}

	if r.StatusCode != 200 {
		return "", lastUrl, errors.New("[" + strconv.Itoa(r.StatusCode) + "] " + title)
	}

	log.Printf("Title for URL %s: %s", url, title)

	return title, lastUrl, nil
}

// parses the incoming HTML fragment and tries to extract text from
// suitable tags. Currently this is the page’s title tag and tweets
// when the HTML-code is similar enough to twitter.com. Returns
// title and tweet.
func titleParseHtml(body io.Reader, searchTweet bool) string {
	z := html.NewTokenizer(body)

	title := ""
	tweetText := ""
	tweetUserName := ""
	tweetUserScreenName := ""
	tweetPicUrl := ""

	titleDepth := -1
	tweetPermalinkDepth := -1
	tweetTextDepth := -1

	depth := 0
TokenizerLoop:
	for {
		tt := z.Next()

		switch tt {
		case html.ErrorToken:
			if z.Err() != io.EOF {
				log.Printf("Could not parse HTML: %s", z.Err())
			}
			break TokenizerLoop

		case html.TextToken:
			text := string(z.Text())
			if titleDepth >= 0 {
				title += text
			}
			if tweetTextDepth >= 0 {
				tweetText += text
			}

		case html.StartTagToken:
			depth++

			tn, hasAttr := z.TagName()

			if bytes.Equal(tn, []byte("title")) {
				titleDepth = depth
				continue
			}

			if !searchTweet {
				continue
			}

			attrs := make(map[string]string)
			for hasAttr {
				var key, val []byte
				key, val, hasAttr = z.TagAttr()
				attrs[atom.String(key)] = string(val)
			}

			if hasClass(attrs, "permalink-tweet") {
				tweetText = ""
				tweetUserName = attrs["data-name"]
				tweetUserScreenName = attrs["data-screen-name"]
				tweetPermalinkDepth = depth
			}

			if hasClass(attrs, "tweet-text") && depth > tweetPermalinkDepth {
				tweetTextDepth = depth
			}

			isMedia := hasClass(attrs, "media") || hasClass(attrs, "media-thumbnail")
			if tweetPicUrl == "" && isMedia && !hasClass(attrs, "profile-picture") {
				tweetPicUrl = attrs["data-url"]
			}

		case html.EndTagToken:
			depth--

			if depth < titleDepth {
				titleDepth = -1
				if title != "" && !searchTweet {
					break TokenizerLoop
				}
			}

			if depth < tweetTextDepth {
				tweetTextDepth = -1
			}

			if tweetText != "" && tweetUserName != "" && tweetUserScreenName != "" && tweetPicUrl != "" {
				break TokenizerLoop
			}
		}
	}

	if tweetText != "" {
		tweetText = twitterPicsRegex.ReplaceAllString(tweetText, "")
		tweetUser := tweetUserName + " (@" + tweetUserScreenName + "): "
		if tweet := clean(tweetUser + tweetText + " " + tweetPicUrl); tweet != "" {
			return tweet
		}
	}

	return clean(title)
}

func hasClass(attrs map[string]string, class string) bool {
	classes := strings.Replace(attrs["class"], "\n", " ", -1)
	class = " " + strings.TrimSpace(class) + " "
	return strings.Contains(" "+classes+" ", class)
}

// Cache ///////////////////////////////////////////////////////////////
type Cache struct {
	url   string
	title string
	date  time.Time
}

var cache = [cacheSize]Cache{}
var cacheIndex = 0

func cacheAdd(url string, title string) {
	if len(cache) == cacheIndex {
		cacheIndex = 0
	}
	cache[cacheIndex] = Cache{url, title, time.Now()}
	cacheIndex += 1
}

func cacheGetByUrl(url string) *Cache {
	for _, cc := range cache {
		if cc.url == url && time.Since(cc.date).Hours() <= cacheValidHours {
			return &cc
		}
	}
	return nil
}

func cacheGetTimeAgo(cc *Cache) string {
	ago := time.Since(cc.date).Minutes()
	if ago < 60 {
		return strconv.Itoa(int(ago)) + "m"
	} else {
		hours := strconv.Itoa(int(ago/60.0 + 0.5))
		return hours + "h"
	}
}

func cacheGetSecondsToLastPost(title string) int {
	var secondsAgo = int(^uint(0) >> 1)
	for _, cc := range cache {
		var a = int(time.Since(cc.date).Seconds())
		if cc.title == title && a < secondsAgo {
			secondsAgo = a
		}
	}
	return secondsAgo
}

// util ////////////////////////////////////////////////////////////////

func postTitle(parsed Message, title string, prefix string) {
	tgt := Target(parsed)

	secondsAgo := cacheGetSecondsToLastPost(title)
	if secondsAgo <= noRepostWithinSeconds {
		log.Printf("Skipping, because posted %d seconds ago (“%s”)", secondsAgo, title)
		return
	}

	if *verbose {
		log.Printf("Title was last posted: %#v (“%s”)", secondsAgo, title)
	}

	log.Printf("nick=%s, target=%s, title=%s", Nick(parsed), tgt, title)
	// if target is our current nick, it was a private message.
	// Answer the users in this case.
	if IsPrivateQuery(parsed) {
		tgt = Nick(parsed)
	}
	if prefix == "" {
		prefix = "Link Info"
	} else {
		prefix = clean(prefix)
	}
	title = clean(title)
	// the IRC spec states that notice should be used instead of msg
	// and that bots should not react to notice at all. However, no
	// real world bot adheres to this. Furthermore, people who can’t
	// configure their client to not highlight them on notices will
	// complain.
	Privmsg(tgt, "["+prefix+"] "+title)
}

func clean(text string) string {
	text = whitespaceRegex.ReplaceAllString(text, " ")
	return strings.TrimSpace(text)
}
<|endoftext|>