<commit_before>// Copyright 2018 Google Inc. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"html/template"
	"io"
	"net/http"
	"path"
	"strings"
	"time"

	"golang.org/x/net/context"
	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/delay"
	"google.golang.org/appengine/log"

	"cloud.google.com/go/storage"
	vision "cloud.google.com/go/vision/apiv1"
	firebase "firebase.google.com/go"
	uuid "github.com/satori/go.uuid"
)

var (
	firebaseConfig = &firebase.Config{
		DatabaseURL:   "https://console.firebase.google.com > Overview > Add Firebase to your web app",
		ProjectID:     "https://console.firebase.google.com > Overview > Add Firebase to your web app",
		StorageBucket: "https://console.firebase.google.com > Overview > Add Firebase to your web app",
	}
	indexTemplate = template.Must(template.ParseFiles("index.html"))
)

// A Label is a description for a post's image.
type Label struct {
	Description string
	Score       float32
}

type Post struct {
	Author   string
	UserID   string
	Message  string
	Posted   time.Time
	ImageURL string
	Labels   []Label
}

type templateParams struct {
	Notice  string
	Name    string
	Message string
	Posts   []Post
}

func main() {
	http.HandleFunc("/", indexHandler)
	appengine.Main()
}

// labelFunc will be called asynchronously as a Cloud Task. labelFunc can
// be executed by calling labelFunc.Call(ctx, postID). If an error is returned
// the function will be retried.
var labelFunc = delay.Func("label-image", func(ctx context.Context, id int64) error {
	// Get the post to label.
	k := datastore.NewKey(ctx, "Post", "", id, nil)
	post := Post{}
	if err := datastore.Get(ctx, k, &post); err != nil {
		log.Errorf(ctx, "getting Post to label: %v", err)
		return err
	}
	if post.ImageURL == "" {
		// Nothing to label.
		return nil
	}

	// Create a new vision client.
	client, err := vision.NewImageAnnotatorClient(ctx)
	if err != nil {
		log.Errorf(ctx, "NewImageAnnotatorClient: %v", err)
		return err
	}
	defer client.Close()

	// Get the image and label it.
	image := vision.NewImageFromURI(post.ImageURL)
	labels, err := client.DetectLabels(ctx, image, nil, 5)
	if err != nil {
		log.Errorf(ctx, "Failed to detect labels: %v", err)
		return err
	}

	for _, l := range labels {
		post.Labels = append(post.Labels, Label{
			Description: l.GetDescription(),
			Score:       l.GetScore(),
		})
	}

	// Update the database with the new labels.
	if _, err := datastore.Put(ctx, k, &post); err != nil {
		log.Errorf(ctx, "Failed to update image: %v", err)
		return err
	}
	return nil
})

// uploadFileFromForm uploads a file if it's present in the "image" form field.
func uploadFileFromForm(ctx context.Context, r *http.Request) (url string, err error) {
	// Read the file from the form.
	f, fh, err := r.FormFile("image")
	if err == http.ErrMissingFile {
		return "", nil
	}
	if err != nil {
		return "", err
	}

	// Ensure the file is an image. http.DetectContentType only uses 512 bytes.
	buf := make([]byte, 512)
	if _, err := f.Read(buf); err != nil {
		return "", err
	}
	if contentType := http.DetectContentType(buf); !strings.HasPrefix(contentType, "image") {
		return "", fmt.Errorf("not an image: %s", contentType)
	}
	// Reset f so subsequent calls to Read start from the beginning of the file.
	f.Seek(0, 0)

	// Create a storage client.
	client, err := storage.NewClient(ctx)
	if err != nil {
		return "", err
	}
	storageBucket := client.Bucket(firebaseConfig.StorageBucket)

	// Random filename, retaining existing extension.
	u, err := uuid.NewV4()
	if err != nil {
		return "", fmt.Errorf("generating UUID: %v", err)
	}
	name := u.String() + path.Ext(fh.Filename)

	w := storageBucket.Object(name).NewWriter(ctx)
	w.ACL = []storage.ACLRule{{Entity: storage.AllUsers, Role: storage.RoleReader}}
	w.ContentType = fh.Header.Get("Content-Type")

	// Entries are immutable, be aggressive about caching (1 day).
	w.CacheControl = "public, max-age=86400"

	if _, err := io.Copy(w, f); err != nil {
		w.CloseWithError(err)
		return "", err
	}
	if err := w.Close(); err != nil {
		return "", err
	}

	const publicURL = "https://storage.googleapis.com/%s/%s"
	return fmt.Sprintf(publicURL, firebaseConfig.StorageBucket, name), nil
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path != "/" {
		http.Redirect(w, r, "/", http.StatusFound)
		return
	}
	ctx := appengine.NewContext(r)
	params := templateParams{}

	q := datastore.NewQuery("Post").Order("-Posted").Limit(20)
	if _, err := q.GetAll(ctx, &params.Posts); err != nil {
		log.Errorf(ctx, "Getting posts: %v", err)
		w.WriteHeader(http.StatusInternalServerError)
		params.Notice = "Couldn't get latest posts. Refresh?"
		indexTemplate.Execute(w, params)
		return
	}

	if r.Method == "GET" {
		indexTemplate.Execute(w, params)
		return
	}
	// It's a POST request, so handle the form submission.

	message := r.FormValue("message")
	if message == "" {
		w.WriteHeader(http.StatusBadRequest)
		params.Notice = "No message provided"
		indexTemplate.Execute(w, params)
		return
	}

	// Create a new Firebase App.
	app, err := firebase.NewApp(ctx, firebaseConfig)
	if err != nil {
		params.Notice = "Couldn't authenticate. Try logging in again?"
		params.Message = message // Preserve their message so they can try again.
		indexTemplate.Execute(w, params)
		return
	}
	// Create a new authenticator for the app.
	auth, err := app.Auth(ctx)
	if err != nil {
		params.Notice = "Couldn't authenticate. Try logging in again?"
		params.Message = message // Preserve their message so they can try again.
		indexTemplate.Execute(w, params)
		return
	}
	// Verify the token passed in by the user is valid.
	tok, err := auth.VerifyIDTokenAndCheckRevoked(ctx, r.FormValue("token"))
	if err != nil {
		params.Notice = "Couldn't authenticate. Try logging in again?"
		params.Message = message // Preserve their message so they can try again.
		indexTemplate.Execute(w, params)
		return
	}
	// Use the validated token to get the user's information.
	user, err := auth.GetUser(ctx, tok.UID)
	if err != nil {
		params.Notice = "Couldn't authenticate. Try logging in again?"
		params.Message = message // Preserve their message so they can try again.
		indexTemplate.Execute(w, params)
		return
	}

	post := Post{
		UserID:  user.UID, // Include UserID in case Author isn't unique.
		Author:  user.DisplayName,
		Message: message,
		Posted:  time.Now(),
	}
	params.Name = post.Author

	// Get the image if there is one.
	imageURL, err := uploadFileFromForm(ctx, r)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		params.Notice = "Error saving image: " + err.Error()
		params.Message = post.Message // Preserve their message so they can try again.
		indexTemplate.Execute(w, params)
		return
	}
	post.ImageURL = imageURL

	key := datastore.NewIncompleteKey(ctx, "Post", nil)
	if key, err = datastore.Put(ctx, key, &post); err != nil {
		log.Errorf(ctx, "datastore.Put: %v", err)
		w.WriteHeader(http.StatusInternalServerError)
		params.Notice = "Couldn't add new post. Try again?"
		params.Message = post.Message // Preserve their message so they can try again.
		indexTemplate.Execute(w, params)
		return
	}

	// Only look for labels if the post has an image.
	if imageURL != "" {
		// Run labelFunc. This will start a new Task in the background.
		if err := labelFunc.Call(ctx, key.IntID()); err != nil {
			log.Errorf(ctx, "delay Call %v", err)
		}
	}

	// Prepend the post that was just added.
	params.Posts = append([]Post{post}, params.Posts...)
	params.Notice = fmt.Sprintf("Thank you for your submission, %s!", post.Author)
	indexTemplate.Execute(w, params)
}
<commit_msg>appengine/gophers/6: add region tags (#489)<commit_after>// Copyright 2018 Google Inc. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"html/template"
	"net/http"
	"time"

	firebase "firebase.google.com/go"
	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/log"

	// [START new_imports]
	"io"
	"path"
	"strings"

	"cloud.google.com/go/storage"
	vision "cloud.google.com/go/vision/apiv1"
	uuid "github.com/satori/go.uuid"
	"golang.org/x/net/context"
	"google.golang.org/appengine/delay"
	// [END new_imports]
)

var (
	firebaseConfig = &firebase.Config{
		DatabaseURL:   "https://console.firebase.google.com > Overview > Add Firebase to your web app",
		ProjectID:     "https://console.firebase.google.com > Overview > Add Firebase to your web app",
		StorageBucket: "https://console.firebase.google.com > Overview > Add Firebase to your web app",
	}
	indexTemplate = template.Must(template.ParseFiles("index.html"))
)

// [START label_struct]

// A Label is a description for a post's image.
type Label struct {
	Description string
	Score       float32
}

// [END label_struct]

// [START new_post_fields]

type Post struct {
	Author   string
	UserID   string
	Message  string
	Posted   time.Time
	ImageURL string
	Labels   []Label
}

// [END new_post_fields]

type templateParams struct {
	Notice  string
	Name    string
	Message string
	Posts   []Post
}

func main() {
	http.HandleFunc("/", indexHandler)
	appengine.Main()
}

// [START var_label_func]

// labelFunc will be called asynchronously as a Cloud Task. labelFunc can
// be executed by calling labelFunc.Call(ctx, postID). If an error is returned
// the function will be retried.
var labelFunc = delay.Func("label-image", func(ctx context.Context, id int64) error {
	// Get the post to label.
	k := datastore.NewKey(ctx, "Post", "", id, nil)
	post := Post{}
	if err := datastore.Get(ctx, k, &post); err != nil {
		log.Errorf(ctx, "getting Post to label: %v", err)
		return err
	}
	if post.ImageURL == "" {
		// Nothing to label.
		return nil
	}

	// Create a new vision client.
	client, err := vision.NewImageAnnotatorClient(ctx)
	if err != nil {
		log.Errorf(ctx, "NewImageAnnotatorClient: %v", err)
		return err
	}
	defer client.Close()

	// Get the image and label it.
	image := vision.NewImageFromURI(post.ImageURL)
	labels, err := client.DetectLabels(ctx, image, nil, 5)
	if err != nil {
		log.Errorf(ctx, "Failed to detect labels: %v", err)
		return err
	}

	for _, l := range labels {
		post.Labels = append(post.Labels, Label{
			Description: l.GetDescription(),
			Score:       l.GetScore(),
		})
	}

	// Update the database with the new labels.
	if _, err := datastore.Put(ctx, k, &post); err != nil {
		log.Errorf(ctx, "Failed to update image: %v", err)
		return err
	}
	return nil
})

// [END var_label_func]

// [START upload_image]

// uploadFileFromForm uploads a file if it's present in the "image" form field.
func uploadFileFromForm(ctx context.Context, r *http.Request) (url string, err error) {
	// Read the file from the form.
	f, fh, err := r.FormFile("image")
	if err == http.ErrMissingFile {
		return "", nil
	}
	if err != nil {
		return "", err
	}

	// Ensure the file is an image. http.DetectContentType only uses 512 bytes.
	buf := make([]byte, 512)
	if _, err := f.Read(buf); err != nil {
		return "", err
	}
	if contentType := http.DetectContentType(buf); !strings.HasPrefix(contentType, "image") {
		return "", fmt.Errorf("not an image: %s", contentType)
	}
	// Reset f so subsequent calls to Read start from the beginning of the file.
	f.Seek(0, 0)

	// Create a storage client.
	client, err := storage.NewClient(ctx)
	if err != nil {
		return "", err
	}
	storageBucket := client.Bucket(firebaseConfig.StorageBucket)

	// Random filename, retaining existing extension.
	u, err := uuid.NewV4()
	if err != nil {
		return "", fmt.Errorf("generating UUID: %v", err)
	}
	name := u.String() + path.Ext(fh.Filename)

	w := storageBucket.Object(name).NewWriter(ctx)
	w.ACL = []storage.ACLRule{{Entity: storage.AllUsers, Role: storage.RoleReader}}
	w.ContentType = fh.Header.Get("Content-Type")

	// Entries are immutable, be aggressive about caching (1 day).
	w.CacheControl = "public, max-age=86400"

	if _, err := io.Copy(w, f); err != nil {
		w.CloseWithError(err)
		return "", err
	}
	if err := w.Close(); err != nil {
		return "", err
	}

	const publicURL = "https://storage.googleapis.com/%s/%s"
	return fmt.Sprintf(publicURL, firebaseConfig.StorageBucket, name), nil
}

// [END upload_image}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path != "/" {
		http.Redirect(w, r, "/", http.StatusFound)
		return
	}
	ctx := appengine.NewContext(r)
	params := templateParams{}

	q := datastore.NewQuery("Post").Order("-Posted").Limit(20)
	if _, err := q.GetAll(ctx, &params.Posts); err != nil {
		log.Errorf(ctx, "Getting posts: %v", err)
		w.WriteHeader(http.StatusInternalServerError)
		params.Notice = "Couldn't get latest posts. Refresh?"
		indexTemplate.Execute(w, params)
		return
	}

	if r.Method == "GET" {
		indexTemplate.Execute(w, params)
		return
	}
	// It's a POST request, so handle the form submission.

	message := r.FormValue("message")
	if message == "" {
		w.WriteHeader(http.StatusBadRequest)
		params.Notice = "No message provided"
		indexTemplate.Execute(w, params)
		return
	}

	// Create a new Firebase App.
	app, err := firebase.NewApp(ctx, firebaseConfig)
	if err != nil {
		params.Notice = "Couldn't authenticate. Try logging in again?"
		params.Message = message // Preserve their message so they can try again.
		indexTemplate.Execute(w, params)
		return
	}
	// Create a new authenticator for the app.
	auth, err := app.Auth(ctx)
	if err != nil {
		params.Notice = "Couldn't authenticate. Try logging in again?"
		params.Message = message // Preserve their message so they can try again.
		indexTemplate.Execute(w, params)
		return
	}
	// Verify the token passed in by the user is valid.
	tok, err := auth.VerifyIDTokenAndCheckRevoked(ctx, r.FormValue("token"))
	if err != nil {
		params.Notice = "Couldn't authenticate. Try logging in again?"
		params.Message = message // Preserve their message so they can try again.
		indexTemplate.Execute(w, params)
		return
	}
	// Use the validated token to get the user's information.
	user, err := auth.GetUser(ctx, tok.UID)
	if err != nil {
		params.Notice = "Couldn't authenticate. Try logging in again?"
		params.Message = message // Preserve their message so they can try again.
		indexTemplate.Execute(w, params)
		return
	}

	post := Post{
		UserID:  user.UID, // Include UserID in case Author isn't unique.
		Author:  user.DisplayName,
		Message: message,
		Posted:  time.Now(),
	}
	params.Name = post.Author

	// [START image_URL]
	// Get the image if there is one.
	imageURL, err := uploadFileFromForm(ctx, r)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		params.Notice = "Error saving image: " + err.Error()
		params.Message = post.Message // Preserve their message so they can try again.
		indexTemplate.Execute(w, params)
		return
	}
	// [END image_URL]

	// [START add_image_URL]
	post.ImageURL = imageURL
	// [END add_image_URL]

	key := datastore.NewIncompleteKey(ctx, "Post", nil)
	if key, err = datastore.Put(ctx, key, &post); err != nil {
		log.Errorf(ctx, "datastore.Put: %v", err)
		w.WriteHeader(http.StatusInternalServerError)
		params.Notice = "Couldn't add new post. Try again?"
		params.Message = post.Message // Preserve their message so they can try again.
		indexTemplate.Execute(w, params)
		return
	}

	// [START empty_image]
	// Only look for labels if the post has an image.
	if imageURL != "" {
		// Run labelFunc. This will start a new Task in the background.
		if err := labelFunc.Call(ctx, key.IntID()); err != nil {
			log.Errorf(ctx, "delay Call %v", err)
		}
	}
	// [END empty_image]

	// Prepend the post that was just added.
	params.Posts = append([]Post{post}, params.Posts...)
	params.Notice = fmt.Sprintf("Thank you for your submission, %s!", post.Author)
	indexTemplate.Execute(w, params)
}
<|endoftext|>


<commit_before>package storage_test

import (
	"context"
	"reflect"
	"testing"

	"github.com/matrix-org/dendrite/mediaapi/storage"
	"github.com/matrix-org/dendrite/mediaapi/types"
	"github.com/matrix-org/dendrite/setup/config"
	"github.com/matrix-org/dendrite/test"
)

func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) {
	connStr, close := test.PrepareDBConnectionString(t, dbType)
	db, err := storage.NewMediaAPIDatasource(&config.DatabaseOptions{
		ConnectionString: config.DataSource(connStr),
	})
	if err != nil {
		t.Fatalf("NewSyncServerDatasource returned %s", err)
	}
	return db, close
}
func TestMediaRepository(t *testing.T) {
	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
		db, close := mustCreateDatabase(t, dbType)
		defer close()
		ctx := context.Background()
		t.Run("can insert media & query media", func(t *testing.T) {
			metadata := &types.MediaMetadata{
				MediaID:       "testing",
				Origin:        "localhost",
				ContentType:   "image/png",
				FileSizeBytes: 10,
				UploadName:    "upload test",
				Base64Hash:    "dGVzdGluZw==",
				UserID:        "@alice:localhost",
			}
			if err := db.StoreMediaMetadata(ctx, metadata); err != nil {
				t.Fatalf("unable to store media metadata: %v", err)
			}
			// query by media id
			gotMetadata, err := db.GetMediaMetadata(ctx, metadata.MediaID, metadata.Origin)
			if err != nil {
				t.Fatalf("unable to query media metadata: %v", err)
			}
			if !reflect.DeepEqual(metadata, gotMetadata) {
				t.Fatalf("expected metadata %+v, got %v", metadata, gotMetadata)
			}
			// query by media hash
			gotMetadata, err = db.GetMediaMetadataByHash(ctx, metadata.Base64Hash, metadata.Origin)
			if err != nil {
				t.Fatalf("unable to query media metadata by hash: %v", err)
			}
			if !reflect.DeepEqual(metadata, gotMetadata) {
				t.Fatalf("expected metadata %+v, got %v", metadata, gotMetadata)
			}
		})
	})
}

func TestThumbnailsStorage(t *testing.T) {
	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
		db, close := mustCreateDatabase(t, dbType)
		defer close()
		ctx := context.Background()
		t.Run("can insert thumbnails & query media", func(t *testing.T) {
			thumbnails := []*types.ThumbnailMetadata{
				{
					MediaMetadata: &types.MediaMetadata{
						MediaID:       "testing",
						Origin:        "localhost",
						ContentType:   "image/png",
						FileSizeBytes: 6,
					},
					ThumbnailSize: types.ThumbnailSize{
						Width:        5,
						Height:       5,
						ResizeMethod: types.Crop,
					},
				},
				{
					MediaMetadata: &types.MediaMetadata{
						MediaID:       "testing",
						Origin:        "localhost",
						ContentType:   "image/png",
						FileSizeBytes: 7,
					},
					ThumbnailSize: types.ThumbnailSize{
						Width:        1,
						Height:       1,
						ResizeMethod: types.Scale,
					},
				},
			}
			for i := range thumbnails {
				if err := db.StoreThumbnail(ctx, thumbnails[i]); err != nil {
					t.Fatalf("unable to store thumbnail metadata: %v", err)
				}
			}
			// query by single thumbnail
			gotMetadata, err := db.GetThumbnail(ctx,
				thumbnails[0].MediaMetadata.MediaID,
				thumbnails[0].MediaMetadata.Origin,
				thumbnails[0].ThumbnailSize.Width, thumbnails[0].ThumbnailSize.Height,
				thumbnails[0].ThumbnailSize.ResizeMethod,
			)
			if err != nil {
				t.Fatalf("unable to query thumbnail metadata: %v", err)
			}
			if !reflect.DeepEqual(thumbnails[0].MediaMetadata, gotMetadata.MediaMetadata) {
				t.Fatalf("expected metadata %+v, got %+v", thumbnails[0].MediaMetadata, gotMetadata.MediaMetadata)
			}
			if !reflect.DeepEqual(thumbnails[0].ThumbnailSize, gotMetadata.ThumbnailSize) {
				t.Fatalf("expected metadata %+v, got %+v", thumbnails[0].MediaMetadata, gotMetadata.MediaMetadata)
			}
			// query by all thumbnails
			gotMediadatas, err := db.GetThumbnails(ctx, thumbnails[0].MediaMetadata.MediaID, thumbnails[0].MediaMetadata.Origin)
			if err != nil {
				t.Fatalf("unable to query media metadata by hash: %v", err)
			}
			if len(gotMediadatas) != len(thumbnails) {
				t.Fatalf("expected %d stored thumbnail metadata, got %d", len(thumbnails), len(gotMediadatas))
			}
			for i := range gotMediadatas {
				if !reflect.DeepEqual(thumbnails[i].MediaMetadata, gotMediadatas[i].MediaMetadata) {
					t.Fatalf("expected metadata %+v, got %v", thumbnails[i].MediaMetadata, gotMediadatas[i].MediaMetadata)
				}
				if !reflect.DeepEqual(thumbnails[i].ThumbnailSize, gotMediadatas[i].ThumbnailSize) {
					t.Fatalf("expected metadata %+v, got %v", thumbnails[i].ThumbnailSize, gotMediadatas[i].ThumbnailSize)
				}
			}
		})
	})
}
<commit_msg>Fix `TestThumbnailsStorage` failing when media results come back in non-deterministic order; silence expected error when tests are run multiple times against the same postgres database (cherry-picked from #2395)<commit_after>package storage_test

import (
	"context"
	"reflect"
	"testing"

	"github.com/matrix-org/dendrite/mediaapi/storage"
	"github.com/matrix-org/dendrite/mediaapi/types"
	"github.com/matrix-org/dendrite/setup/config"
	"github.com/matrix-org/dendrite/test"
)

func mustCreateDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) {
	connStr, close := test.PrepareDBConnectionString(t, dbType)
	db, err := storage.NewMediaAPIDatasource(&config.DatabaseOptions{
		ConnectionString: config.DataSource(connStr),
	})
	if err != nil {
		t.Fatalf("NewSyncServerDatasource returned %s", err)
	}
	return db, close
}
func TestMediaRepository(t *testing.T) {
	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
		db, close := mustCreateDatabase(t, dbType)
		defer close()
		ctx := context.Background()
		t.Run("can insert media & query media", func(t *testing.T) {
			metadata := &types.MediaMetadata{
				MediaID:       "testing",
				Origin:        "localhost",
				ContentType:   "image/png",
				FileSizeBytes: 10,
				UploadName:    "upload test",
				Base64Hash:    "dGVzdGluZw==",
				UserID:        "@alice:localhost",
			}
			if err := db.StoreMediaMetadata(ctx, metadata); err != nil {
				t.Fatalf("unable to store media metadata: %v", err)
			}
			// query by media id
			gotMetadata, err := db.GetMediaMetadata(ctx, metadata.MediaID, metadata.Origin)
			if err != nil {
				t.Fatalf("unable to query media metadata: %v", err)
			}
			if !reflect.DeepEqual(metadata, gotMetadata) {
				t.Fatalf("expected metadata %+v, got %v", metadata, gotMetadata)
			}
			// query by media hash
			gotMetadata, err = db.GetMediaMetadataByHash(ctx, metadata.Base64Hash, metadata.Origin)
			if err != nil {
				t.Fatalf("unable to query media metadata by hash: %v", err)
			}
			if !reflect.DeepEqual(metadata, gotMetadata) {
				t.Fatalf("expected metadata %+v, got %v", metadata, gotMetadata)
			}
		})
	})
}

func TestThumbnailsStorage(t *testing.T) {
	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
		db, close := mustCreateDatabase(t, dbType)
		defer close()
		ctx := context.Background()
		t.Run("can insert thumbnails & query media", func(t *testing.T) {
			thumbnails := []*types.ThumbnailMetadata{
				{
					MediaMetadata: &types.MediaMetadata{
						MediaID:       "testing",
						Origin:        "localhost",
						ContentType:   "image/png",
						FileSizeBytes: 6,
					},
					ThumbnailSize: types.ThumbnailSize{
						Width:        5,
						Height:       5,
						ResizeMethod: types.Crop,
					},
				},
				{
					MediaMetadata: &types.MediaMetadata{
						MediaID:       "testing",
						Origin:        "localhost",
						ContentType:   "image/png",
						FileSizeBytes: 7,
					},
					ThumbnailSize: types.ThumbnailSize{
						Width:        1,
						Height:       1,
						ResizeMethod: types.Scale,
					},
				},
			}
			for i := range thumbnails {
				if err := db.StoreThumbnail(ctx, thumbnails[i]); err != nil {
					t.Fatalf("unable to store thumbnail metadata: %v", err)
				}
			}
			// query by single thumbnail
			gotMetadata, err := db.GetThumbnail(ctx,
				thumbnails[0].MediaMetadata.MediaID,
				thumbnails[0].MediaMetadata.Origin,
				thumbnails[0].ThumbnailSize.Width, thumbnails[0].ThumbnailSize.Height,
				thumbnails[0].ThumbnailSize.ResizeMethod,
			)
			if err != nil {
				t.Fatalf("unable to query thumbnail metadata: %v", err)
			}
			if !reflect.DeepEqual(thumbnails[0].MediaMetadata, gotMetadata.MediaMetadata) {
				t.Fatalf("expected metadata %+v, got %+v", thumbnails[0].MediaMetadata, gotMetadata.MediaMetadata)
			}
			if !reflect.DeepEqual(thumbnails[0].ThumbnailSize, gotMetadata.ThumbnailSize) {
				t.Fatalf("expected metadata %+v, got %+v", thumbnails[0].MediaMetadata, gotMetadata.MediaMetadata)
			}
			// query by all thumbnails
			gotMediadatas, err := db.GetThumbnails(ctx, thumbnails[0].MediaMetadata.MediaID, thumbnails[0].MediaMetadata.Origin)
			if err != nil {
				t.Fatalf("unable to query media metadata by hash: %v", err)
			}
			if len(gotMediadatas) != len(thumbnails) {
				t.Fatalf("expected %d stored thumbnail metadata, got %d", len(thumbnails), len(gotMediadatas))
			}
			for i := range gotMediadatas {
				// metadata may be returned in a different order than it was stored, perform a search
				metaDataMatches := func() bool {
					for _, t := range thumbnails {
						if reflect.DeepEqual(t.MediaMetadata, gotMediadatas[i].MediaMetadata) && reflect.DeepEqual(t.ThumbnailSize, gotMediadatas[i].ThumbnailSize) {
							return true
						}
					}
					return false
				}

				if !metaDataMatches() {
					t.Fatalf("expected metadata %+v, got %+v", thumbnails[i].MediaMetadata, gotMediadatas[i].MediaMetadata)

				}
			}
		})
	})
}
<|endoftext|>


<commit_before>package server

import (
	"io"
	"net/http"
	"strconv"

	"github.com/docker/distribution"
	"github.com/docker/distribution/context"
	"github.com/docker/distribution/digest"

	"k8s.io/kubernetes/pkg/api/errors"

	imageapi "github.com/openshift/origin/pkg/image/api"
	"github.com/openshift/origin/pkg/image/importer"
)

// pullthroughBlobStore wraps a distribution.BlobStore and allows remote repositories to serve blobs from remote
// repositories.
type pullthroughBlobStore struct {
	distribution.BlobStore

	repo          *repository
	digestToStore map[string]distribution.BlobStore
}

var _ distribution.BlobStore = &pullthroughBlobStore{}

// Stat makes a local check for the blob, then falls through to the other servers referenced by
// the image stream and looks for those that have the layer.
func (r *pullthroughBlobStore) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) {
	// check the local store for the blob
	desc, err := r.BlobStore.Stat(ctx, dgst)
	switch {
	case err == distribution.ErrBlobUnknown:
		// continue on to the code below and look up the blob in a remote store since it is not in
		// the local store
	case err != nil:
		context.GetLogger(r.repo.ctx).Errorf("Failed to find blob %q: %#v", dgst.String(), err)
		fallthrough
	default:
		return desc, err
	}

	// look up the potential remote repositories that this blob could be part of (at this time,
	// we don't know which image in the image stream surfaced the content).
	is, err := r.repo.getImageStream()
	if err != nil {
		if errors.IsNotFound(err) || errors.IsForbidden(err) {
			return distribution.Descriptor{}, distribution.ErrBlobUnknown
		}
		context.GetLogger(r.repo.ctx).Errorf("Error retrieving image stream for blob: %s", err)
		return distribution.Descriptor{}, err
	}

	var localRegistry string
	if local, err := imageapi.ParseDockerImageReference(is.Status.DockerImageRepository); err == nil {
		// TODO: normalize further?
		localRegistry = local.Registry
	}

	retriever := r.repo.importContext()
	cached := r.repo.cachedLayers.RepositoriesForDigest(dgst)

	// look at the first level of tagged repositories first
	search := identifyCandidateRepositories(is, localRegistry, true)
	if desc, err := r.findCandidateRepository(ctx, search, cached, dgst, retriever); err == nil {
		return desc, nil
	}

	// look at all other repositories tagged by the server
	secondary := identifyCandidateRepositories(is, localRegistry, false)
	for k := range search {
		delete(secondary, k)
	}
	if desc, err := r.findCandidateRepository(ctx, secondary, cached, dgst, retriever); err == nil {
		return desc, nil
	}

	return distribution.Descriptor{}, distribution.ErrBlobUnknown
}

// proxyStat attempts to locate the digest in the provided remote repository or returns an error. If the digest is found,
// r.digestToStore saves the store.
func (r *pullthroughBlobStore) proxyStat(ctx context.Context, retriever importer.RepositoryRetriever, ref imageapi.DockerImageReference, dgst digest.Digest) (distribution.Descriptor, error) {
	context.GetLogger(r.repo.ctx).Infof("Trying to stat %q from %q", dgst, ref.Exact())
	repo, err := retriever.Repository(ctx, ref.RegistryURL(), ref.RepositoryName(), false)
	if err != nil {
		context.GetLogger(r.repo.ctx).Errorf("Error getting remote repository for image %q: %v", ref.Exact(), err)
		return distribution.Descriptor{}, err
	}
	pullthroughBlobStore := repo.Blobs(ctx)
	desc, err := pullthroughBlobStore.Stat(r.repo.ctx, dgst)
	if err != nil {
		if err != distribution.ErrBlobUnknown {
			context.GetLogger(r.repo.ctx).Errorf("Error getting pullthroughBlobStore for image %q: %v", ref.Exact(), err)
		}
		return distribution.Descriptor{}, err
	}

	r.digestToStore[dgst.String()] = pullthroughBlobStore
	return desc, nil
}

// ServeBlob attempts to serve the requested digest onto w, using a remote proxy store if necessary.
func (r *pullthroughBlobStore) ServeBlob(ctx context.Context, w http.ResponseWriter, req *http.Request, dgst digest.Digest) error {
	store, ok := r.digestToStore[dgst.String()]
	if !ok {
		return r.BlobStore.ServeBlob(ctx, w, req, dgst)
	}

	desc, err := store.Stat(ctx, dgst)
	if err != nil {
		context.GetLogger(r.repo.ctx).Errorf("Failed to stat digest %q: %v", dgst.String(), err)
		return err
	}

	remoteReader, err := store.Open(ctx, dgst)
	if err != nil {
		context.GetLogger(r.repo.ctx).Errorf("Failure to open remote store %q: %v", dgst.String(), err)
		return err
	}

	setResponseHeaders(w, desc.Size, desc.MediaType, dgst)

	context.GetLogger(r.repo.ctx).Infof("Copying %d bytes of type %q for %q", desc.Size, desc.MediaType, dgst.String())
	if _, err := io.CopyN(w, remoteReader, desc.Size); err != nil {
		context.GetLogger(r.repo.ctx).Errorf("Failed copying content from remote store %q: %v", dgst.String(), err)
		return err
	}
	return nil
}

// findCandidateRepository looks in search for a particular blob, referring to previously cached items
func (r *pullthroughBlobStore) findCandidateRepository(ctx context.Context, search map[string]*imageapi.DockerImageReference, cachedLayers []string, dgst digest.Digest, retriever importer.RepositoryRetriever) (distribution.Descriptor, error) {
	// no possible remote locations to search, exit early
	if len(search) == 0 {
		return distribution.Descriptor{}, distribution.ErrBlobUnknown
	}

	// see if any of the previously located repositories containing this digest are in this
	// image stream
	for _, repo := range cachedLayers {
		ref, ok := search[repo]
		if !ok {
			continue
		}
		desc, err := r.proxyStat(ctx, retriever, *ref, dgst)
		if err != nil {
			delete(search, repo)
			continue
		}
		context.GetLogger(r.repo.ctx).Infof("Found digest location from cache %q in %q: %v", dgst, repo, err)
		return desc, nil
	}

	// search the remaining registries for this digest
	for repo, ref := range search {
		desc, err := r.proxyStat(ctx, retriever, *ref, dgst)
		if err != nil {
			continue
		}
		r.repo.cachedLayers.RememberDigest(dgst, repo)
		context.GetLogger(r.repo.ctx).Infof("Found digest location by search %q in %q: %v", dgst, repo, err)
		return desc, nil
	}

	return distribution.Descriptor{}, distribution.ErrBlobUnknown
}

// identifyCandidateRepositories returns a map of remote repositories referenced by this image stream.
func identifyCandidateRepositories(is *imageapi.ImageStream, localRegistry string, primary bool) map[string]*imageapi.DockerImageReference {
	// identify the canonical location of referenced registries to search
	search := make(map[string]*imageapi.DockerImageReference)
	for _, tagEvent := range is.Status.Tags {
		var candidates []imageapi.TagEvent
		if primary {
			if len(tagEvent.Items) == 0 {
				continue
			}
			candidates = tagEvent.Items[:1]
		} else {
			if len(tagEvent.Items) <= 1 {
				continue
			}
			candidates = tagEvent.Items[1:]
		}
		for _, event := range candidates {
			ref, err := imageapi.ParseDockerImageReference(event.DockerImageReference)
			if err != nil {
				continue
			}
			// skip anything that matches the innate registry
			// TODO: there may be a better way to make this determination
			if len(localRegistry) != 0 && localRegistry == ref.Registry {
				continue
			}
			ref = ref.DockerClientDefaults()
			search[ref.AsRepository().Exact()] = &ref
		}
	}
	return search
}

// setResponseHeaders sets the appropriate content serving headers
func setResponseHeaders(w http.ResponseWriter, length int64, mediaType string, digest digest.Digest) {
	w.Header().Set("Content-Length", strconv.FormatInt(length, 10))
	w.Header().Set("Content-Type", mediaType)
	w.Header().Set("Docker-Content-Digest", digest.String())
	w.Header().Set("Etag", digest.String())
}
<commit_msg>Change the context being used when Stat-ing remote registry<commit_after>package server

import (
	"io"
	"net/http"
	"strconv"

	"github.com/docker/distribution"
	"github.com/docker/distribution/context"
	"github.com/docker/distribution/digest"

	"k8s.io/kubernetes/pkg/api/errors"

	imageapi "github.com/openshift/origin/pkg/image/api"
	"github.com/openshift/origin/pkg/image/importer"
)

// pullthroughBlobStore wraps a distribution.BlobStore and allows remote repositories to serve blobs from remote
// repositories.
type pullthroughBlobStore struct {
	distribution.BlobStore

	repo          *repository
	digestToStore map[string]distribution.BlobStore
}

var _ distribution.BlobStore = &pullthroughBlobStore{}

// Stat makes a local check for the blob, then falls through to the other servers referenced by
// the image stream and looks for those that have the layer.
func (r *pullthroughBlobStore) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) {
	// check the local store for the blob
	desc, err := r.BlobStore.Stat(ctx, dgst)
	switch {
	case err == distribution.ErrBlobUnknown:
		// continue on to the code below and look up the blob in a remote store since it is not in
		// the local store
	case err != nil:
		context.GetLogger(r.repo.ctx).Errorf("Failed to find blob %q: %#v", dgst.String(), err)
		fallthrough
	default:
		return desc, err
	}

	// look up the potential remote repositories that this blob could be part of (at this time,
	// we don't know which image in the image stream surfaced the content).
	is, err := r.repo.getImageStream()
	if err != nil {
		if errors.IsNotFound(err) || errors.IsForbidden(err) {
			return distribution.Descriptor{}, distribution.ErrBlobUnknown
		}
		context.GetLogger(r.repo.ctx).Errorf("Error retrieving image stream for blob: %s", err)
		return distribution.Descriptor{}, err
	}

	var localRegistry string
	if local, err := imageapi.ParseDockerImageReference(is.Status.DockerImageRepository); err == nil {
		// TODO: normalize further?
		localRegistry = local.Registry
	}

	retriever := r.repo.importContext()
	cached := r.repo.cachedLayers.RepositoriesForDigest(dgst)

	// look at the first level of tagged repositories first
	search := identifyCandidateRepositories(is, localRegistry, true)
	if desc, err := r.findCandidateRepository(ctx, search, cached, dgst, retriever); err == nil {
		return desc, nil
	}

	// look at all other repositories tagged by the server
	secondary := identifyCandidateRepositories(is, localRegistry, false)
	for k := range search {
		delete(secondary, k)
	}
	if desc, err := r.findCandidateRepository(ctx, secondary, cached, dgst, retriever); err == nil {
		return desc, nil
	}

	return distribution.Descriptor{}, distribution.ErrBlobUnknown
}

// proxyStat attempts to locate the digest in the provided remote repository or returns an error. If the digest is found,
// r.digestToStore saves the store.
func (r *pullthroughBlobStore) proxyStat(ctx context.Context, retriever importer.RepositoryRetriever, ref imageapi.DockerImageReference, dgst digest.Digest) (distribution.Descriptor, error) {
	context.GetLogger(r.repo.ctx).Infof("Trying to stat %q from %q", dgst, ref.Exact())
	repo, err := retriever.Repository(ctx, ref.RegistryURL(), ref.RepositoryName(), false)
	if err != nil {
		context.GetLogger(r.repo.ctx).Errorf("Error getting remote repository for image %q: %v", ref.Exact(), err)
		return distribution.Descriptor{}, err
	}
	pullthroughBlobStore := repo.Blobs(ctx)
	desc, err := pullthroughBlobStore.Stat(ctx, dgst)
	if err != nil {
		if err != distribution.ErrBlobUnknown {
			context.GetLogger(r.repo.ctx).Errorf("Error getting pullthroughBlobStore for image %q: %v", ref.Exact(), err)
		}
		return distribution.Descriptor{}, err
	}

	r.digestToStore[dgst.String()] = pullthroughBlobStore
	return desc, nil
}

// ServeBlob attempts to serve the requested digest onto w, using a remote proxy store if necessary.
func (r *pullthroughBlobStore) ServeBlob(ctx context.Context, w http.ResponseWriter, req *http.Request, dgst digest.Digest) error {
	store, ok := r.digestToStore[dgst.String()]
	if !ok {
		return r.BlobStore.ServeBlob(ctx, w, req, dgst)
	}

	desc, err := store.Stat(ctx, dgst)
	if err != nil {
		context.GetLogger(r.repo.ctx).Errorf("Failed to stat digest %q: %v", dgst.String(), err)
		return err
	}

	remoteReader, err := store.Open(ctx, dgst)
	if err != nil {
		context.GetLogger(r.repo.ctx).Errorf("Failure to open remote store %q: %v", dgst.String(), err)
		return err
	}

	setResponseHeaders(w, desc.Size, desc.MediaType, dgst)

	context.GetLogger(r.repo.ctx).Infof("Copying %d bytes of type %q for %q", desc.Size, desc.MediaType, dgst.String())
	if _, err := io.CopyN(w, remoteReader, desc.Size); err != nil {
		context.GetLogger(r.repo.ctx).Errorf("Failed copying content from remote store %q: %v", dgst.String(), err)
		return err
	}
	return nil
}

// findCandidateRepository looks in search for a particular blob, referring to previously cached items
func (r *pullthroughBlobStore) findCandidateRepository(ctx context.Context, search map[string]*imageapi.DockerImageReference, cachedLayers []string, dgst digest.Digest, retriever importer.RepositoryRetriever) (distribution.Descriptor, error) {
	// no possible remote locations to search, exit early
	if len(search) == 0 {
		return distribution.Descriptor{}, distribution.ErrBlobUnknown
	}

	// see if any of the previously located repositories containing this digest are in this
	// image stream
	for _, repo := range cachedLayers {
		ref, ok := search[repo]
		if !ok {
			continue
		}
		desc, err := r.proxyStat(ctx, retriever, *ref, dgst)
		if err != nil {
			delete(search, repo)
			continue
		}
		context.GetLogger(r.repo.ctx).Infof("Found digest location from cache %q in %q: %v", dgst, repo, err)
		return desc, nil
	}

	// search the remaining registries for this digest
	for repo, ref := range search {
		desc, err := r.proxyStat(ctx, retriever, *ref, dgst)
		if err != nil {
			continue
		}
		r.repo.cachedLayers.RememberDigest(dgst, repo)
		context.GetLogger(r.repo.ctx).Infof("Found digest location by search %q in %q: %v", dgst, repo, err)
		return desc, nil
	}

	return distribution.Descriptor{}, distribution.ErrBlobUnknown
}

// identifyCandidateRepositories returns a map of remote repositories referenced by this image stream.
func identifyCandidateRepositories(is *imageapi.ImageStream, localRegistry string, primary bool) map[string]*imageapi.DockerImageReference {
	// identify the canonical location of referenced registries to search
	search := make(map[string]*imageapi.DockerImageReference)
	for _, tagEvent := range is.Status.Tags {
		var candidates []imageapi.TagEvent
		if primary {
			if len(tagEvent.Items) == 0 {
				continue
			}
			candidates = tagEvent.Items[:1]
		} else {
			if len(tagEvent.Items) <= 1 {
				continue
			}
			candidates = tagEvent.Items[1:]
		}
		for _, event := range candidates {
			ref, err := imageapi.ParseDockerImageReference(event.DockerImageReference)
			if err != nil {
				continue
			}
			// skip anything that matches the innate registry
			// TODO: there may be a better way to make this determination
			if len(localRegistry) != 0 && localRegistry == ref.Registry {
				continue
			}
			ref = ref.DockerClientDefaults()
			search[ref.AsRepository().Exact()] = &ref
		}
	}
	return search
}

// setResponseHeaders sets the appropriate content serving headers
func setResponseHeaders(w http.ResponseWriter, length int64, mediaType string, digest digest.Digest) {
	w.Header().Set("Content-Length", strconv.FormatInt(length, 10))
	w.Header().Set("Content-Type", mediaType)
	w.Header().Set("Docker-Content-Digest", digest.String())
	w.Header().Set("Etag", digest.String())
}
<|endoftext|>


<commit_before>package sparkle

import (
	"errors"
	"net/http"
	"reflect"
	"strconv"
)

var ErrNilUnmarshalTarget = errors.New("Passed nil object as unmarshal target")
var ErrNotPointerTarget = errors.New("Target is not a pointer")
var ErrNotStructTarget = errors.New("Target pointer does not point to struct")
var ErrCouldNotGetContextRequest = errors.New("Could not obtain request from context")
var ErrUnsupportedType = errors.New("Can not unmarshall to field type as it is unsupported")

var maxMemoryForMultipartForm int64

// Sets the Max Memory to be used when UnmarshalParameters encounters a
// multipart form.
func SetMaxMemory(maxMemory int64) {
	maxMemoryForMultipartForm = maxMemory
}

func canUnmarshal(v interface{}) error {
	if v == nil {
		return ErrNilUnmarshalTarget
	}

	it := reflect.ValueOf(v)
	if it.Kind() != reflect.Ptr {
		return ErrNotPointerTarget
	}

	if rt := it.Elem(); rt.Kind() != reflect.Struct {
		return ErrNotStructTarget
	}

	return nil
}

func parseFormData(r *http.Request) error {
	if r.Header.Get("Content-Type") == "multipart/form-data" {
		return r.ParseMultipartForm(maxMemoryForMultipartForm)
	}

	return r.ParseForm()
}

func getFieldAndKind(rt reflect.Value, fieldName string) (reflect.Value, reflect.Kind) {
	// Look for property on structure with the same name
	fieldValue := rt.FieldByName(fieldName)
	fieldKind := fieldValue.Kind()
	if fieldKind == reflect.Ptr {
		// Step one in if it's a pointer
		fieldValue = fieldValue.Elem()
		fieldKind = fieldValue.Kind()
	}

	return fieldValue, fieldKind
}

// Unmarshals Query and Post parameters to an object supplied in v
//
// If successful returns nil with the values in v set accordingly
// Returns ErrNilUnmarshalTarget if v was nil
// Returns ErrNotPointerTarget if v was not a pointer
// Returns ErrNotStructTarget if v was not a pointer to a struct
func (c *Context) UnmarshalParameters(v interface{}) error {

	if err := canUnmarshal(v); err != nil {
		return err
	}

	request := c.Request()
	if request == nil {
		// Should never happen
		return ErrCouldNotGetContextRequest
	}

	if err := parseFormData(request); err != nil {
		return err
	}

	rt := reflect.ValueOf(v).Elem()

	// Okay, so we should step through the values in the form now
	for key, value := range request.Form {
		fieldValue, fieldKind := getFieldAndKind(rt, key)

		// if we can't set the result, then ignore it
		if !fieldValue.CanSet() {
			continue
		}

		switch fieldKind {
		case reflect.String:
			fieldValue.SetString(value[0])

		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
			if v, err := strconv.ParseUint(value[0], 10, 64); err != nil {
				return err
			} else {
				fieldValue.SetUint(v)
			}
		case reflect.Int:
			if v, err := strconv.ParseInt(value[0], 10, 64); err != nil {
				return err
			} else {
				fieldValue.SetInt(v)
			}
		case reflect.Bool:
			if v, err := strconv.ParseBool(value[0]); err != nil {
				return err
			} else {
				fieldValue.SetBool(v)
			}
		default:
			return ErrUnsupportedType
		}
	}

	return nil
}
<commit_msg>Updated documentation on UnmarshalParameters<commit_after>package sparkle

import (
	"errors"
	"net/http"
	"reflect"
	"strconv"
)

var ErrNilUnmarshalTarget = errors.New("Passed nil object as unmarshal target")
var ErrNotPointerTarget = errors.New("Target is not a pointer")
var ErrNotStructTarget = errors.New("Target pointer does not point to struct")
var ErrCouldNotGetContextRequest = errors.New("Could not obtain request from context")
var ErrUnsupportedType = errors.New("Can not unmarshall to field type as it is unsupported")

var maxMemoryForMultipartForm int64

// Sets the Max Memory to be used when UnmarshalParameters encounters a
// multipart form.
func SetMaxMemory(maxMemory int64) {
	maxMemoryForMultipartForm = maxMemory
}

func canUnmarshal(v interface{}) error {
	if v == nil {
		return ErrNilUnmarshalTarget
	}

	it := reflect.ValueOf(v)
	if it.Kind() != reflect.Ptr {
		return ErrNotPointerTarget
	}

	if rt := it.Elem(); rt.Kind() != reflect.Struct {
		return ErrNotStructTarget
	}

	return nil
}

func parseFormData(r *http.Request) error {
	if r.Header.Get("Content-Type") == "multipart/form-data" {
		return r.ParseMultipartForm(maxMemoryForMultipartForm)
	}

	return r.ParseForm()
}

func getFieldAndKind(rt reflect.Value, fieldName string) (reflect.Value, reflect.Kind) {
	// Look for property on structure with the same name
	fieldValue := rt.FieldByName(fieldName)
	fieldKind := fieldValue.Kind()
	if fieldKind == reflect.Ptr {
		// Step one in if it's a pointer
		fieldValue = fieldValue.Elem()
		fieldKind = fieldValue.Kind()
	}

	return fieldValue, fieldKind
}

// Unmarshals Query and Post parameters to an object supplied in v
//
// If a parameter doesn't have a corresponding field in the struct, it is
// ignored. If the struct has a field that doesn't have a corresponding
// parameter, then the struct field will not be altered.
//
// If successful returns nil with the values in v set accordingly
// Returns ErrNilUnmarshalTarget if v was nil
// Returns ErrNotPointerTarget if v was not a pointer
// Returns ErrNotStructTarget if v was not a pointer to a struct
func (c *Context) UnmarshalParameters(v interface{}) error {

	if err := canUnmarshal(v); err != nil {
		return err
	}

	request := c.Request()
	if request == nil {
		// Should never happen
		return ErrCouldNotGetContextRequest
	}

	if err := parseFormData(request); err != nil {
		return err
	}

	rt := reflect.ValueOf(v).Elem()

	// Okay, so we should step through the values in the form now
	for key, value := range request.Form {
		fieldValue, fieldKind := getFieldAndKind(rt, key)

		// if we can't set the result, then ignore it
		if !fieldValue.CanSet() {
			continue
		}

		switch fieldKind {
		case reflect.String:
			fieldValue.SetString(value[0])

		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
			if v, err := strconv.ParseUint(value[0], 10, 64); err != nil {
				return err
			} else {
				fieldValue.SetUint(v)
			}
		case reflect.Int:
			if v, err := strconv.ParseInt(value[0], 10, 64); err != nil {
				return err
			} else {
				fieldValue.SetInt(v)
			}
		case reflect.Bool:
			if v, err := strconv.ParseBool(value[0]); err != nil {
				return err
			} else {
				fieldValue.SetBool(v)
			}
		default:
			return ErrUnsupportedType
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package twitter

import (
	"gnd.la/app"
)

// Handler represents a function type which receives the
// result of authenticating a Twitter user.
type Handler func(*app.Context, *User, *Token)

// AuthHandler takes a Handler a returns a app.Handler which
// can be added to a app. When users are directed to this
// handler, they're first asked to authenticate with Twitter.
// If the user accepts, Handler is called with a non-nil user
// and a non-nil token. Otherwise, Handler is called with
// both parameters set to nil.
func AuthHandler(twApp *App, handler Handler) app.Handler {
	return func(ctx *app.Context) {
		token := ctx.FormValue("oauth_token")
		verifier := ctx.FormValue("oauth_verifier")
		if token != "" && verifier != "" {
			at, err := twApp.Exchange(token, verifier)
			if err != nil {
				panic(err)
			}
			user, err := twApp.Verify(at)
			if err != nil {
				panic(err)
			}
			handler(ctx, user, at)
		} else if denied := ctx.FormValue("denied"); denied != "" {
			purgeToken(denied)
			handler(ctx, nil, nil)
		} else {
			callback := ctx.URL().String()
			auth, err := twApp.Authenticate(callback)
			if err != nil {
				panic(err)
			}
			ctx.Redirect(auth, false)
		}
	}
}
<commit_msg>Clone the App with the current Context in the handler<commit_after>package twitter

import (
	"gnd.la/app"
)

// Handler represents a function type which receives the
// result of authenticating a Twitter user.
type Handler func(*app.Context, *User, *Token)

// AuthHandler takes a Handler a returns a app.Handler which
// can be added to a app. When users are directed to this
// handler, they're first asked to authenticate with Twitter.
// If the user accepts, Handler is called with a non-nil user
// and a non-nil token. Otherwise, Handler is called with
// both parameters set to nil.
func AuthHandler(twApp *App, handler Handler) app.Handler {
	return func(ctx *app.Context) {
		token := ctx.FormValue("oauth_token")
		verifier := ctx.FormValue("oauth_verifier")
		cloned := twApp.Clone(ctx)
		if token != "" && verifier != "" {
			at, err := cloned.Exchange(token, verifier)
			if err != nil {
				panic(err)
			}
			user, err := cloned.Verify(at)
			if err != nil {
				panic(err)
			}
			handler(ctx, user, at)
		} else if denied := ctx.FormValue("denied"); denied != "" {
			purgeToken(denied)
			handler(ctx, nil, nil)
		} else {
			callback := ctx.URL().String()
			auth, err := cloned.Authenticate(callback)
			if err != nil {
				panic(err)
			}
			ctx.Redirect(auth, false)
		}
	}
}
<|endoftext|>


<commit_before>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2018 Red Hat, Inc.
 *
 */

package network

import (
	"io/ioutil"
	"net"
	"strings"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/vishvananda/netlink"

	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/api"
)

var _ = Describe("Common Methods", func() {
	pid := "self"
	Context("Functions Read and Write from cache", func() {
		It("should persist interface payload", func() {
			tmpDir, _ := ioutil.TempDir("", "commontest")
			setInterfaceCacheFile(tmpDir + "/cache-%s.json")

			ifaceName := "iface_name"
			iface := api.Interface{Type: "fake_type", Source: api.InterfaceSource{Bridge: "fake_br"}}
			err := writeToCachedFile(&iface, interfaceCacheFile, pid, ifaceName)
			Expect(err).ToNot(HaveOccurred())

			var cached_iface api.Interface
			isExist, err := readFromCachedFile(pid, ifaceName, interfaceCacheFile, &cached_iface)
			Expect(err).ToNot(HaveOccurred())
			Expect(isExist).To(BeTrue())

			Expect(iface).To(Equal(cached_iface))
		})
		It("should persist qemu arg payload", func() {
			tmpDir, _ := ioutil.TempDir("", "commontest")
			setInterfaceCacheFile(tmpDir + "/cache-%s.json")

			qemuArgName := "iface_name"
			qemuArg := api.Arg{Value: "test_value"}
			err := writeToCachedFile(&qemuArg, interfaceCacheFile, pid, qemuArgName)
			Expect(err).ToNot(HaveOccurred())

			var cached_qemuArg api.Arg
			isExist, err := readFromCachedFile(pid, qemuArgName, interfaceCacheFile, &cached_qemuArg)
			Expect(err).ToNot(HaveOccurred())
			Expect(isExist).To(BeTrue())

			Expect(qemuArg).To(Equal(cached_qemuArg))
		})
	})
	Context("GetAvailableAddrsFromCIDR function", func() {
		It("Should return 2 addresses", func() {
			networkHandler := NetworkUtilsHandler{}
			gw, vm, err := networkHandler.GetHostAndGwAddressesFromCIDR("10.0.0.0/30")
			Expect(err).ToNot(HaveOccurred())
			Expect(gw).To(Equal("10.0.0.1/30"))
			Expect(vm).To(Equal("10.0.0.2/30"))
		})
		It("Should return 2 IPV6 addresses", func() {
			networkHandler := NetworkUtilsHandler{}
			gw, vm, err := networkHandler.GetHostAndGwAddressesFromCIDR("fd10:0:2::/120")
			Expect(err).ToNot(HaveOccurred())
			Expect(gw).To(Equal("fd10:0:2::1/120"))
			Expect(vm).To(Equal("fd10:0:2::2/120"))
		})
		It("Should fail when the subnet is too small", func() {
			networkHandler := NetworkUtilsHandler{}
			_, _, err := networkHandler.GetHostAndGwAddressesFromCIDR("10.0.0.0/31")
			Expect(err).To(HaveOccurred())
		})
		It("Should fail when the IPV6 subnet is too small", func() {
			networkHandler := NetworkUtilsHandler{}
			_, _, err := networkHandler.GetHostAndGwAddressesFromCIDR("fd10:0:2::/127")
			Expect(err).To(HaveOccurred())
		})
	})
	Context("GenerateRandomMac function", func() {
		It("should return a valid mac address", func() {
			networkHandler := NetworkUtilsHandler{}
			mac, err := networkHandler.GenerateRandomMac()
			Expect(err).ToNot(HaveOccurred())
			Expect(strings.HasPrefix(mac.String(), "02:00:00")).To(BeTrue())
		})
	})
})

var _ = Describe("VIF", func() {
	Context("String", func() {
		It("returns correct string representation", func() {
			addr, _ := netlink.ParseAddr("10.0.0.200/24")
			mac, _ := net.ParseMAC("de:ad:00:00:be:ef")
			gw := net.ParseIP("10.0.0.1")
			vif := &VIF{
				Name:      "test-vif",
				IP:        *addr,
				MAC:       mac,
				Gateway:   gw,
				Mtu:       1450,
				TapDevice: "myTap0",
			}
			Expect(vif.String()).To(Equal("VIF: { Name: test-vif, IP: 10.0.0.200, Mask: ffffff00, MAC: de:ad:00:00:be:ef, Gateway: 10.0.0.1, MTU: 1450, IPAMDisabled: false, TapDevice: myTap0}"))
		})
	})
})
<commit_msg>cleanup tempfiles for common_test<commit_after>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2018 Red Hat, Inc.
 *
 */

package network

import (
	"io/ioutil"
	"net"
	"os"
	"strings"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/vishvananda/netlink"

	"kubevirt.io/kubevirt/pkg/virt-launcher/virtwrap/api"
)

var _ = Describe("Common Methods", func() {
	pid := "self"
	Context("Functions Read and Write from cache", func() {
		It("should persist interface payload", func() {
			tmpDir, _ := ioutil.TempDir("", "commontest")
			defer os.RemoveAll(tmpDir)
			setInterfaceCacheFile(tmpDir + "/cache-%s.json")

			ifaceName := "iface_name"
			iface := api.Interface{Type: "fake_type", Source: api.InterfaceSource{Bridge: "fake_br"}}
			err := writeToCachedFile(&iface, interfaceCacheFile, pid, ifaceName)
			Expect(err).ToNot(HaveOccurred())

			var cached_iface api.Interface
			isExist, err := readFromCachedFile(pid, ifaceName, interfaceCacheFile, &cached_iface)
			Expect(err).ToNot(HaveOccurred())
			Expect(isExist).To(BeTrue())

			Expect(iface).To(Equal(cached_iface))
		})
		It("should persist qemu arg payload", func() {
			tmpDir, _ := ioutil.TempDir("", "commontest")
			defer os.RemoveAll(tmpDir)
			setInterfaceCacheFile(tmpDir + "/cache-%s.json")

			qemuArgName := "iface_name"
			qemuArg := api.Arg{Value: "test_value"}
			err := writeToCachedFile(&qemuArg, interfaceCacheFile, pid, qemuArgName)
			Expect(err).ToNot(HaveOccurred())

			var cached_qemuArg api.Arg
			isExist, err := readFromCachedFile(pid, qemuArgName, interfaceCacheFile, &cached_qemuArg)
			Expect(err).ToNot(HaveOccurred())
			Expect(isExist).To(BeTrue())

			Expect(qemuArg).To(Equal(cached_qemuArg))
		})
	})
	Context("GetAvailableAddrsFromCIDR function", func() {
		It("Should return 2 addresses", func() {
			networkHandler := NetworkUtilsHandler{}
			gw, vm, err := networkHandler.GetHostAndGwAddressesFromCIDR("10.0.0.0/30")
			Expect(err).ToNot(HaveOccurred())
			Expect(gw).To(Equal("10.0.0.1/30"))
			Expect(vm).To(Equal("10.0.0.2/30"))
		})
		It("Should return 2 IPV6 addresses", func() {
			networkHandler := NetworkUtilsHandler{}
			gw, vm, err := networkHandler.GetHostAndGwAddressesFromCIDR("fd10:0:2::/120")
			Expect(err).ToNot(HaveOccurred())
			Expect(gw).To(Equal("fd10:0:2::1/120"))
			Expect(vm).To(Equal("fd10:0:2::2/120"))
		})
		It("Should fail when the subnet is too small", func() {
			networkHandler := NetworkUtilsHandler{}
			_, _, err := networkHandler.GetHostAndGwAddressesFromCIDR("10.0.0.0/31")
			Expect(err).To(HaveOccurred())
		})
		It("Should fail when the IPV6 subnet is too small", func() {
			networkHandler := NetworkUtilsHandler{}
			_, _, err := networkHandler.GetHostAndGwAddressesFromCIDR("fd10:0:2::/127")
			Expect(err).To(HaveOccurred())
		})
	})
	Context("GenerateRandomMac function", func() {
		It("should return a valid mac address", func() {
			networkHandler := NetworkUtilsHandler{}
			mac, err := networkHandler.GenerateRandomMac()
			Expect(err).ToNot(HaveOccurred())
			Expect(strings.HasPrefix(mac.String(), "02:00:00")).To(BeTrue())
		})
	})
})

var _ = Describe("VIF", func() {
	Context("String", func() {
		It("returns correct string representation", func() {
			addr, _ := netlink.ParseAddr("10.0.0.200/24")
			mac, _ := net.ParseMAC("de:ad:00:00:be:ef")
			gw := net.ParseIP("10.0.0.1")
			vif := &VIF{
				Name:      "test-vif",
				IP:        *addr,
				MAC:       mac,
				Gateway:   gw,
				Mtu:       1450,
				TapDevice: "myTap0",
			}
			Expect(vif.String()).To(Equal("VIF: { Name: test-vif, IP: 10.0.0.200, Mask: ffffff00, MAC: de:ad:00:00:be:ef, Gateway: 10.0.0.1, MTU: 1450, IPAMDisabled: false, TapDevice: myTap0}"))
		})
	})
})
<|endoftext|>


<commit_before>package fbapi

import (
	"net/url"
	"strconv"
	"strings"
)

// Param augment given url.Values.
type Param interface {
	Set(v url.Values) error
}

// Build url.Values from the given Params.
func ParamValues(params ...Param) (v url.Values, err error) {
	v = make(url.Values)
	for _, p := range params {
		err = p.Set(v)
		if err != nil {
			return nil, err
		}
	}
	return v, nil
}

type paramLimit uint64

func (p paramLimit) Set(v url.Values) error {
	v.Add("limit", strconv.FormatUint(uint64(p), 10))
	return nil
}

// Specify a limit. Note, 0 values are also sent.
func ParamLimit(limit uint64) Param {
	return paramLimit(limit)
}

type paramOffset uint64

func (p paramOffset) Set(v url.Values) error {
	if p != 0 {
		v.Add("offset", strconv.FormatUint(uint64(p), 10))
	}
	return nil
}

// Specify number of items to offset. Note, 0 values are not sent.
func ParamOffset(offset uint64) Param {
	return paramOffset(offset)
}

type paramFields []string

func (p paramFields) Set(values url.Values) error {
	if len(p) > 0 {
		values.Set("fields", strings.Join(p, ","))
	}
	return nil
}

// Specify the fields to include.
func ParamFields(fields []string) Param {
	return paramFields(fields)
}

type paramAccessToken string

func (p paramAccessToken) Set(values url.Values) error {
	if p != "" {
		values.Set("access_token", string(p))
	}
	return nil
}

// An access_token parameter.
func ParamAccessToken(token string) Param {
	return paramAccessToken(token)
}

type paramDateFormat string

func (p paramDateFormat) Set(values url.Values) error {
	if p != "" {
		values.Add("date_format", string(p))
	}
	return nil
}

// The date format parameter.
func ParamDateFormat(format string) Param {
	return paramDateFormat(format)
}

// Sets the RFC 3339 format that Go expects when unmarshalling time.Time JSON
// values.
var DateFormat = ParamDateFormat(`Y-m-d\TH:i:s\Z`)
<commit_msg>ParamFields should be varadic<commit_after>package fbapi

import (
	"net/url"
	"strconv"
	"strings"
)

// Param augment given url.Values.
type Param interface {
	Set(v url.Values) error
}

// Build url.Values from the given Params.
func ParamValues(params ...Param) (v url.Values, err error) {
	v = make(url.Values)
	for _, p := range params {
		err = p.Set(v)
		if err != nil {
			return nil, err
		}
	}
	return v, nil
}

type paramLimit uint64

func (p paramLimit) Set(v url.Values) error {
	v.Add("limit", strconv.FormatUint(uint64(p), 10))
	return nil
}

// Specify a limit. Note, 0 values are also sent.
func ParamLimit(limit uint64) Param {
	return paramLimit(limit)
}

type paramOffset uint64

func (p paramOffset) Set(v url.Values) error {
	if p != 0 {
		v.Add("offset", strconv.FormatUint(uint64(p), 10))
	}
	return nil
}

// Specify number of items to offset. Note, 0 values are not sent.
func ParamOffset(offset uint64) Param {
	return paramOffset(offset)
}

type paramFields []string

func (p paramFields) Set(values url.Values) error {
	if len(p) > 0 {
		values.Set("fields", strings.Join(p, ","))
	}
	return nil
}

// Specify the fields to include.
func ParamFields(fields ...string) Param {
	return paramFields(fields)
}

type paramAccessToken string

func (p paramAccessToken) Set(values url.Values) error {
	if p != "" {
		values.Set("access_token", string(p))
	}
	return nil
}

// An access_token parameter.
func ParamAccessToken(token string) Param {
	return paramAccessToken(token)
}

type paramDateFormat string

func (p paramDateFormat) Set(values url.Values) error {
	if p != "" {
		values.Add("date_format", string(p))
	}
	return nil
}

// The date format parameter.
func ParamDateFormat(format string) Param {
	return paramDateFormat(format)
}

// Sets the RFC 3339 format that Go expects when unmarshalling time.Time JSON
// values.
var DateFormat = ParamDateFormat(`Y-m-d\TH:i:s\Z`)
<|endoftext|>


<commit_before>package buffer

import (
	"fmt"
)

type ProfilesService struct {
	client *Client
}

type Profiles struct {
	Items []Profile
}

type Profile struct {
	Avatar            string                 `json:"avatar,omitempty"`
	CreatedAt         int                    `json:"created_at,omitempty"`
	Default           bool                   `json:"default,omitempty"`
	FormattedUsername string                 `json:"formatted_username,omitempty"`
	ID                string                 `json:"id,omitempty"`
	Schedules         []ProfileSchedule      `json:"schedules,omitempty"`
	Service           string                 `json:"service,omitempty"`
	ServiceID         string                 `json:"service_id,omitempty"`
	ServiceUsername   string                 `json:"service_username,omitempty"`
	Statistics        map[string]interface{} `json:"statistics,omitempty"`
	TeamMembers       []string               `json:"team_members,omitempty"`
	Timezone          string                 `json:"timezone,omitempty"`
	UserID            string                 `json:"user_id,omitempty"`
}

type ProfileSchedule struct {
	Days  []string `json:"days,omitempty"`
	Times []string `json:"times,omitempty"`
}

func (s *ProfilesService) Get(profileID string) (*Profile, error) {
	u := fmt.Sprintf("/1/profiles/%v.json", profileID)

	req, err := s.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, err
	}

	profile := new(Profile)
	_, err = s.client.Do(req, profile)

	return profile, err
}

func (s *ProfilesService) GetAll() (*Profiles, error) {
	u := "/1/profiles.json"

	req, err := s.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, err
	}

	profiles := new(Profiles)
	_, err = s.client.Do(req, profiles)

	return profiles, err
}

func (s *ProfilesService) GetSchedules(profileID string) (*ProfileSchedule, error) {
	u := fmt.Sprintf("/1/profiles/%v.json", profileID)

	req, err := s.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, err
	}

	schedule := new(ProfileSchedule)
	_, err = s.client.Do(req, schedule)

	return schedule, err
}

func (s *ProfilesService) GetPendingUpdates(profileID string, params *PagingParams) (*Updates, error) {
	u := fmt.Sprintf("/1/profiles/%v/updates/pending.json", profileID)

	req, err := s.client.NewRequest("GET", u, params)
	if err != nil {
		return nil, err
	}

	updates := new(Updates)
	_, err = s.client.Do(req, updates)

	return updates, err
}

func (s *ProfilesService) GetSentUpdates(profileID string, params *PagingParams) (*Updates, error) {
	u := fmt.Sprintf("/1/profiles/%v/updates/sent.json", profileID)

	req, err := s.client.NewRequest("GET", u, params)
	if err != nil {
		return nil, err
	}

	updates := new(Updates)
	_, err = s.client.Do(req, updates)

	return updates, err
}
<commit_msg>Add /profiles/:id/updates/shuffle endpoint<commit_after>package buffer

import (
	"fmt"
)

type ProfilesService struct {
	client *Client
}

type Profiles struct {
	Items []Profile
}

type Profile struct {
	Avatar            string                 `json:"avatar,omitempty"`
	CreatedAt         int                    `json:"created_at,omitempty"`
	Default           bool                   `json:"default,omitempty"`
	FormattedUsername string                 `json:"formatted_username,omitempty"`
	ID                string                 `json:"id,omitempty"`
	Schedules         []ProfileSchedule      `json:"schedules,omitempty"`
	Service           string                 `json:"service,omitempty"`
	ServiceID         string                 `json:"service_id,omitempty"`
	ServiceUsername   string                 `json:"service_username,omitempty"`
	Statistics        map[string]interface{} `json:"statistics,omitempty"`
	TeamMembers       []string               `json:"team_members,omitempty"`
	Timezone          string                 `json:"timezone,omitempty"`
	UserID            string                 `json:"user_id,omitempty"`
}

type ProfileSchedule struct {
	Days  []string `json:"days,omitempty"`
	Times []string `json:"times,omitempty"`
}

func (s *ProfilesService) Get(profileID string) (*Profile, error) {
	u := fmt.Sprintf("/1/profiles/%v.json", profileID)

	req, err := s.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, err
	}

	profile := new(Profile)
	_, err = s.client.Do(req, profile)

	return profile, err
}

func (s *ProfilesService) GetAll() (*Profiles, error) {
	u := "/1/profiles.json"

	req, err := s.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, err
	}

	profiles := new(Profiles)
	_, err = s.client.Do(req, profiles)

	return profiles, err
}

func (s *ProfilesService) GetSchedules(profileID string) (*ProfileSchedule, error) {
	u := fmt.Sprintf("/1/profiles/%v.json", profileID)

	req, err := s.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, err
	}

	schedule := new(ProfileSchedule)
	_, err = s.client.Do(req, schedule)

	return schedule, err
}

func (s *ProfilesService) GetPendingUpdates(profileID string, params *PagingParams) (*Updates, error) {
	u := fmt.Sprintf("/1/profiles/%v/updates/pending.json", profileID)

	req, err := s.client.NewRequest("GET", u, params)
	if err != nil {
		return nil, err
	}

	updates := new(Updates)
	_, err = s.client.Do(req, updates)

	return updates, err
}

func (s *ProfilesService) GetSentUpdates(profileID string, params *PagingParams) (*Updates, error) {
	u := fmt.Sprintf("/1/profiles/%v/updates/sent.json", profileID)

	req, err := s.client.NewRequest("GET", u, params)
	if err != nil {
		return nil, err
	}

	updates := new(Updates)
	_, err = s.client.Do(req, updates)

	return updates, err
}

func (s *ProfilesService) ShuffleUpdates(profileID string, params *PagingParams) (*Updates, error) {
	u := fmt.Sprintf("/1/profiles/%v/updates/shuffle.json", profileID)

	req, err := s.client.NewRequest("POST", u, params)
	if err != nil {
		return nil, err
	}

	updates := new(Updates)
	_, err = s.client.Do(req, updates)

	return updates, err
}
<|endoftext|>


<commit_before>package ics

import (
	"bufio"
	"bytes"
	"errors"
	"io"
	"strings"
)

const (
	ianaTokenChars  = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-"
	invSafeChars    = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\",:;\x7f"
	invQSafeChars   = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\"\x7f"
	invValueChars   = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x7f"
	paramDelim      = ";"
	paramValueDelim = "="
	nameValueDelim  = ":"
)

type token struct {
	typ  tokenType
	data string
}

type tokenType uint8

const (
	TokenError tokenType = iota
	TokenName
	TokenParamName
	TokenParamValue
	TokenParamQValue
	TokenValue
)

type stateFn func() (token, stateFn)

type parser struct {
	br                 *bufio.Reader
	buf                bytes.Buffer
	state              stateFn
	pos, lineNo, colNo int
	lastWidth          int
	wasNewLine         bool
	err                error
}

func newParser(r io.Reader) *parser {
	p := &parser{
		br: bufio.NewReader(r),
	}
	p.state = p.parseName
	return p
}

func (p *parser) GetToken() (token, error) {
	var t token
	p.buf.Reset()
	t, p.state = p.state()
	if p.err == io.EOF {
		p.state = p.errorFn
		if t.typ == TokenError {
			p.err = io.ErrUnexpectedEOF
		} else {
			return t, nil
		}
	}
	return t, p.err
}

func (p *parser) next() rune {
	if p.err != nil {
		return -1
	}
	r, s, err := p.br.ReadRune()
	if err != nil {
		p.lastWidth = 0
		p.err = err
		return -1
	}
	p.buf.WriteRune(r)
	p.pos += s
	p.lastWidth = s
	if s > 0 {
		if r == '\n' {
			p.lineNo++
			p.wasNewLine = true
			p.colNo = 0
		} else {
			p.colNo++
			p.wasNewLine = false
		}
	}
	return r
}

func (p *parser) backup() {
	if p.lastWidth > 0 {
		p.pos -= p.lastWidth
		if p.wasNewLine {
			p.lineNo--
		}
		p.colNo--
		p.br.UnreadRune()
		p.buf.Truncate(p.buf.Len() - p.lastWidth)
		p.lastWidth = 0
	}
}

func (p *parser) accept(valid string) bool {
	if strings.ContainsRune(valid, p.next()) {
		return true
	}
	p.backup()
	return false
}

func (p *parser) acceptRun(valid string) {
	for {
		r := p.next()
		if r == -1 {
			return
		}
		if !strings.ContainsRune(valid, r) {
			p.backup()
			return
		}
	}
}

func (p *parser) except(invalid string) bool {
	r := p.next()
	if r == -1 {
		return false
	}
	if !strings.ContainsRune(invalid, r) {
		return true
	}
	p.backup()
	return false
}

func (p *parser) exceptRun(invalid string) {
	for {
		r := p.next()
		if r == -1 {
			return
		}
		if strings.ContainsRune(invalid, r) {
			p.backup()
			return
		}
	}
}

func (p *parser) parseName() (token, stateFn) {
	p.acceptRun(ianaTokenChars)
	t := token{
		TokenName,
		p.buf.String(),
	}
	if p.buf.Len() == 0 {
		p.err = ErrNoName
	} else if p.accept(paramDelim) {
		return t, p.parseParamName
	} else if p.accept(nameValueDelim) {
		return t, p.parseValue
	} else if p.err == nil {
		p.err = ErrInvalidChar
	}
	return p.errorFn()
}

func (p *parser) parseParamName() (token, stateFn) {
	p.acceptRun(ianaTokenChars)
	t := token{
		TokenName,
		p.buf.String(),
	}
	if p.buf.Len() == 0 {
		p.err = ErrNoParamName
	} else if p.accept(paramValueDelim) {
		return t, p.parseParamValue
	} else if p.err == nil {
		p.err = ErrInvalidChar
	}
	return p.errorFn()
}

func (p *parser) parseParamValue() (token, stateFn) {
	var t token
	if p.accept("\"") {
		p.exceptRun(invQSafeChars)
		if !p.accept("\"") {
			p.err = ErrInvalidChar
			return p.errorFn()
		}
		t.typ = TokenParamQValue
		t.data = p.buf.String()
	} else {
		p.exceptRun(invSafeChars)
		t.typ = TokenParamQValue
		t.data = p.buf.String()
	}
	if p.accept(paramDelim) {
		return t, p.parseParamName
	} else if p.accept(nameValueDelim) {
		return t, p.parseValue
	} else if p.err == nil {
		p.err = ErrInvalidChar
	}
	return p.errorFn()
}

func (p *parser) parseValue() (token, stateFn) {
	var toRet []byte
	for {
		p.exceptRun(invValueChars)
		if !p.accept("\r") && !p.accept("\n") {
			if p.err == nil {
				p.err = ErrInvalidChar
			}
			return p.errorFn()
		}
		toAdd := p.buf.Bytes()
		toRet = append(toRet, toAdd[:len(toAdd)-2]...)
		if !p.accept(" ") {
			break
		}
		p.buf.Reset()
	}
	return token{
		TokenValue,
		string(toRet),
	}, p.parseName
}

func (p *parser) errorFn() (token, stateFn) {
	return token{
		TokenError,
		p.err.Error(),
	}, p.errorFn
}

// Errors

var (
	ErrInvalidChar = errors.New("invalid character")
	ErrNoName      = errors.New("zero length name")
	ErrNoParamName = errors.New("zero length param name")
)
<commit_msg>Corrected logic<commit_after>package ics

import (
	"bufio"
	"bytes"
	"errors"
	"io"
	"strings"
)

const (
	ianaTokenChars  = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-"
	invSafeChars    = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\",:;\x7f"
	invQSafeChars   = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\"\x7f"
	invValueChars   = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x7f"
	paramDelim      = ";"
	paramValueDelim = "="
	nameValueDelim  = ":"
)

type token struct {
	typ  tokenType
	data string
}

type tokenType uint8

const (
	TokenError tokenType = iota
	TokenName
	TokenParamName
	TokenParamValue
	TokenParamQValue
	TokenValue
)

type stateFn func() (token, stateFn)

type parser struct {
	br                 *bufio.Reader
	buf                bytes.Buffer
	state              stateFn
	pos, lineNo, colNo int
	lastWidth          int
	wasNewLine         bool
	err                error
}

func newParser(r io.Reader) *parser {
	p := &parser{
		br: bufio.NewReader(r),
	}
	p.state = p.parseName
	return p
}

func (p *parser) GetToken() (token, error) {
	var t token
	p.buf.Reset()
	t, p.state = p.state()
	if p.err == io.EOF {
		p.state = p.errorFn
		if t.typ == TokenError {
			p.err = io.ErrUnexpectedEOF
		} else {
			return t, nil
		}
	}
	return t, p.err
}

func (p *parser) next() rune {
	if p.err != nil {
		return -1
	}
	r, s, err := p.br.ReadRune()
	if err != nil {
		p.lastWidth = 0
		p.err = err
		return -1
	}
	p.buf.WriteRune(r)
	p.pos += s
	p.lastWidth = s
	if s > 0 {
		if r == '\n' {
			p.lineNo++
			p.wasNewLine = true
			p.colNo = 0
		} else {
			p.colNo++
			p.wasNewLine = false
		}
	}
	return r
}

func (p *parser) backup() {
	if p.lastWidth > 0 {
		p.pos -= p.lastWidth
		if p.wasNewLine {
			p.lineNo--
		}
		p.colNo--
		p.br.UnreadRune()
		p.buf.Truncate(p.buf.Len() - p.lastWidth)
		p.lastWidth = 0
	}
}

func (p *parser) accept(valid string) bool {
	if strings.ContainsRune(valid, p.next()) {
		return true
	}
	p.backup()
	return false
}

func (p *parser) acceptRun(valid string) {
	for {
		r := p.next()
		if r == -1 {
			return
		}
		if !strings.ContainsRune(valid, r) {
			p.backup()
			return
		}
	}
}

func (p *parser) except(invalid string) bool {
	r := p.next()
	if r == -1 {
		return false
	}
	if !strings.ContainsRune(invalid, r) {
		return true
	}
	p.backup()
	return false
}

func (p *parser) exceptRun(invalid string) {
	for {
		r := p.next()
		if r == -1 {
			return
		}
		if strings.ContainsRune(invalid, r) {
			p.backup()
			return
		}
	}
}

func (p *parser) parseName() (token, stateFn) {
	p.acceptRun(ianaTokenChars)
	t := token{
		TokenName,
		p.buf.String(),
	}
	if p.buf.Len() == 0 {
		p.err = ErrNoName
	} else if p.accept(paramDelim) {
		return t, p.parseParamName
	} else if p.accept(nameValueDelim) {
		return t, p.parseValue
	} else if p.err == nil {
		p.err = ErrInvalidChar
	}
	return p.errorFn()
}

func (p *parser) parseParamName() (token, stateFn) {
	p.acceptRun(ianaTokenChars)
	t := token{
		TokenName,
		p.buf.String(),
	}
	if p.buf.Len() == 0 {
		p.err = ErrNoParamName
	} else if p.accept(paramValueDelim) {
		return t, p.parseParamValue
	} else if p.err == nil {
		p.err = ErrInvalidChar
	}
	return p.errorFn()
}

func (p *parser) parseParamValue() (token, stateFn) {
	var t token
	if p.accept("\"") {
		p.exceptRun(invQSafeChars)
		if !p.accept("\"") {
			p.err = ErrInvalidChar
			return p.errorFn()
		}
		t.typ = TokenParamQValue
		t.data = p.buf.String()
	} else {
		p.exceptRun(invSafeChars)
		t.typ = TokenParamQValue
		t.data = p.buf.String()
	}
	if p.accept(paramDelim) {
		return t, p.parseParamName
	} else if p.accept(nameValueDelim) {
		return t, p.parseValue
	} else if p.err == nil {
		p.err = ErrInvalidChar
	}
	return p.errorFn()
}

func (p *parser) parseValue() (token, stateFn) {
	var toRet []byte
	for {
		p.exceptRun(invValueChars)
		if !p.accept("\r") || !p.accept("\n") {
			if p.err == nil {
				p.err = ErrInvalidChar
			}
			return p.errorFn()
		}
		toAdd := p.buf.Bytes()
		toRet = append(toRet, toAdd[:len(toAdd)-2]...)
		if !p.accept(" ") {
			break
		}
		p.buf.Reset()
	}
	return token{
		TokenValue,
		string(toRet),
	}, p.parseName
}

func (p *parser) errorFn() (token, stateFn) {
	return token{
		TokenError,
		p.err.Error(),
	}, p.errorFn
}

// Errors

var (
	ErrInvalidChar = errors.New("invalid character")
	ErrNoName      = errors.New("zero length name")
	ErrNoParamName = errors.New("zero length param name")
)
<|endoftext|>


<commit_before>package hl7parser

import (
	"bytes"
	"errors"
)

var (
	// ErrTooShort is returned if a message isn't long enough to contain a valid
	// header
	ErrTooShort = errors.New("message must be at least eight bytes long")
	// ErrInvalidHeader is returned if a message doesn't start with "MSH"
	ErrInvalidHeader = errors.New("expected message to begin with MSH")
)

type (
	Message      []Segment
	Segment      []Field
	Field        []FieldItem
	FieldItem    []Component
	Component    []Subcomponent
	Subcomponent string
)

type Delimiters struct {
	Field, Component, Repeat, Escape, Subcomponent byte
}

func Parse(buf []byte) (Message, *Delimiters, error) {
	if len(buf) < 8 {
		return nil, nil, ErrTooShort
	}

	if !bytes.HasPrefix(buf, []byte("MSH")) {
		return nil, nil, ErrInvalidHeader
	}

	fs := buf[3]
	cs := buf[4]
	rs := buf[5]
	ec := buf[6]
	ss := buf[7]

	d := Delimiters{fs, cs, rs, ec, ss}

	var (
		message   Message
		segment   Segment
		field     Field
		fieldItem FieldItem
		component Component
		s         []byte
	)

	segment = Segment{
		Field{FieldItem{Component{Subcomponent("MSH")}}},
		Field{FieldItem{Component{Subcomponent(string(buf[3:8]))}}},
	}

	commitBuffer := func(force bool) {
		if s != nil || force {
			component = append(component, Subcomponent(unescape(s, &d)))
			s = nil
		}
	}

	commitComponent := func(force bool) {
		commitBuffer(false)

		if component != nil || force {
			fieldItem = append(fieldItem, component)
			component = nil
		}
	}

	commitFieldItem := func(force bool) {
		commitComponent(false)

		if fieldItem != nil || force {
			field = append(field, fieldItem)
			fieldItem = nil
		}
	}

	commitField := func(force bool) {
		commitFieldItem(false)

		if field != nil || force {
			segment = append(segment, field)
			field = nil
		}
	}

	commitSegment := func(force bool) {
		commitField(false)

		if segment != nil || force {
			message = append(message, segment)
			segment = nil
		}
	}

	for _, c := range buf[9:] {
		switch c {
		case '\r':
			commitSegment(true)
		case fs:
			commitField(true)
		case rs:
			commitFieldItem(true)
		case cs:
			commitComponent(true)
		case ss:
			commitBuffer(true)
		default:
			s = append(s, c)
		}
	}

	commitSegment(false)

	return message, &d, nil
}

func unescape(b []byte, d *Delimiters) []byte {
	r := make([]byte, len(b))

	j, e := 0, false
	for i := 0; i < len(b); i++ {
		c := b[i]

		switch e {
		case true:
			switch c {
			case 'F':
				r[j] = d.Field
				i++
			case 'S':
				r[j] = d.Component
				i++
			case 'T':
				r[j] = d.Subcomponent
				i++
			case 'R':
				r[j] = d.Repeat
				i++
			case 'E':
				r[j] = d.Escape
				i++
			default:
				r[j] = d.Escape
				j++
				r[j] = c
			}

			j++

			e = false
		case false:
			switch c {
			case d.Escape:
				e = true
			default:
				r[j] = c
				j++
			}
		}
	}

	return r[:j]
}
<commit_msg>support \r or \n for segment separator<commit_after>package hl7parser

import (
	"bytes"
	"errors"
)

var (
	// ErrTooShort is returned if a message isn't long enough to contain a valid
	// header
	ErrTooShort = errors.New("message must be at least eight bytes long")
	// ErrInvalidHeader is returned if a message doesn't start with "MSH"
	ErrInvalidHeader = errors.New("expected message to begin with MSH")
)

type (
	Message      []Segment
	Segment      []Field
	Field        []FieldItem
	FieldItem    []Component
	Component    []Subcomponent
	Subcomponent string
)

type Delimiters struct {
	Field, Component, Repeat, Escape, Subcomponent byte
}

func Parse(buf []byte) (Message, *Delimiters, error) {
	if len(buf) < 8 {
		return nil, nil, ErrTooShort
	}

	if !bytes.HasPrefix(buf, []byte("MSH")) {
		return nil, nil, ErrInvalidHeader
	}

	fs := buf[3]
	cs := buf[4]
	rs := buf[5]
	ec := buf[6]
	ss := buf[7]

	d := Delimiters{fs, cs, rs, ec, ss}

	var (
		message   Message
		segment   Segment
		field     Field
		fieldItem FieldItem
		component Component
		s         []byte
	)

	segment = Segment{
		Field{FieldItem{Component{Subcomponent("MSH")}}},
		Field{FieldItem{Component{Subcomponent(string(buf[3:8]))}}},
	}

	commitBuffer := func(force bool) {
		if s != nil || force {
			component = append(component, Subcomponent(unescape(s, &d)))
			s = nil
		}
	}

	commitComponent := func(force bool) {
		commitBuffer(false)

		if component != nil || force {
			fieldItem = append(fieldItem, component)
			component = nil
		}
	}

	commitFieldItem := func(force bool) {
		commitComponent(false)

		if fieldItem != nil || force {
			field = append(field, fieldItem)
			fieldItem = nil
		}
	}

	commitField := func(force bool) {
		commitFieldItem(false)

		if field != nil || force {
			segment = append(segment, field)
			field = nil
		}
	}

	commitSegment := func(force bool) {
		commitField(false)

		if segment != nil || force {
			message = append(message, segment)
			segment = nil
		}
	}

	sawNewline := false
	for _, c := range buf[9:] {
		switch c {
		case '\r', '\n':
			if !sawNewline {
				commitSegment(true)
			}
			sawNewline = true
		case fs:
			sawNewline = false
			commitField(true)
		case rs:
			sawNewline = false
			commitFieldItem(true)
		case cs:
			sawNewline = false
			commitComponent(true)
		case ss:
			sawNewline = false
			commitBuffer(true)
		default:
			sawNewline = false
			s = append(s, c)
		}
	}

	commitSegment(false)

	return message, &d, nil
}

func unescape(b []byte, d *Delimiters) []byte {
	r := make([]byte, len(b))

	j, e := 0, false
	for i := 0; i < len(b); i++ {
		c := b[i]

		switch e {
		case true:
			switch c {
			case 'F':
				r[j] = d.Field
				i++
			case 'S':
				r[j] = d.Component
				i++
			case 'T':
				r[j] = d.Subcomponent
				i++
			case 'R':
				r[j] = d.Repeat
				i++
			case 'E':
				r[j] = d.Escape
				i++
			default:
				r[j] = d.Escape
				j++
				r[j] = c
			}

			j++

			e = false
		case false:
			switch c {
			case d.Escape:
				e = true
			default:
				r[j] = c
				j++
			}
		}
	}

	return r[:j]
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"strconv"
	"strings"

	"github.com/garyburd/redigo/redis"
)

type MonitorCmd struct {
	Timestamp float64  `json:"timestamp"`
	Text      string   `json:"text"`
	Params    []string `json:"params"`
}

// Parses a line from the monitor redis command.
// Do we care about Host, Port? If so need to implement..
// TODO: This parse sucks but MVP and all....
func ParseMonitorLine(l string) (*MonitorCmd, error) {
	if l == "OK" {
		return nil, nil
	}
	m := &MonitorCmd{}
	si := strings.Index(l, "[")
	ei := strings.Index(l, "]")
	t, err := strconv.ParseFloat(l[0:si-1], 10)
	if err != nil {
		log.Printf("Could not convert timestamp from string to float: %s", t)
	}
	m.Timestamp = t
	cmdPart := strings.Split(l[ei+2:], " ")
	// Upper case for consistency the command and trim and extra "
	m.Text = strings.ToUpper(strings.Trim(cmdPart[0], "\""))
	parts := cmdPart[1:]
	m.Params = make([]string, len(parts))

	// Trim off " from params
	for i, p := range parts {
		m.Params[i] = strings.Trim(p, "\"")
	}
	return m, nil
}

type Slowlog struct {
	ID        int64
	Timestamp int64
	Duration  int64
	Command   []string
}

// Parse the slowlog
// XXX: Not working yet need to figure out how to conver slowlog to a struct
func ParesSlowlogLine(entries []interface{}, err error) ([]Slowlog, error) {
	logs := make([]Slowlog, 0)
	//Trace.Printf("Slowlog data is: %v\n", reply)
	if err != nil {
		log.Fatal(err)
	}
	for _, entry := range entries {
		e, ok := entry.([]interface{})
		if !ok {
			Error.Println("Bad Slowlog entry")
			continue
		}
		l := Slowlog{}
		_, err = redis.Scan(e, &l.ID, &l.Timestamp, &l.Duration, &l.Command)
		if err != nil {
			Error.Printf("Error trying to scan slowlog is %s", err)
			continue
		}
		Trace.Printf("Ok log is: %v\n", l)
		logs = append(logs, l)
	}
	return logs, nil
}
<commit_msg>Cleanup some debugging<commit_after>package main

import (
	"log"
	"strconv"
	"strings"

	"github.com/garyburd/redigo/redis"
)

type MonitorCmd struct {
	Timestamp float64  `json:"timestamp"`
	Text      string   `json:"text"`
	Params    []string `json:"params"`
}

// Parses a line from the monitor redis command.
// Do we care about Host, Port? If so need to implement..
// TODO: This parse sucks but MVP and all....
func ParseMonitorLine(l string) (*MonitorCmd, error) {
	if l == "OK" {
		return nil, nil
	}
	m := &MonitorCmd{}
	si := strings.Index(l, "[")
	ei := strings.Index(l, "]")
	t, err := strconv.ParseFloat(l[0:si-1], 10)
	if err != nil {
		log.Printf("Could not convert timestamp from string to float: %s", t)
	}
	m.Timestamp = t
	cmdPart := strings.Split(l[ei+2:], " ")
	// Upper case for consistency the command and trim and extra "
	m.Text = strings.ToUpper(strings.Trim(cmdPart[0], "\""))
	parts := cmdPart[1:]
	m.Params = make([]string, len(parts))

	// Trim off " from params
	for i, p := range parts {
		m.Params[i] = strings.Trim(p, "\"")
	}
	return m, nil
}

type Slowlog struct {
	ID        int64
	Timestamp int64
	Duration  int64
	Command   []string
}

// Parse the slowlog
func ParesSlowlogLine(entries []interface{}, err error) ([]Slowlog, error) {
	logs := make([]Slowlog, 0)
	if err != nil {
		log.Fatal(err)
	}
	for _, entry := range entries {
		e, ok := entry.([]interface{})
		if !ok {
			Error.Println("Bad Slowlog entry")
			continue
		}
		l := Slowlog{}
		_, err = redis.Scan(e, &l.ID, &l.Timestamp, &l.Duration, &l.Command)
		if err != nil {
			Error.Printf("Error trying to scan slowlog is %s", err)
			continue
		}
		logs = append(logs, l)
	}
	return logs, nil
}
<|endoftext|>


<commit_before>package rivescript

// parse loads the RiveScript code into the bot's memory.
func (rs *RiveScript) parse(path string, lines []string) error {
	rs.say("Parsing code...")

	// Get the abstract syntax tree of this file.
	AST, err := rs.parser.Parse(path, lines)
	if err != nil {
		return err
	}

	// Get all of the "begin" type variables
	for k, v := range AST.Begin.Global {
		if v == UNDEFTAG {
			delete(rs.global, k)
		} else {
			rs.global[k] = v
		}
	}
	for k, v := range AST.Begin.Var {
		if v == UNDEFTAG {
			delete(rs.vars, k)
		} else {
			rs.vars[k] = v
		}
	}
	for k, v := range AST.Begin.Sub {
		if v == UNDEFTAG {
			delete(rs.sub, k)
		} else {
			rs.sub[k] = v
		}
	}
	for k, v := range AST.Begin.Person {
		if v == UNDEFTAG {
			delete(rs.person, k)
		} else {
			rs.person[k] = v
		}
	}
	for k, v := range AST.Begin.Array {
		rs.array[k] = v
	}

	// Consume all the parsed triggers.
	for topic, data := range AST.Topics {
		// Keep a map of the topics that are included/inherited under this topic.
		if _, ok := rs.includes[topic]; !ok {
			rs.includes[topic] = map[string]bool{}
		}
		if _, ok := rs.inherits[topic]; !ok {
			rs.inherits[topic] = map[string]bool{}
		}

		// Merge in the topic inclusions/inherits.
		for included := range data.Includes {
			rs.includes[topic][included] = true
		}
		for inherited := range data.Inherits {
			rs.inherits[topic][inherited] = true
		}

		// Initialize the topic structure.
		if _, ok := rs.topics[topic]; !ok {
			rs.topics[topic] = new(astTopic)
			rs.topics[topic].triggers = []*astTrigger{}
		}

		// Consume the AST triggers into the brain.
		for _, trig := range data.Triggers {
			// Convert this AST trigger into an internal astmap trigger.
			trigger := new(astTrigger)
			trigger.trigger = trig.Trigger
			trigger.reply = trig.Reply
			trigger.condition = trig.Condition
			trigger.redirect = trig.Redirect
			trigger.previous = trig.Previous

			rs.topics[topic].triggers = append(rs.topics[topic].triggers, trigger)
		}
	}

	// Load all the parsed objects.
	for _, object := range AST.Objects {
		// Have a language handler for this?
		if _, ok := rs.handlers[object.Language]; ok {
			rs.say("Loading object macro %s (%s)", object.Name, object.Language)
			rs.handlers[object.Language].Load(object.Name, object.Code)
			rs.objlangs[object.Name] = object.Language
		}
	}

	return nil
}
<commit_msg>faster reloading in parser for alicebot reloading<commit_after>package rivescript

// parse loads the RiveScript code into the bot's memory.
func (rs *RiveScript) parse(path string, lines []string) error {
	rs.say("Parsing code...")

	// Get the abstract syntax tree of this file.
	AST, err := rs.parser.Parse(path, lines)
	if err != nil {
		return err
	}

	// Get all of the "begin" type variables
	for k, v := range AST.Begin.Global {
		if v == UNDEFTAG {
			delete(rs.global, k)
		} else {
			rs.global[k] = v
		}
	}
	for k, v := range AST.Begin.Var {
		if v == UNDEFTAG {
			delete(rs.vars, k)
		} else {
			rs.vars[k] = v
		}
	}
	for k, v := range AST.Begin.Sub {
		if v == UNDEFTAG {
			delete(rs.sub, k)
		} else {
			rs.sub[k] = v
		}
	}
	for k, v := range AST.Begin.Person {
		if v == UNDEFTAG {
			delete(rs.person, k)
		} else {
			rs.person[k] = v
		}
	}
	for k, v := range AST.Begin.Array {
		rs.array[k] = v
	}

	// Consume all the parsed triggers.
	for topic, data := range AST.Topics {
		// Keep a map of the topics that are included/inherited under this topic.
		if _, ok := rs.includes[topic]; !ok {
			rs.includes[topic] = map[string]bool{}
		}
		if _, ok := rs.inherits[topic]; !ok {
			rs.inherits[topic] = map[string]bool{}
		}

		// Merge in the topic inclusions/inherits.
		for included := range data.Includes {
			rs.includes[topic][included] = true
		}
		for inherited := range data.Inherits {
			rs.inherits[topic][inherited] = true
		}

		// Initialize the topic structure.
		if _, ok := rs.topics[topic]; !ok {
			rs.topics[topic] = new(astTopic)
			rs.topics[topic].triggers = []*astTrigger{}
		}

		// Consume the AST triggers into the brain.
		for _, trig := range data.Triggers {
			// Convert this AST trigger into an internal astmap trigger.
			foundtrigger := false
			for _, previous := range rs.topics[topic].triggers {
				if previous.trigger == trig.Trigger && previous.previous == trig.Previous {
					previous.redirect = trig.Redirect
					foundtrigger = true
					for _, cond := range trig.Condition {
						foundcond := false
						for _, oldcond := range previous.condition {
							if oldcond == cond {
								foundcond = true
								break
							}
						}
						if foundcond == false {
							previous.condition = append(previous.condition, cond)
						}
					}
					for _, reply := range trig.Reply {
						newreply := true
						for _, oldreply := range previous.reply {
							if oldreply == reply {
								newreply = false
								break
							}
						}
						if newreply == true {
							previous.reply = append(previous.reply, reply)
						}
					}
					rs.say("Found previous trigger: %s == %s", trig.Trigger, previous.trigger)
				}
			}
			if foundtrigger == false {
				trigger := new(astTrigger)
				trigger.trigger = trig.Trigger
				trigger.reply = trig.Reply
				trigger.condition = trig.Condition
				trigger.redirect = trig.Redirect
				trigger.previous = trig.Previous

				rs.topics[topic].triggers = append(rs.topics[topic].triggers, trigger)
			}
		}
	}

	// Load all the parsed objects.
	for _, object := range AST.Objects {
		// Have a language handler for this?
		if _, ok := rs.handlers[object.Language]; ok {
			rs.say("Loading object macro %s (%s)", object.Name, object.Language)
			rs.handlers[object.Language].Load(object.Name, object.Code)
			rs.objlangs[object.Name] = object.Language
		}
	}

	return nil
}
<|endoftext|>