<commit_before>// Package filestore provide a storage backend based on the local file system.
//
// FileStore is a storage backend used as a handler.DataStore in handler.NewHandler.
// It stores the uploads in a directory specified in two different files: The
// `[id].info` files are used to store the fileinfo in JSON format. The
// `[id]` files without an extension contain the raw binary data uploaded.
// No cleanup is performed so you may want to run a cronjob to ensure your disk
// is not filled up with old and finished uploads.
package filestore

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/tus/tusd/internal/uid"
	"github.com/tus/tusd/pkg/handler"
)

var defaultFilePerm = os.FileMode(0664)

// See the handler.DataStore interface for documentation about the different
// methods.
type FileStore struct {
	// Relative or absolute path to store files in. FileStore does not check
	// whether the path exists, use os.MkdirAll in this case on your own.
	Path string
}

// New creates a new file based storage backend. The directory specified will
// be used as the only storage entry. This method does not check
// whether the path exists, use os.MkdirAll to ensure.
// In addition, a locking mechanism is provided.
func New(path string) FileStore {
	return FileStore{path}
}

// UseIn sets this store as the core data store in the passed composer and adds
// all possible extension to it.
func (store FileStore) UseIn(composer *handler.StoreComposer) {
	composer.UseCore(store)
	composer.UseTerminater(store)
	composer.UseConcater(store)
	composer.UseLengthDeferrer(store)
}

func (store FileStore) NewUpload(ctx context.Context, info handler.FileInfo) (handler.Upload, error) {
	id := uid.Uid()
	binPath := store.binPath(id)
	info.ID = id
	info.Storage = map[string]string{
		"Type": "filestore",
		"Path": binPath,
	}

	// Create binary file with no content
	file, err := os.OpenFile(binPath, os.O_CREATE|os.O_WRONLY, defaultFilePerm)
	if err != nil {
		if os.IsNotExist(err) {
			err = fmt.Errorf("upload directory does not exist: %s", store.Path)
		}
		return nil, err
	}
	err = file.Close()
	if err != nil {
		return nil, err
	}

	upload := &fileUpload{
		info:     info,
		infoPath: store.infoPath(id),
		binPath:  store.binPath(id),
	}

	// writeInfo creates the file by itself if necessary
	err = upload.writeInfo()
	if err != nil {
		return nil, err
	}

	return upload, nil
}

func (store FileStore) GetUpload(ctx context.Context, id string) (handler.Upload, error) {
	info := handler.FileInfo{}
	data, err := ioutil.ReadFile(store.infoPath(id))
	if err != nil {
		if os.IsNotExist(err) {
			// Interpret os.ErrNotExist as 404 Not Found
			err = handler.ErrNotFound
		}
		return nil, err
	}
	if err := json.Unmarshal(data, &info); err != nil {
		return nil, err
	}

	binPath := store.binPath(id)
	infoPath := store.infoPath(id)
	stat, err := os.Stat(binPath)
	if err != nil {
		if os.IsNotExist(err) {
			// Interpret os.ErrNotExist as 404 Not Found
			err = handler.ErrNotFound
		}
		return nil, err
	}

	info.Offset = stat.Size()

	return &fileUpload{
		info:     info,
		binPath:  binPath,
		infoPath: infoPath,
	}, nil
}

func (store FileStore) AsTerminatableUpload(upload handler.Upload) handler.TerminatableUpload {
	return upload.(*fileUpload)
}

func (store FileStore) AsLengthDeclarableUpload(upload handler.Upload) handler.LengthDeclarableUpload {
	return upload.(*fileUpload)
}

func (store FileStore) AsConcatableUpload(upload handler.Upload) handler.ConcatableUpload {
	return upload.(*fileUpload)
}

// binPath returns the path to the file storing the binary data.
func (store FileStore) binPath(id string) string {
	return filepath.Join(store.Path, id)
}

// infoPath returns the path to the .info file storing the file's info.
func (store FileStore) infoPath(id string) string {
	return filepath.Join(store.Path, id+".info")
}

type fileUpload struct {
	// info stores the current information about the upload
	info handler.FileInfo
	// infoPath is the path to the .info file
	infoPath string
	// binPath is the path to the binary file (which has no extension)
	binPath string
}

func (upload *fileUpload) GetInfo(ctx context.Context) (handler.FileInfo, error) {
	return upload.info, nil
}

func (upload *fileUpload) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) {
	file, err := os.OpenFile(upload.binPath, os.O_WRONLY|os.O_APPEND, defaultFilePerm)
	if err != nil {
		return 0, err
	}
	defer file.Close()

	n, err := io.Copy(file, src)

	upload.info.Offset += n
	return n, err
}

func (upload *fileUpload) GetReader(ctx context.Context) (io.Reader, error) {
	return os.Open(upload.binPath)
}

func (upload *fileUpload) Terminate(ctx context.Context) error {
	if err := os.Remove(upload.infoPath); err != nil {
		return err
	}
	if err := os.Remove(upload.binPath); err != nil {
		return err
	}
	return nil
}

func (upload *fileUpload) ConcatUploads(ctx context.Context, uploads []handler.Upload) (err error) {
	file, err := os.OpenFile(upload.binPath, os.O_WRONLY|os.O_APPEND, defaultFilePerm)
	if err != nil {
		return err
	}
	defer file.Close()

	for _, partialUpload := range uploads {
		fileUpload := partialUpload.(*fileUpload)

		src, err := os.Open(fileUpload.binPath)
		if err != nil {
			return err
		}

		if _, err := io.Copy(file, src); err != nil {
			return err
		}
	}

	return
}

func (upload *fileUpload) DeclareLength(ctx context.Context, length int64) error {
	upload.info.Size = length
	upload.info.SizeIsDeferred = false
	return upload.writeInfo()
}

// writeInfo updates the entire information. Everything will be overwritten.
func (upload *fileUpload) writeInfo() error {
	data, err := json.Marshal(upload.info)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(upload.infoPath, data, defaultFilePerm)
}

func (upload *fileUpload) FinishUpload(ctx context.Context) error {
	return nil
}
<commit_msg>filestore: keep ID passed to NewUpload, if available (#820)<commit_after>// Package filestore provide a storage backend based on the local file system.
//
// FileStore is a storage backend used as a handler.DataStore in handler.NewHandler.
// It stores the uploads in a directory specified in two different files: The
// `[id].info` files are used to store the fileinfo in JSON format. The
// `[id]` files without an extension contain the raw binary data uploaded.
// No cleanup is performed so you may want to run a cronjob to ensure your disk
// is not filled up with old and finished uploads.
package filestore

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	"github.com/tus/tusd/internal/uid"
	"github.com/tus/tusd/pkg/handler"
)

var defaultFilePerm = os.FileMode(0664)

// See the handler.DataStore interface for documentation about the different
// methods.
type FileStore struct {
	// Relative or absolute path to store files in. FileStore does not check
	// whether the path exists, use os.MkdirAll in this case on your own.
	Path string
}

// New creates a new file based storage backend. The directory specified will
// be used as the only storage entry. This method does not check
// whether the path exists, use os.MkdirAll to ensure.
// In addition, a locking mechanism is provided.
func New(path string) FileStore {
	return FileStore{path}
}

// UseIn sets this store as the core data store in the passed composer and adds
// all possible extension to it.
func (store FileStore) UseIn(composer *handler.StoreComposer) {
	composer.UseCore(store)
	composer.UseTerminater(store)
	composer.UseConcater(store)
	composer.UseLengthDeferrer(store)
}

func (store FileStore) NewUpload(ctx context.Context, info handler.FileInfo) (handler.Upload, error) {
	if info.ID == "" {
		info.ID = uid.Uid()
	}
	binPath := store.binPath(info.ID)
	info.Storage = map[string]string{
		"Type": "filestore",
		"Path": binPath,
	}

	// Create binary file with no content
	file, err := os.OpenFile(binPath, os.O_CREATE|os.O_WRONLY, defaultFilePerm)
	if err != nil {
		if os.IsNotExist(err) {
			err = fmt.Errorf("upload directory does not exist: %s", store.Path)
		}
		return nil, err
	}
	err = file.Close()
	if err != nil {
		return nil, err
	}

	upload := &fileUpload{
		info:     info,
		infoPath: store.infoPath(info.ID),
		binPath:  binPath,
	}

	// writeInfo creates the file by itself if necessary
	err = upload.writeInfo()
	if err != nil {
		return nil, err
	}

	return upload, nil
}

func (store FileStore) GetUpload(ctx context.Context, id string) (handler.Upload, error) {
	info := handler.FileInfo{}
	data, err := ioutil.ReadFile(store.infoPath(id))
	if err != nil {
		if os.IsNotExist(err) {
			// Interpret os.ErrNotExist as 404 Not Found
			err = handler.ErrNotFound
		}
		return nil, err
	}
	if err := json.Unmarshal(data, &info); err != nil {
		return nil, err
	}

	binPath := store.binPath(id)
	infoPath := store.infoPath(id)
	stat, err := os.Stat(binPath)
	if err != nil {
		if os.IsNotExist(err) {
			// Interpret os.ErrNotExist as 404 Not Found
			err = handler.ErrNotFound
		}
		return nil, err
	}

	info.Offset = stat.Size()

	return &fileUpload{
		info:     info,
		binPath:  binPath,
		infoPath: infoPath,
	}, nil
}

func (store FileStore) AsTerminatableUpload(upload handler.Upload) handler.TerminatableUpload {
	return upload.(*fileUpload)
}

func (store FileStore) AsLengthDeclarableUpload(upload handler.Upload) handler.LengthDeclarableUpload {
	return upload.(*fileUpload)
}

func (store FileStore) AsConcatableUpload(upload handler.Upload) handler.ConcatableUpload {
	return upload.(*fileUpload)
}

// binPath returns the path to the file storing the binary data.
func (store FileStore) binPath(id string) string {
	return filepath.Join(store.Path, id)
}

// infoPath returns the path to the .info file storing the file's info.
func (store FileStore) infoPath(id string) string {
	return filepath.Join(store.Path, id+".info")
}

type fileUpload struct {
	// info stores the current information about the upload
	info handler.FileInfo
	// infoPath is the path to the .info file
	infoPath string
	// binPath is the path to the binary file (which has no extension)
	binPath string
}

func (upload *fileUpload) GetInfo(ctx context.Context) (handler.FileInfo, error) {
	return upload.info, nil
}

func (upload *fileUpload) WriteChunk(ctx context.Context, offset int64, src io.Reader) (int64, error) {
	file, err := os.OpenFile(upload.binPath, os.O_WRONLY|os.O_APPEND, defaultFilePerm)
	if err != nil {
		return 0, err
	}
	defer file.Close()

	n, err := io.Copy(file, src)

	upload.info.Offset += n
	return n, err
}

func (upload *fileUpload) GetReader(ctx context.Context) (io.Reader, error) {
	return os.Open(upload.binPath)
}

func (upload *fileUpload) Terminate(ctx context.Context) error {
	if err := os.Remove(upload.infoPath); err != nil {
		return err
	}
	if err := os.Remove(upload.binPath); err != nil {
		return err
	}
	return nil
}

func (upload *fileUpload) ConcatUploads(ctx context.Context, uploads []handler.Upload) (err error) {
	file, err := os.OpenFile(upload.binPath, os.O_WRONLY|os.O_APPEND, defaultFilePerm)
	if err != nil {
		return err
	}
	defer file.Close()

	for _, partialUpload := range uploads {
		fileUpload := partialUpload.(*fileUpload)

		src, err := os.Open(fileUpload.binPath)
		if err != nil {
			return err
		}

		if _, err := io.Copy(file, src); err != nil {
			return err
		}
	}

	return
}

func (upload *fileUpload) DeclareLength(ctx context.Context, length int64) error {
	upload.info.Size = length
	upload.info.SizeIsDeferred = false
	return upload.writeInfo()
}

// writeInfo updates the entire information. Everything will be overwritten.
func (upload *fileUpload) writeInfo() error {
	data, err := json.Marshal(upload.info)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(upload.infoPath, data, defaultFilePerm)
}

func (upload *fileUpload) FinishUpload(ctx context.Context) error {
	return nil
}
<|endoftext|>


<commit_before>package renderer

import (
	"fmt"
	"io/ioutil"
	"mime"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	gopi "github.com/djthorpe/gopi/v3"
	"github.com/djthorpe/gopi/v3/pkg/http/handler"
)

/////////////////////////////////////////////////////////////////////
// TYPES

type HttpIndexRenderer struct {
	gopi.Unit
	gopi.Logger
	gopi.HttpTemplate

	template string
}

type HttpIndexContent struct {
	Path    string         `json:"path"`
	Content []IndexContent `json:"content"`
}

type IndexContent struct {
	Path    string      `json:"path"`
	Name    string      `json:"name"`
	IsDir   bool        `json:"isdir"`
	Size    int64       `json:"size"`
	ModTime time.Time   `json:"modtime"`
	Content interface{} `json:"content"`
}

/////////////////////////////////////////////////////////////////////
// LIFECYCLE

func (this *HttpIndexRenderer) New(gopi.Config) error {
	this.Require(this.Logger, this.HttpTemplate)

	if err := this.HttpTemplate.RegisterRenderer(this); err != nil {
		return err
	} else {
		this.template = "index.tmpl"
	}

	// Return success
	return nil
}

/////////////////////////////////////////////////////////////////////
// PUBLIC METHODS

func (this *HttpIndexRenderer) ServeContent(docroot string, req *http.Request) (gopi.HttpRenderContext, error) {
	// Add a slash if not at the end
	if strings.HasSuffix(req.URL.Path, "/") == false {
		return gopi.HttpRenderContext{}, handler.Redirect(req, http.StatusPermanentRedirect, req.URL.Path+"/")
	}

	// Compute physical path
	abspath := filepath.Join(docroot, req.URL.Path)

	// Update modified time based on all eligible files
	content := HttpIndexContent{
		Path: req.URL.Path,
	}

	if mtime, err := filesForFolder(abspath, func(file os.FileInfo) error {
		if relpath, err := filepath.Rel(docroot, abspath); err != nil {
			return err
		} else {
			content.Content = append(content.Content, this.GetContent(content.Path, relpath, file))
		}
		return nil
	}); err != nil {
		return gopi.HttpRenderContext{}, err
	} else {
		return gopi.HttpRenderContext{
			Template: this.template,
			Type:     mime.TypeByExtension(".html"),
			Content:  content,
			Modified: mtime,
		}, nil
	}
}

func (this *HttpIndexRenderer) IsModifiedSince(docroot string, req *http.Request, t time.Time) bool {
	path := filepath.Join(docroot, req.URL.Path)

	// Update modified time based on all eligible files
	if mtime, err := filesForFolder(path, nil); err != nil {
		return false
	} else {
		return mtime.After(t)
	}
}

func (this *HttpIndexRenderer) GetContent(base, relpath string, file os.FileInfo) IndexContent {
	// Don't render directories
	result := IndexContent{
		Path:    filepath.Join(base, relpath, file.Name()),
		Name:    file.Name(),
		IsDir:   file.IsDir(),
		Size:    file.Size(),
		ModTime: file.ModTime(),
	}

	// Read content if it's not a directory
	if file.IsDir() {
		result.Path += "/"
	} else if r, err := http.NewRequest(http.MethodGet, result.Path, nil); err == nil {
		if c, err := this.HttpTemplate.Render(r); err != nil {
			this.Debugf("GetContent: %q: %v", r.URL, err)
		} else {
			result.Content = c
		}
	}

	// Return result
	return result
}

/////////////////////////////////////////////////////////////////////
// STRINGIFY

func (this *HttpIndexRenderer) String() string {
	str := "<http.indexrenderer"
	if this.template != "" {
		str += fmt.Sprintf(" template=%q", this.template)
	}
	return str + ">"
}

/////////////////////////////////////////////////////////////////////
// PRIVATE METHODS

func filesForFolder(path string, fn func(os.FileInfo) error) (time.Time, error) {
	var mtime time.Time

	// Check path
	if stat, err := os.Stat(path); err != nil {
		return time.Time{}, err
	} else if stat.IsDir() == false {
		return time.Time{}, gopi.ErrBadParameter
	} else {
		mtime = stat.ModTime()
	}

	// Walk through files in this folder in alphabetical order
	files, err := ioutil.ReadDir(path)
	if err != nil {
		return time.Time{}, err
	}

	// Render files, ignoring any subsequent folders
	for _, file := range files {
		if file.Mode().IsRegular() == false && file.Mode().IsDir() == false {
			continue
		}
		if strings.HasPrefix(file.Name(), ".") {
			continue
		}
		if file.ModTime().After(mtime) {
			mtime = file.ModTime()
		}
		if fn != nil {
			if err := fn(file); err != nil {
				return time.Time{}, err
			}
		}
	}

	// Return success
	return mtime, nil
}
<commit_msg>Added Render method for templating<commit_after>package renderer

import (
	"fmt"
	"io/ioutil"
	"mime"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	gopi "github.com/djthorpe/gopi/v3"
	"github.com/djthorpe/gopi/v3/pkg/http/handler"
)

/////////////////////////////////////////////////////////////////////
// TYPES

type HttpIndexRenderer struct {
	gopi.Unit
	gopi.Logger
	gopi.HttpTemplate

	template string
}

type HttpIndexContent struct {
	Path    string         `json:"path"`
	Content []IndexContent `json:"content"`
}

type IndexContent struct {
	Path    string      `json:"path"`
	Name    string      `json:"name"`
	IsDir   bool        `json:"isdir"`
	Size    int64       `json:"size"`
	ModTime time.Time   `json:"modtime"`
	Content interface{} `json:"content"`
}

/////////////////////////////////////////////////////////////////////
// LIFECYCLE

func (this *HttpIndexRenderer) New(gopi.Config) error {
	this.Require(this.Logger, this.HttpTemplate)

	if err := this.HttpTemplate.RegisterRenderer(this); err != nil {
		return err
	} else {
		this.template = "index.tmpl"
	}

	// Return success
	return nil
}

/////////////////////////////////////////////////////////////////////
// PUBLIC METHODS

func (this *HttpIndexRenderer) ServeContent(docroot string, req *http.Request) (gopi.HttpRenderContext, error) {
	// Add a slash if not at the end
	if strings.HasSuffix(req.URL.Path, "/") == false {
		return gopi.HttpRenderContext{}, handler.Redirect(req, http.StatusPermanentRedirect, req.URL.Path+"/")
	}

	// Compute physical path
	abspath := filepath.Join(docroot, req.URL.Path)

	// Update modified time based on all eligible files
	content := HttpIndexContent{
		Path: req.URL.Path,
	}

	if mtime, err := filesForFolder(abspath, func(file os.FileInfo) error {
		if relpath, err := filepath.Rel(docroot, abspath); err != nil {
			return err
		} else {
			content.Content = append(content.Content, this.GetContent(content.Path, relpath, file))
		}
		return nil
	}); err != nil {
		return gopi.HttpRenderContext{}, err
	} else {
		return gopi.HttpRenderContext{
			Template: this.template,
			Type:     mime.TypeByExtension(".html"),
			Content:  content,
			Modified: mtime,
		}, nil
	}
}

func (this *HttpIndexRenderer) IsModifiedSince(docroot string, req *http.Request, t time.Time) bool {
	path := filepath.Join(docroot, req.URL.Path)

	// Update modified time based on all eligible files
	if mtime, err := filesForFolder(path, nil); err != nil {
		return false
	} else {
		return mtime.After(t)
	}
}

func (this *HttpIndexRenderer) GetContent(base, relpath string, file os.FileInfo) IndexContent {
	// Don't render directories
	result := IndexContent{
		Path:    filepath.Join(base, relpath, file.Name()),
		Name:    file.Name(),
		IsDir:   file.IsDir(),
		Size:    file.Size(),
		ModTime: file.ModTime(),
	}

	// Read content if it's not a directory
	if file.IsDir() {
		result.Path += "/"
	} else if r, err := http.NewRequest(http.MethodGet, result.Path, nil); err == nil {
		if c, err := this.HttpTemplate.Render(r); err != nil {
			this.Debugf("GetContent: %q: %v", r.URL, err)
		} else {
			result.Content = c.Content
		}
	}

	// Return result
	return result
}

/////////////////////////////////////////////////////////////////////
// STRINGIFY

func (this *HttpIndexRenderer) String() string {
	str := "<http.indexrenderer"
	if this.template != "" {
		str += fmt.Sprintf(" template=%q", this.template)
	}
	return str + ">"
}

/////////////////////////////////////////////////////////////////////
// PRIVATE METHODS

func filesForFolder(path string, fn func(os.FileInfo) error) (time.Time, error) {
	var mtime time.Time

	// Check path
	if stat, err := os.Stat(path); err != nil {
		return time.Time{}, err
	} else if stat.IsDir() == false {
		return time.Time{}, gopi.ErrBadParameter
	} else {
		mtime = stat.ModTime()
	}

	// Walk through files in this folder in alphabetical order
	files, err := ioutil.ReadDir(path)
	if err != nil {
		return time.Time{}, err
	}

	// Render files, ignoring any subsequent folders
	for _, file := range files {
		if file.Mode().IsRegular() == false && file.Mode().IsDir() == false {
			continue
		}
		if strings.HasPrefix(file.Name(), ".") {
			continue
		}
		if file.ModTime().After(mtime) {
			mtime = file.ModTime()
		}
		if fn != nil {
			if err := fn(file); err != nil {
				return time.Time{}, err
			}
		}
	}

	// Return success
	return mtime, nil
}
<|endoftext|>


<commit_before>package landscaper

import (
	"errors"
	"fmt"

	"github.com/Sirupsen/logrus"
	"google.golang.org/grpc"
	"k8s.io/helm/pkg/helm"
)

// Executor is responsible for applying a desired landscape to the actual landscape
type Executor interface {
	Apply([]*Component, []*Component) error

	CreateComponent(*Component) error
	UpdateComponent(*Component) error
	DeleteComponent(*Component) error
}

type executor struct {
	env *Environment
}

// NewExecutor is a factory method to create a new Executor
func NewExecutor(env *Environment) (Executor, error) {
	if err := env.EnsureHelmClient(); err != nil {
		return nil, err
	}

	return &executor{env: env}, nil
}

// Apply transforms the current state into the desired state
func (e *executor) Apply(desired, current []*Component) error {
	create, update, delete := diff(desired, current)

	logrus.WithFields(logrus.Fields{
		"desired": desired,
		"current": current,
		"create":  create,
		"update":  update,
		"delete":  delete,
		"dryrun":  e.env.DryRun,
	}).Info("apply desired state")

	for _, cmp := range delete {
		if err := e.DeleteComponent(cmp); err != nil {
			return err
		}
	}

	for _, cmp := range create {
		if err := e.CreateComponent(cmp); err != nil {
			return err
		}
	}

	for _, cmp := range update {
		if err := e.UpdateComponent(cmp); err != nil {
			return err
		}
	}

	return nil
}

// CreateComponent creates the given Component
func (e *executor) CreateComponent(cmp *Component) error {
	chartRef := fmt.Sprintf("%s/%s", e.env.HelmRepositoryName, cmp.Release.Chart)
	releaseName := e.env.ReleaseName(cmp.Name)

	// We need to ensure the chart is available on the local system. LoadChart will ensure
	// this is the case by downloading the chart if it is not there yet
	_, chartPath, err := e.env.ChartLoader.Load(chartRef)
	if err != nil {
		return err
	}

	rawValues, err := cmp.Configuration.YAML()
	if err != nil {
		return err
	}

	logrus.WithFields(logrus.Fields{
		"release":   releaseName,
		"chartRef":  chartRef,
		"chartPath": chartPath,
		"values":    cmp.Configuration,
		"dryrun":    e.env.DryRun,
	}).Info("create component")

	_, err = e.env.HelmClient.InstallRelease(
		chartPath,
		e.env.Namespace,
		helm.ValueOverrides([]byte(rawValues)),
		helm.ReleaseName(releaseName),
		helm.InstallDryRun(e.env.DryRun),
		helm.InstallReuseName(true),
	)
	if err != nil {
		return errors.New(grpc.ErrorDesc(err))
	}

	return nil
}

// UpdateComponent updates the given Component
func (e *executor) UpdateComponent(cmp *Component) error {
	releaseName := e.env.ReleaseName(cmp.Name)
	chartRef := fmt.Sprintf("%s/%s", e.env.HelmRepositoryName, cmp.Release.Chart)

	// We need to ensure the chart is available on the local system. LoadChart will ensure
	// this is the case by downloading the chart if it is not there yet
	_, chartPath, err := e.env.ChartLoader.Load(chartRef)
	if err != nil {
		return err
	}

	rawValues, err := cmp.Configuration.YAML()
	if err != nil {
		return err
	}

	logrus.WithFields(logrus.Fields{
		"release":   releaseName,
		"chartRef":  chartRef,
		"chartPath": chartPath,
		"values":    cmp.Configuration,
		"dryrun":    e.env.DryRun,
	}).Info("update component")

	_, err = e.env.HelmClient.UpdateRelease(
		releaseName,
		chartPath,
		helm.UpdateValueOverrides([]byte(rawValues)),
		helm.UpgradeDryRun(e.env.DryRun),
	)
	if err != nil {
		return errors.New(grpc.ErrorDesc(err))
	}

	return nil
}

// DeleteComponent removes the given Component
func (e *executor) DeleteComponent(cmp *Component) error {
	releaseName := e.env.ReleaseName(cmp.Name)

	logrus.WithFields(logrus.Fields{
		"release": releaseName,
		"values":  cmp.Configuration,
		"dryrun":  e.env.DryRun,
	}).Info("delete component")

	_, err := e.env.HelmClient.DeleteRelease(
		releaseName,
		helm.DeletePurge(true),
		helm.DeleteDryRun(e.env.DryRun),
	)
	if err != nil {
		return errors.New(grpc.ErrorDesc(err))
	}

	return nil
}

// diff takes desired and current components, and returns the components to create, update and delete to get from current to desired
func diff(desired, current []*Component) (create, update, delete []*Component) {
	desiredMap := make(map[string]*Component)
	currentMap := make(map[string]*Component)

	for _, c := range desired {
		desiredMap[c.Name] = c
	}
	for _, c := range current {
		currentMap[c.Name] = c
	}

	for name, desiredCmp := range desiredMap {
		if currentCmp, ok := currentMap[name]; ok {
			if !desiredCmp.Equals(currentCmp) {
				update = append(update, desiredCmp)
			}
		} else {
			create = append(create, desiredCmp)
		}
	}

	for name, currentCmp := range currentMap {
		if _, ok := desiredMap[name]; !ok {
			delete = append(delete, currentCmp)
		}
	}

	return create, update, delete
}
<commit_msg>Work around helm delete not respecting dryrun<commit_after>package landscaper

import (
	"errors"
	"fmt"

	"github.com/Sirupsen/logrus"
	"google.golang.org/grpc"
	"k8s.io/helm/pkg/helm"
)

// Executor is responsible for applying a desired landscape to the actual landscape
type Executor interface {
	Apply([]*Component, []*Component) error

	CreateComponent(*Component) error
	UpdateComponent(*Component) error
	DeleteComponent(*Component) error
}

type executor struct {
	env *Environment
}

// NewExecutor is a factory method to create a new Executor
func NewExecutor(env *Environment) (Executor, error) {
	if err := env.EnsureHelmClient(); err != nil {
		return nil, err
	}

	return &executor{env: env}, nil
}

// Apply transforms the current state into the desired state
func (e *executor) Apply(desired, current []*Component) error {
	create, update, delete := diff(desired, current)

	logrus.WithFields(logrus.Fields{
		"desired": desired,
		"current": current,
		"create":  create,
		"update":  update,
		"delete":  delete,
		"dryrun":  e.env.DryRun,
	}).Info("apply desired state")

	for _, cmp := range delete {
		if err := e.DeleteComponent(cmp); err != nil {
			return err
		}
	}

	for _, cmp := range create {
		if err := e.CreateComponent(cmp); err != nil {
			return err
		}
	}

	for _, cmp := range update {
		if err := e.UpdateComponent(cmp); err != nil {
			return err
		}
	}

	return nil
}

// CreateComponent creates the given Component
func (e *executor) CreateComponent(cmp *Component) error {
	chartRef := fmt.Sprintf("%s/%s", e.env.HelmRepositoryName, cmp.Release.Chart)
	releaseName := e.env.ReleaseName(cmp.Name)

	// We need to ensure the chart is available on the local system. LoadChart will ensure
	// this is the case by downloading the chart if it is not there yet
	_, chartPath, err := e.env.ChartLoader.Load(chartRef)
	if err != nil {
		return err
	}

	rawValues, err := cmp.Configuration.YAML()
	if err != nil {
		return err
	}

	logrus.WithFields(logrus.Fields{
		"release":   releaseName,
		"chartRef":  chartRef,
		"chartPath": chartPath,
		"values":    cmp.Configuration,
		"dryrun":    e.env.DryRun,
	}).Info("create component")

	_, err = e.env.HelmClient.InstallRelease(
		chartPath,
		e.env.Namespace,
		helm.ValueOverrides([]byte(rawValues)),
		helm.ReleaseName(releaseName),
		helm.InstallDryRun(e.env.DryRun),
		helm.InstallReuseName(true),
	)
	if err != nil {
		return errors.New(grpc.ErrorDesc(err))
	}

	return nil
}

// UpdateComponent updates the given Component
func (e *executor) UpdateComponent(cmp *Component) error {
	releaseName := e.env.ReleaseName(cmp.Name)
	chartRef := fmt.Sprintf("%s/%s", e.env.HelmRepositoryName, cmp.Release.Chart)

	// We need to ensure the chart is available on the local system. LoadChart will ensure
	// this is the case by downloading the chart if it is not there yet
	_, chartPath, err := e.env.ChartLoader.Load(chartRef)
	if err != nil {
		return err
	}

	rawValues, err := cmp.Configuration.YAML()
	if err != nil {
		return err
	}

	logrus.WithFields(logrus.Fields{
		"release":   releaseName,
		"chartRef":  chartRef,
		"chartPath": chartPath,
		"values":    cmp.Configuration,
		"dryrun":    e.env.DryRun,
	}).Info("update component")

	_, err = e.env.HelmClient.UpdateRelease(
		releaseName,
		chartPath,
		helm.UpdateValueOverrides([]byte(rawValues)),
		helm.UpgradeDryRun(e.env.DryRun),
	)
	if err != nil {
		return errors.New(grpc.ErrorDesc(err))
	}

	return nil
}

// DeleteComponent removes the given Component
func (e *executor) DeleteComponent(cmp *Component) error {
	releaseName := e.env.ReleaseName(cmp.Name)

	logrus.WithFields(logrus.Fields{
		"release": releaseName,
		"values":  cmp.Configuration,
		"dryrun":  e.env.DryRun,
	}).Info("delete component")

    // TODO: work around https://github.com/kubernetes/helm/pull/1527 as long as needed
	if (e.env.DryRun) {
	    return nil
	}

	_, err := e.env.HelmClient.DeleteRelease(
		releaseName,
		helm.DeletePurge(true),
		helm.DeleteDryRun(e.env.DryRun),
	)
	if err != nil {
		return errors.New(grpc.ErrorDesc(err))
	}

	return nil
}

// diff takes desired and current components, and returns the components to create, update and delete to get from current to desired
func diff(desired, current []*Component) (create, update, delete []*Component) {
	desiredMap := make(map[string]*Component)
	currentMap := make(map[string]*Component)

	for _, c := range desired {
		desiredMap[c.Name] = c
	}
	for _, c := range current {
		currentMap[c.Name] = c
	}

	for name, desiredCmp := range desiredMap {
		if currentCmp, ok := currentMap[name]; ok {
			if !desiredCmp.Equals(currentCmp) {
				update = append(update, desiredCmp)
			}
		} else {
			create = append(create, desiredCmp)
		}
	}

	for name, currentCmp := range currentMap {
		if _, ok := desiredMap[name]; !ok {
			delete = append(delete, currentCmp)
		}
	}

	return create, update, delete
}
<|endoftext|>


<commit_before>package serverapi

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/skatteetaten/aoc/pkg/jsonutil"
	"github.com/skatteetaten/aoc/pkg/openshift"
	"io/ioutil"
	"net/http"
	"strings"
)

const apiNotInstalledResponse = "Application is not available"

// Structs to represent return data from the API interface
type ApiReturnObjects struct {
	Sources          json.RawMessage            `json:"sources"`
	Errors           []string                   `json:"errors"`
	Valid            bool                       `json:"valid"`
	Config           json.RawMessage            `json:"config"`
	OpenshiftObjects map[string]json.RawMessage `json:"openshiftObjects"`
}

type ApiReturn struct {
	Sources          json.RawMessage `json:"sources"`
	Errors           []string        `json:"errors"`
	Valid            bool            `json:"valid"`
	Config           json.RawMessage `json:"config"`
	OpenshiftObjects json.RawMessage `json:"openshiftObjects"`
}

func GetApiAddress(clusterName string, localhost bool) (apiAddress string) {
	if localhost {
		apiAddress = "http://localhost:8080"
	} else {
		apiAddress = "http://boober-aos-bas-dev." + clusterName + ".paas.skead.no"
	}
	return
}

// Check for valid login, that is we have a configuration with at least one reachable cluster
func ValidateLogin(openshiftConfig *openshift.OpenshiftConfig) (output bool) {
	var openshiftCluster *openshift.OpenshiftCluster
	openshiftCluster, _ = openshiftConfig.GetApiCluster()
	if openshiftCluster != nil {
		if !openshiftCluster.HasValidToken() {
			return false
		}
	}
	return true
}

func GetApiSetupUrl(clusterName string, localhost bool) string {
	return GetApiAddress(clusterName, localhost) + "/setup"
}

func CallApi(combindedJson string, showConfig bool, showObjects bool, api bool, localhost bool, verbose bool,
	openshiftConfig *openshift.OpenshiftConfig, dryRun bool) (output string, err error) {
	//var openshiftConfig *openshift.OpenshiftConfig
	var apiCluster *openshift.OpenshiftCluster

	if localhost {
		var token string = ""
		apiCluster, err = openshiftConfig.GetApiCluster()
		if apiCluster != nil {
			token = apiCluster.Token
		}
		output, err = callApiInstance(combindedJson, showConfig, showObjects, verbose,
			GetApiSetupUrl("localhost", localhost), token, dryRun)
		if err != nil {
			return
		}
	} else {
		var errorString string
		var newline string
		for i := range openshiftConfig.Clusters {
			if openshiftConfig.Clusters[i].Reachable {
				if !api || openshiftConfig.Clusters[i].Name == openshiftConfig.APICluster {
					out, err := callApiInstance(combindedJson, showConfig, showObjects, verbose,
						GetApiSetupUrl(openshiftConfig.Clusters[i].Name, localhost),
						openshiftConfig.Clusters[i].Token, dryRun)
					if err == nil {
						output += fmt.Sprintf("%v\n", out)
					} else {
						if err.Error() != "" {
							errorString += newline + err.Error()
							newline = "\n"
						}
					}
				}
			}
		}
		if errorString != "" {
			return output, errors.New(errorString)
		}
	}
	return output, nil
}

func callApiInstance(combindedJson string, showConfig bool, showObjects bool, verbose bool, url string, token string, dryRun bool) (string, error) {
	var output string

	if verbose {
		fmt.Print("Sending config to Boober at " + url + "... ")
	}

	var jsonStr = []byte(combindedJson)

	req, err := http.NewRequest(http.MethodPut, url, bytes.NewBuffer(jsonStr))
	req.Header.Set("Content-Type", "application/json")
	if err != nil {
		return "", errors.New(fmt.Sprintf("Internal error in NewRequest: %v", err))
	}

	req.Header.Set("Authentication", "Bearer: "+token)
	req.Header.Add("dryrun", fmt.Sprintf("%v", dryRun))
	client := &http.Client{}

	resp, err := client.Do(req)
	if err != nil {
		if verbose {
			fmt.Println("FAIL.  Error connecting to Boober service")
		}
		return "", errors.New(fmt.Sprintf("Error connecting to the Boober service on %v: %v", url, err))
	}

	defer resp.Body.Close()
	body, _ := ioutil.ReadAll(resp.Body)
	bodyStr := string(body)

	//fmt.Println("HTTP Status code: " + strconv.Itoa(resp.StatusCode))
	if (resp.StatusCode != http.StatusOK) && (resp.StatusCode != http.StatusBadRequest) {
		//fmt.Println("Not StatusOK and not StatusBadRequest")
		var errorstring string
		if !strings.Contains(bodyStr, apiNotInstalledResponse) {
			errorstring = fmt.Sprintf("Internal error on %v: %v", url, bodyStr)
		}
		if verbose {
			if strings.Contains(bodyStr, apiNotInstalledResponse) {
				fmt.Println("WARN.  Boober not available")
			} else {
				fmt.Println("FAIL.  Internal error")
			}
		}
		return "", errors.New(fmt.Sprintf(errorstring))
	}

	var booberReturn ApiReturn

	if resp.StatusCode == http.StatusBadRequest {
		// We have a validation situation, give error
		if verbose {
			fmt.Println("FAIL.  Error in configuration")
		}
		return "", errors.New(fmt.Sprintf(bodyStr))
	}

	err = json.Unmarshal(body, &booberReturn)
	if err != nil {
		return "", errors.New(fmt.Sprintf("Error unmarshalling Boober return: %v\n", err.Error()))
	}
	for _, message := range booberReturn.Errors {
		fmt.Println("DEBUG: Error from Boober:  " + message)
	}
	if verbose {
		var apiReturnObjects ApiReturnObjects
		err = json.Unmarshal(body, &apiReturnObjects)
		if err != nil {
			return "", errors.New(fmt.Sprintf("Error unmarshalling Boober return: %v\n", err.Error()))
		}
		var countMap map[string]int = make(map[string]int)
		for key := range apiReturnObjects.OpenshiftObjects {
			countMap[key]++
		}

		var space string
		var count int
		var out string
		for key := range countMap {
			out += fmt.Sprintf("%v%v: %v", space, key, countMap[key])
			space = "  "
			count++
		}
		if count > 0 {
			output := fmt.Sprintf("OK.  Objects: %v  (%v)", count, out)
			fmt.Println(output)
		}
	}

	if showConfig {
		output += jsonutil.PrettyPrintJson(string(booberReturn.Config))
	}

	if showObjects {
		output += jsonutil.PrettyPrintJson(string(booberReturn.OpenshiftObjects))
	}

	return output, nil
}
<commit_msg>Pretty printing entire payload when in verbose mode (temporary code)<commit_after>package serverapi

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/skatteetaten/aoc/pkg/jsonutil"
	"github.com/skatteetaten/aoc/pkg/openshift"
	"io/ioutil"
	"net/http"
	"strings"
)

const apiNotInstalledResponse = "Application is not available"

// Structs to represent return data from the API interface
type ApiReturnObjects struct {
	Sources          json.RawMessage            `json:"sources"`
	Errors           []string                   `json:"errors"`
	Valid            bool                       `json:"valid"`
	Config           json.RawMessage            `json:"config"`
	OpenshiftObjects map[string]json.RawMessage `json:"openshiftObjects"`
}

type ApiReturn struct {
	Sources          json.RawMessage `json:"sources"`
	Errors           []string        `json:"errors"`
	Valid            bool            `json:"valid"`
	Config           json.RawMessage `json:"config"`
	OpenshiftObjects json.RawMessage `json:"openshiftObjects"`
}

func GetApiAddress(clusterName string, localhost bool) (apiAddress string) {
	if localhost {
		apiAddress = "http://localhost:8080"
	} else {
		apiAddress = "http://boober-aos-bas-dev." + clusterName + ".paas.skead.no"
	}
	return
}

// Check for valid login, that is we have a configuration with at least one reachable cluster
func ValidateLogin(openshiftConfig *openshift.OpenshiftConfig) (output bool) {
	var openshiftCluster *openshift.OpenshiftCluster
	openshiftCluster, _ = openshiftConfig.GetApiCluster()
	if openshiftCluster != nil {
		if !openshiftCluster.HasValidToken() {
			return false
		}
	}
	return true
}

func GetApiSetupUrl(clusterName string, localhost bool) string {
	return GetApiAddress(clusterName, localhost) + "/setup"
}

func CallApi(combindedJson string, showConfig bool, showObjects bool, api bool, localhost bool, verbose bool,
	openshiftConfig *openshift.OpenshiftConfig, dryRun bool) (output string, err error) {
	//var openshiftConfig *openshift.OpenshiftConfig
	var apiCluster *openshift.OpenshiftCluster

	if localhost {
		var token string = ""
		apiCluster, err = openshiftConfig.GetApiCluster()
		if apiCluster != nil {
			token = apiCluster.Token
		}
		output, err = callApiInstance(combindedJson, showConfig, showObjects, verbose,
			GetApiSetupUrl("localhost", localhost), token, dryRun)
		if err != nil {
			return
		}
	} else {
		var errorString string
		var newline string
		for i := range openshiftConfig.Clusters {
			if openshiftConfig.Clusters[i].Reachable {
				if !api || openshiftConfig.Clusters[i].Name == openshiftConfig.APICluster {
					out, err := callApiInstance(combindedJson, showConfig, showObjects, verbose,
						GetApiSetupUrl(openshiftConfig.Clusters[i].Name, localhost),
						openshiftConfig.Clusters[i].Token, dryRun)
					if err == nil {
						output += fmt.Sprintf("%v\n", out)
					} else {
						if err.Error() != "" {
							errorString += newline + err.Error()
							newline = "\n"
						}
					}
				}
			}
		}
		if errorString != "" {
			return output, errors.New(errorString)
		}
	}
	return output, nil
}

func callApiInstance(combindedJson string, showConfig bool, showObjects bool, verbose bool, url string, token string, dryRun bool) (string, error) {
	var output string

	if verbose {
		fmt.Print("Sending config to Boober at " + url + "... ")
	}

	var jsonStr = []byte(combindedJson)

	req, err := http.NewRequest(http.MethodPut, url, bytes.NewBuffer(jsonStr))
	req.Header.Set("Content-Type", "application/json")
	if err != nil {
		return "", errors.New(fmt.Sprintf("Internal error in NewRequest: %v", err))
	}

	req.Header.Set("Authentication", "Bearer: "+token)
	req.Header.Add("dryrun", fmt.Sprintf("%v", dryRun))
	client := &http.Client{}

	resp, err := client.Do(req)
	if err != nil {
		if verbose {
			fmt.Println("FAIL.  Error connecting to Boober service")
		}
		return "", errors.New(fmt.Sprintf("Error connecting to the Boober service on %v: %v", url, err))
	}

	defer resp.Body.Close()
	body, _ := ioutil.ReadAll(resp.Body)
	bodyStr := string(body)

	//fmt.Println("HTTP Status code: " + strconv.Itoa(resp.StatusCode))
	if (resp.StatusCode != http.StatusOK) && (resp.StatusCode != http.StatusBadRequest) {
		//fmt.Println("Not StatusOK and not StatusBadRequest")
		var errorstring string
		if !strings.Contains(bodyStr, apiNotInstalledResponse) {
			errorstring = fmt.Sprintf("Internal error on %v: %v", url, bodyStr)
		}
		if verbose {
			if strings.Contains(bodyStr, apiNotInstalledResponse) {
				fmt.Println("WARN.  Boober not available")
			} else {
				fmt.Println("FAIL.  Internal error")
			}
		}
		return "", errors.New(fmt.Sprintf(errorstring))
	}

	var booberReturn ApiReturn

	if resp.StatusCode == http.StatusBadRequest {
		// We have a validation situation, give error
		if verbose {
			fmt.Println("FAIL.  Error in configuration")
		}
		return "", errors.New(fmt.Sprintf(bodyStr))
	}

	err = json.Unmarshal(body, &booberReturn)
	if err != nil {
		return "", errors.New(fmt.Sprintf("Error unmarshalling Boober return: %v\n", err.Error()))
	}
	for _, message := range booberReturn.Errors {
		fmt.Println("DEBUG: Error from Boober:  " + message)
	}
	if verbose {
		var prettyJSON bytes.Buffer
		json.Indent(&prettyJSON, body, "", "\t")
		fmt.Println("")
		fmt.Println(string(prettyJSON.Bytes()))

		var apiReturnObjects ApiReturnObjects
		err = json.Unmarshal(body, &apiReturnObjects)
		if err != nil {
			return "", errors.New(fmt.Sprintf("Error unmarshalling Boober return: %v\n", err.Error()))
		}
		var countMap map[string]int = make(map[string]int)
		for key := range apiReturnObjects.OpenshiftObjects {
			countMap[key]++
		}

		var space string
		var count int
		var out string
		for key := range countMap {
			out += fmt.Sprintf("%v%v: %v", space, key, countMap[key])
			space = "  "
			count++
		}
		if count > 0 {
			output := fmt.Sprintf("OK.  Objects: %v  (%v)", count, out)
			fmt.Println(output)
		}
	}

	if showConfig {
		output += jsonutil.PrettyPrintJson(string(booberReturn.Config))
	}

	if showObjects {
		output += jsonutil.PrettyPrintJson(string(booberReturn.OpenshiftObjects))
	}

	return output, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package storage

import (
	"fmt"
	"sort"
	"strconv"
	"sync"
	"time"

	"k8s.io/kubernetes/pkg/api/errors"
	"k8s.io/kubernetes/pkg/api/meta"
	"k8s.io/kubernetes/pkg/client/cache"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/util"
	"k8s.io/kubernetes/pkg/util/clock"
	"k8s.io/kubernetes/pkg/watch"
)

const (
	// MaximumListWait determines how long we're willing to wait for a
	// list if a client specified a resource version in the future.
	MaximumListWait = 60 * time.Second
)

// watchCacheEvent is a single "watch event" that is send to users of
// watchCache. Additionally to a typical "watch.Event" it contains
// the previous value of the object to enable proper filtering in the
// upper layers.
type watchCacheEvent struct {
	Type            watch.EventType
	Object          runtime.Object
	PrevObject      runtime.Object
	ResourceVersion uint64
}

// watchCacheElement is a single "watch event" stored in a cache.
// It contains the resource version of the object and the object
// itself.
type watchCacheElement struct {
	resourceVersion uint64
	watchCacheEvent watchCacheEvent
}

// watchCache implements a Store interface.
// However, it depends on the elements implementing runtime.Object interface.
//
// watchCache is a "sliding window" (with a limited capacity) of objects
// observed from a watch.
type watchCache struct {
	sync.RWMutex

	// Condition on which lists are waiting for the fresh enough
	// resource version.
	cond *sync.Cond

	// Maximum size of history window.
	capacity int

	// cache is used a cyclic buffer - its first element (with the smallest
	// resourceVersion) is defined by startIndex, its last element is defined
	// by endIndex (if cache is full it will be startIndex + capacity).
	// Both startIndex and endIndex can be greater than buffer capacity -
	// you should always apply modulo capacity to get an index in cache array.
	cache      []watchCacheElement
	startIndex int
	endIndex   int

	// store will effectively support LIST operation from the "end of cache
	// history" i.e. from the moment just after the newest cached watched event.
	// It is necessary to effectively allow clients to start watching at now.
	store cache.Store

	// ResourceVersion up to which the watchCache is propagated.
	resourceVersion uint64

	// This handler is run at the end of every successful Replace() method.
	onReplace func()

	// This handler is run at the end of every Add/Update/Delete method
	// and additionally gets the previous value of the object.
	onEvent func(watchCacheEvent)

	// for testing timeouts.
	clock clock.Clock
}

func newWatchCache(capacity int) *watchCache {
	wc := &watchCache{
		capacity:        capacity,
		cache:           make([]watchCacheElement, capacity),
		startIndex:      0,
		endIndex:        0,
		store:           cache.NewStore(cache.MetaNamespaceKeyFunc),
		resourceVersion: 0,
		clock:           clock.RealClock{},
	}
	wc.cond = sync.NewCond(wc.RLocker())
	return wc
}

func (w *watchCache) Add(obj interface{}) error {
	object, resourceVersion, err := objectToVersionedRuntimeObject(obj)
	if err != nil {
		return err
	}
	event := watch.Event{Type: watch.Added, Object: object}

	f := func(obj runtime.Object) error { return w.store.Add(obj) }
	return w.processEvent(event, resourceVersion, f)
}

func (w *watchCache) Update(obj interface{}) error {
	object, resourceVersion, err := objectToVersionedRuntimeObject(obj)
	if err != nil {
		return err
	}
	event := watch.Event{Type: watch.Modified, Object: object}

	f := func(obj runtime.Object) error { return w.store.Update(obj) }
	return w.processEvent(event, resourceVersion, f)
}

func (w *watchCache) Delete(obj interface{}) error {
	object, resourceVersion, err := objectToVersionedRuntimeObject(obj)
	if err != nil {
		return err
	}
	event := watch.Event{Type: watch.Deleted, Object: object}

	f := func(obj runtime.Object) error { return w.store.Delete(obj) }
	return w.processEvent(event, resourceVersion, f)
}

func objectToVersionedRuntimeObject(obj interface{}) (runtime.Object, uint64, error) {
	object, ok := obj.(runtime.Object)
	if !ok {
		return nil, 0, fmt.Errorf("obj does not implement runtime.Object interface: %v", obj)
	}
	meta, err := meta.Accessor(object)
	if err != nil {
		return nil, 0, err
	}
	resourceVersion, err := parseResourceVersion(meta.GetResourceVersion())
	if err != nil {
		return nil, 0, err
	}
	return object, resourceVersion, nil
}

func parseResourceVersion(resourceVersion string) (uint64, error) {
	if resourceVersion == "" {
		return 0, nil
	}
	return strconv.ParseUint(resourceVersion, 10, 64)
}

func (w *watchCache) processEvent(event watch.Event, resourceVersion uint64, updateFunc func(runtime.Object) error) error {
	w.Lock()
	defer w.Unlock()
	previous, exists, err := w.store.Get(event.Object)
	if err != nil {
		return err
	}
	var prevObject runtime.Object
	if exists {
		prevObject = previous.(runtime.Object)
	}
	watchCacheEvent := watchCacheEvent{event.Type, event.Object, prevObject, resourceVersion}
	if w.onEvent != nil {
		w.onEvent(watchCacheEvent)
	}
	w.updateCache(resourceVersion, watchCacheEvent)
	w.resourceVersion = resourceVersion
	w.cond.Broadcast()
	return updateFunc(event.Object)
}

// Assumes that lock is already held for write.
func (w *watchCache) updateCache(resourceVersion uint64, event watchCacheEvent) {
	if w.endIndex == w.startIndex+w.capacity {
		// Cache is full - remove the oldest element.
		w.startIndex++
	}
	w.cache[w.endIndex%w.capacity] = watchCacheElement{resourceVersion, event}
	w.endIndex++
}

func (w *watchCache) List() []interface{} {
	w.RLock()
	defer w.RUnlock()
	return w.store.List()
}

func (w *watchCache) WaitUntilFreshAndList(resourceVersion uint64, trace *util.Trace) ([]interface{}, uint64, error) {
	startTime := w.clock.Now()
	go func() {
		// Wake us up when the time limit has expired.  The docs
		// promise that time.After (well, NewTimer, which it calls)
		// will wait *at least* the duration given. Since this go
		// routine starts sometime after we record the start time, and
		// it will wake up the loop below sometime after the broadcast,
		// we don't need to worry about waking it up before the time
		// has expired accidentally.
		<-w.clock.After(MaximumListWait)
		w.cond.Broadcast()
	}()

	w.RLock()
	defer w.RUnlock()
	for w.resourceVersion < resourceVersion {
		if w.clock.Since(startTime) >= MaximumListWait {
			return nil, 0, fmt.Errorf("time limit exceeded while waiting for resource version %v (current value: %v)", resourceVersion, w.resourceVersion)
		}
		w.cond.Wait()
	}
	if trace != nil {
		trace.Step("Cache is fresh enough")
	}
	return w.store.List(), w.resourceVersion, nil
}

func (w *watchCache) ListKeys() []string {
	w.RLock()
	defer w.RUnlock()
	return w.store.ListKeys()
}

func (w *watchCache) Get(obj interface{}) (interface{}, bool, error) {
	w.RLock()
	defer w.RUnlock()
	return w.store.Get(obj)
}

func (w *watchCache) GetByKey(key string) (interface{}, bool, error) {
	w.RLock()
	defer w.RUnlock()
	return w.store.GetByKey(key)
}

func (w *watchCache) Replace(objs []interface{}, resourceVersion string) error {
	version, err := parseResourceVersion(resourceVersion)
	if err != nil {
		return err
	}

	w.Lock()
	defer w.Unlock()

	w.startIndex = 0
	w.endIndex = 0
	if err := w.store.Replace(objs, resourceVersion); err != nil {
		return err
	}
	w.resourceVersion = version
	if w.onReplace != nil {
		w.onReplace()
	}
	w.cond.Broadcast()
	return nil
}

func (w *watchCache) SetOnReplace(onReplace func()) {
	w.Lock()
	defer w.Unlock()
	w.onReplace = onReplace
}

func (w *watchCache) SetOnEvent(onEvent func(watchCacheEvent)) {
	w.Lock()
	defer w.Unlock()
	w.onEvent = onEvent
}

func (w *watchCache) GetAllEventsSinceThreadUnsafe(resourceVersion uint64) ([]watchCacheEvent, error) {
	size := w.endIndex - w.startIndex
	oldest := w.resourceVersion
	if size > 0 {
		oldest = w.cache[w.startIndex%w.capacity].resourceVersion
	}
	if resourceVersion == 0 {
		// resourceVersion = 0 means that we don't require any specific starting point
		// and we would like to start watching from ~now.
		// However, to keep backward compatibility, we additionally need to return the
		// current state and only then start watching from that point.
		//
		// TODO: In v2 api, we should stop returning the current state - #13969.
		allItems := w.store.List()
		result := make([]watchCacheEvent, len(allItems))
		for i, item := range allItems {
			result[i] = watchCacheEvent{Type: watch.Added, Object: item.(runtime.Object)}
		}
		return result, nil
	}
	if resourceVersion < oldest-1 {
		return nil, errors.NewGone(fmt.Sprintf("too old resource version: %d (%d)", resourceVersion, oldest-1))
	}

	// Binary search the smallest index at which resourceVersion is greater than the given one.
	f := func(i int) bool {
		return w.cache[(w.startIndex+i)%w.capacity].resourceVersion > resourceVersion
	}
	first := sort.Search(size, f)
	result := make([]watchCacheEvent, size-first)
	for i := 0; i < size-first; i++ {
		result[i] = w.cache[(w.startIndex+first+i)%w.capacity].watchCacheEvent
	}
	return result, nil
}

func (w *watchCache) GetAllEventsSince(resourceVersion uint64) ([]watchCacheEvent, error) {
	w.RLock()
	defer w.RUnlock()
	return w.GetAllEventsSinceThreadUnsafe(resourceVersion)
}

func (w *watchCache) Resync() error {
	// Nothing to do
	return nil
}
<commit_msg>Extend tracing in watchCache<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package storage

import (
	"fmt"
	"sort"
	"strconv"
	"sync"
	"time"

	"k8s.io/kubernetes/pkg/api/errors"
	"k8s.io/kubernetes/pkg/api/meta"
	"k8s.io/kubernetes/pkg/client/cache"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/util"
	"k8s.io/kubernetes/pkg/util/clock"
	"k8s.io/kubernetes/pkg/watch"
)

const (
	// MaximumListWait determines how long we're willing to wait for a
	// list if a client specified a resource version in the future.
	MaximumListWait = 60 * time.Second
)

// watchCacheEvent is a single "watch event" that is send to users of
// watchCache. Additionally to a typical "watch.Event" it contains
// the previous value of the object to enable proper filtering in the
// upper layers.
type watchCacheEvent struct {
	Type            watch.EventType
	Object          runtime.Object
	PrevObject      runtime.Object
	ResourceVersion uint64
}

// watchCacheElement is a single "watch event" stored in a cache.
// It contains the resource version of the object and the object
// itself.
type watchCacheElement struct {
	resourceVersion uint64
	watchCacheEvent watchCacheEvent
}

// watchCache implements a Store interface.
// However, it depends on the elements implementing runtime.Object interface.
//
// watchCache is a "sliding window" (with a limited capacity) of objects
// observed from a watch.
type watchCache struct {
	sync.RWMutex

	// Condition on which lists are waiting for the fresh enough
	// resource version.
	cond *sync.Cond

	// Maximum size of history window.
	capacity int

	// cache is used a cyclic buffer - its first element (with the smallest
	// resourceVersion) is defined by startIndex, its last element is defined
	// by endIndex (if cache is full it will be startIndex + capacity).
	// Both startIndex and endIndex can be greater than buffer capacity -
	// you should always apply modulo capacity to get an index in cache array.
	cache      []watchCacheElement
	startIndex int
	endIndex   int

	// store will effectively support LIST operation from the "end of cache
	// history" i.e. from the moment just after the newest cached watched event.
	// It is necessary to effectively allow clients to start watching at now.
	store cache.Store

	// ResourceVersion up to which the watchCache is propagated.
	resourceVersion uint64

	// This handler is run at the end of every successful Replace() method.
	onReplace func()

	// This handler is run at the end of every Add/Update/Delete method
	// and additionally gets the previous value of the object.
	onEvent func(watchCacheEvent)

	// for testing timeouts.
	clock clock.Clock
}

func newWatchCache(capacity int) *watchCache {
	wc := &watchCache{
		capacity:        capacity,
		cache:           make([]watchCacheElement, capacity),
		startIndex:      0,
		endIndex:        0,
		store:           cache.NewStore(cache.MetaNamespaceKeyFunc),
		resourceVersion: 0,
		clock:           clock.RealClock{},
	}
	wc.cond = sync.NewCond(wc.RLocker())
	return wc
}

func (w *watchCache) Add(obj interface{}) error {
	object, resourceVersion, err := objectToVersionedRuntimeObject(obj)
	if err != nil {
		return err
	}
	event := watch.Event{Type: watch.Added, Object: object}

	f := func(obj runtime.Object) error { return w.store.Add(obj) }
	return w.processEvent(event, resourceVersion, f)
}

func (w *watchCache) Update(obj interface{}) error {
	object, resourceVersion, err := objectToVersionedRuntimeObject(obj)
	if err != nil {
		return err
	}
	event := watch.Event{Type: watch.Modified, Object: object}

	f := func(obj runtime.Object) error { return w.store.Update(obj) }
	return w.processEvent(event, resourceVersion, f)
}

func (w *watchCache) Delete(obj interface{}) error {
	object, resourceVersion, err := objectToVersionedRuntimeObject(obj)
	if err != nil {
		return err
	}
	event := watch.Event{Type: watch.Deleted, Object: object}

	f := func(obj runtime.Object) error { return w.store.Delete(obj) }
	return w.processEvent(event, resourceVersion, f)
}

func objectToVersionedRuntimeObject(obj interface{}) (runtime.Object, uint64, error) {
	object, ok := obj.(runtime.Object)
	if !ok {
		return nil, 0, fmt.Errorf("obj does not implement runtime.Object interface: %v", obj)
	}
	meta, err := meta.Accessor(object)
	if err != nil {
		return nil, 0, err
	}
	resourceVersion, err := parseResourceVersion(meta.GetResourceVersion())
	if err != nil {
		return nil, 0, err
	}
	return object, resourceVersion, nil
}

func parseResourceVersion(resourceVersion string) (uint64, error) {
	if resourceVersion == "" {
		return 0, nil
	}
	return strconv.ParseUint(resourceVersion, 10, 64)
}

func (w *watchCache) processEvent(event watch.Event, resourceVersion uint64, updateFunc func(runtime.Object) error) error {
	w.Lock()
	defer w.Unlock()
	previous, exists, err := w.store.Get(event.Object)
	if err != nil {
		return err
	}
	var prevObject runtime.Object
	if exists {
		prevObject = previous.(runtime.Object)
	}
	watchCacheEvent := watchCacheEvent{event.Type, event.Object, prevObject, resourceVersion}
	if w.onEvent != nil {
		w.onEvent(watchCacheEvent)
	}
	w.updateCache(resourceVersion, watchCacheEvent)
	w.resourceVersion = resourceVersion
	w.cond.Broadcast()
	return updateFunc(event.Object)
}

// Assumes that lock is already held for write.
func (w *watchCache) updateCache(resourceVersion uint64, event watchCacheEvent) {
	if w.endIndex == w.startIndex+w.capacity {
		// Cache is full - remove the oldest element.
		w.startIndex++
	}
	w.cache[w.endIndex%w.capacity] = watchCacheElement{resourceVersion, event}
	w.endIndex++
}

func (w *watchCache) List() []interface{} {
	w.RLock()
	defer w.RUnlock()
	return w.store.List()
}

func (w *watchCache) WaitUntilFreshAndList(resourceVersion uint64, trace *util.Trace) ([]interface{}, uint64, error) {
	startTime := w.clock.Now()
	go func() {
		// Wake us up when the time limit has expired.  The docs
		// promise that time.After (well, NewTimer, which it calls)
		// will wait *at least* the duration given. Since this go
		// routine starts sometime after we record the start time, and
		// it will wake up the loop below sometime after the broadcast,
		// we don't need to worry about waking it up before the time
		// has expired accidentally.
		<-w.clock.After(MaximumListWait)
		w.cond.Broadcast()
	}()

	w.RLock()
	defer w.RUnlock()
	if trace != nil {
		trace.Step("watchCache locked acquired")
	}
	for w.resourceVersion < resourceVersion {
		if w.clock.Since(startTime) >= MaximumListWait {
			return nil, 0, fmt.Errorf("time limit exceeded while waiting for resource version %v (current value: %v)", resourceVersion, w.resourceVersion)
		}
		w.cond.Wait()
	}
	if trace != nil {
		trace.Step("watchCache fresh enough")
	}
	return w.store.List(), w.resourceVersion, nil
}

func (w *watchCache) ListKeys() []string {
	w.RLock()
	defer w.RUnlock()
	return w.store.ListKeys()
}

func (w *watchCache) Get(obj interface{}) (interface{}, bool, error) {
	w.RLock()
	defer w.RUnlock()
	return w.store.Get(obj)
}

func (w *watchCache) GetByKey(key string) (interface{}, bool, error) {
	w.RLock()
	defer w.RUnlock()
	return w.store.GetByKey(key)
}

func (w *watchCache) Replace(objs []interface{}, resourceVersion string) error {
	version, err := parseResourceVersion(resourceVersion)
	if err != nil {
		return err
	}

	w.Lock()
	defer w.Unlock()

	w.startIndex = 0
	w.endIndex = 0
	if err := w.store.Replace(objs, resourceVersion); err != nil {
		return err
	}
	w.resourceVersion = version
	if w.onReplace != nil {
		w.onReplace()
	}
	w.cond.Broadcast()
	return nil
}

func (w *watchCache) SetOnReplace(onReplace func()) {
	w.Lock()
	defer w.Unlock()
	w.onReplace = onReplace
}

func (w *watchCache) SetOnEvent(onEvent func(watchCacheEvent)) {
	w.Lock()
	defer w.Unlock()
	w.onEvent = onEvent
}

func (w *watchCache) GetAllEventsSinceThreadUnsafe(resourceVersion uint64) ([]watchCacheEvent, error) {
	size := w.endIndex - w.startIndex
	oldest := w.resourceVersion
	if size > 0 {
		oldest = w.cache[w.startIndex%w.capacity].resourceVersion
	}
	if resourceVersion == 0 {
		// resourceVersion = 0 means that we don't require any specific starting point
		// and we would like to start watching from ~now.
		// However, to keep backward compatibility, we additionally need to return the
		// current state and only then start watching from that point.
		//
		// TODO: In v2 api, we should stop returning the current state - #13969.
		allItems := w.store.List()
		result := make([]watchCacheEvent, len(allItems))
		for i, item := range allItems {
			result[i] = watchCacheEvent{Type: watch.Added, Object: item.(runtime.Object)}
		}
		return result, nil
	}
	if resourceVersion < oldest-1 {
		return nil, errors.NewGone(fmt.Sprintf("too old resource version: %d (%d)", resourceVersion, oldest-1))
	}

	// Binary search the smallest index at which resourceVersion is greater than the given one.
	f := func(i int) bool {
		return w.cache[(w.startIndex+i)%w.capacity].resourceVersion > resourceVersion
	}
	first := sort.Search(size, f)
	result := make([]watchCacheEvent, size-first)
	for i := 0; i < size-first; i++ {
		result[i] = w.cache[(w.startIndex+first+i)%w.capacity].watchCacheEvent
	}
	return result, nil
}

func (w *watchCache) GetAllEventsSince(resourceVersion uint64) ([]watchCacheEvent, error) {
	w.RLock()
	defer w.RUnlock()
	return w.GetAllEventsSinceThreadUnsafe(resourceVersion)
}

func (w *watchCache) Resync() error {
	// Nothing to do
	return nil
}
<|endoftext|>


<commit_before>package sansan

import (
	"fmt"
	"sync"
	"sync/atomic"
)

const heapsize = 30000

type program struct {
	code []byte
	heap []int32
}

func NewProg(c []byte) *program {
	return &program{
		c,
		make([]int32, heapsize),
	}
}

func (p program) Run() {
	var wg sync.WaitGroup
	defer wg.Wait()

	wg.Add(1)
	p.run(0, &wg)
}

func (p program) run(heapPos int, wg *sync.WaitGroup) {
	defer wg.Done()

	var childWG sync.WaitGroup
	defer childWG.Wait()

	for i := 0; i < len(p.code); i++ {
		switch ins := p.code[i]; ins {
		case '>':
			heapPos++
			heapPos = (heapPos%heapsize + heapsize) % heapsize
		case '<':
			heapPos--
			heapPos = (heapPos%heapsize + heapsize) % heapsize
		case '+':
			atomic.AddInt32(&p.heap[heapPos], 1)
		case '-':
			atomic.AddInt32(&p.heap[heapPos], -1)
		case '[':
			end := i + findClosing(p.code[i:])
			if atomic.LoadInt32(&p.heap[heapPos]) != 0 {
				// enter loop
				childWG.Add(1) // TODO: remove this on loops
				program{p.code[i+1:], p.heap}.run(heapPos, &childWG)
			}
			i = end // goto end
		case ']':
			if atomic.LoadInt32(&p.heap[heapPos]) == 0 {
				return
			}
			i = -1

		case '{':
			end := i + findClosing(p.code[i:])
			childWG.Add(1)
			go program{p.code[i+1:], p.heap}.run(heapPos, &childWG)

			i = end // continue parrent thread
		case '}':
			return // kill thread

		case '.':
			fmt.Printf("%d", atomic.LoadInt32(&p.heap[heapPos]))
		case ',':
			var n int32
			if _, err := fmt.Scanf("%d\n", &n); err != nil {
				panic(err)
			}
			atomic.SwapInt32(&p.heap[heapPos], n)
		case ' ', '\t', '\n':
		default:
			panic("instruction not implemented")
		}

	}
}

func findClosing(prog []byte) int {
	braces := 0
	for i := 0; i < len(prog); i++ {
		switch prog[i] {
		case '[', '{':
			braces++
		case ']', '}':
			braces--
			if braces < 0 {
				panic("invalid program: unbalanced braces")
			}
			if braces == 0 {
				return i
			}
		}
	}
	panic("invalid program: could not find closing ']'")
}
<commit_msg>fix heap pointer reset when exiting loops<commit_after>package sansan

import (
	"fmt"
	"sync"
	"sync/atomic"
)

const heapsize = 30000

type program struct {
	code []byte
	heap []int32
}

func NewProg(c []byte) *program {
	return &program{
		c,
		make([]int32, heapsize),
	}
}

func (p program) Run() {
	var wg sync.WaitGroup
	defer wg.Wait()

	wg.Add(1)
	p.run(0, &wg)
}

func (p program) run(heapPos int, wg *sync.WaitGroup) int {
	defer wg.Done()

	var childWG sync.WaitGroup
	defer childWG.Wait()

	for i := 0; i < len(p.code); i++ {
		switch ins := p.code[i]; ins {
		case '>':
			heapPos++
			heapPos = (heapPos%heapsize + heapsize) % heapsize
		case '<':
			heapPos--
			heapPos = (heapPos%heapsize + heapsize) % heapsize
		case '+':
			atomic.AddInt32(&p.heap[heapPos], 1)
		case '-':
			atomic.AddInt32(&p.heap[heapPos], -1)
		case '[':
			end := i + findClosing(p.code[i:])
			if atomic.LoadInt32(&p.heap[heapPos]) != 0 {
				// enter loop
				childWG.Add(1) // TODO: remove this on loops
				heapPos = program{p.code[i+1:], p.heap}.run(heapPos, &childWG)
			}
			i = end // goto end
		case ']':
			if atomic.LoadInt32(&p.heap[heapPos]) == 0 {
				return heapPos
			}
			i = -1

		case '{':
			end := i + findClosing(p.code[i:])
			childWG.Add(1)
			go program{p.code[i+1:], p.heap}.run(heapPos, &childWG)

			i = end // continue parrent thread
		case '}':
			return -1 // kill thread

		case '.':
			fmt.Printf("%c", atomic.LoadInt32(&p.heap[heapPos]))
		case ',':
			var n int32
			if _, err := fmt.Scanf("%d\n", &n); err != nil {
				panic(err)
			}
			atomic.SwapInt32(&p.heap[heapPos], n)
		case ' ', '\t', '\n':
		default:
			panic("instruction not implemented")
		}
	}
	return -1
}

func findClosing(prog []byte) int {
	braces := 0
	for i := 0; i < len(prog); i++ {
		switch prog[i] {
		case '[', '{':
			braces++
		case ']', '}':
			braces--
			if braces < 0 {
				panic("invalid program: unbalanced braces")
			}
			if braces == 0 {
				return i
			}
		}
	}
	panic("invalid program: could not find closing ']'")
}
<|endoftext|>


<commit_before>package main

import (
	"net/http"
	"strings"

	"github.com/PuerkitoBio/goquery"
)

func ScrapeWord(word string) *Entry {
	return Scrape("http://lema.rae.es/drae/srv/search?val="+Escape(word), word)
}

func Scrape(url string, word string) *Entry {
	r1, err := http.Get(url)

	if err != nil {
		panic(err)
	}

	r2 := Solve(r1)
	defer r2.Body.Close()

	doc, err := goquery.NewDocumentFromResponse(r2)
	if err != nil {
		panic(err)
	}

	//Look for definitions.
	nodes := doc.Find("body").Children().Filter("div")
	//If no definitions were found, there is probably a list of links to definitions.
	if nodes.Length() == 0 {
		//Choose the link for the word that is not a verb.
		doc.Find("li").EachWithBreak(func(i int, s *goquery.Selection) bool {
			url, _ = s.Find("a").Attr("href")
			if !strings.HasSuffix(s.Find("b").First().Text(), "r") {
				return false
			}
			return true
		})
		return Scrape("http://lema.rae.es/drae/srv/"+url, word)
	}
	etymology := strings.TrimSpace(nodes.First().Find("span.a").Text())
	defs := []*Definition{}
	vars := []*Variation{}

	nodes.Each(func(k int, s *goquery.Selection) {
		delimiter := s.Children().Filter("p:not([class])").Eq(1)
		delimiter.NextAll().EachWithBreak(func(i int, s *goquery.Selection) bool {
			//Skip empty P tags.
			if s.Children().Length() == 0 {
				return true
			}
			//Skip morfologies.
			if s.Has("a[title='MORFOLOGÍA.']").Length() > 0 {
				return true
			}
			//Break when the first variation is reached
			if s.HasClass("p") {
				return false
			}
			defs = append(defs, ScrapeDefinition(s))
			return true
		})

		delimiters := s.Find(".p span.k").Parent()
		delimiters.Each(func(i int, v *goquery.Selection) {
			vars = append(vars, &Variation{Variation: strings.TrimSpace(v.Text())})
			i = len(vars) - 1
			v.NextAll().EachWithBreak(func(j int, s *goquery.Selection) bool {
				//Done with this variation.
				if s.HasClass("p") {
					return false
				}
				vars[i].Definitions = append(vars[i].Definitions, ScrapeDefinition(s))
				return true
			})
		})
	})

	entry := &Entry{
		Word:        word,
		Etymology:   etymology,
		Definitions: defs,
		Variations:  vars,
	}

	return entry
}

func ScrapeDefinition(s *goquery.Selection) *Definition {
	category, _ := s.Find("span[title]").First().Attr("title")

	return &Definition{
		Category:   category,
		Definition: strings.TrimSpace(s.Find("span.b").Clone().Children().Not("a").Remove().End().End().Text()),
		Origin:     ScrapeOrigins(s),
		Notes:      ScrapeNotes(s),
		Examples:   ScrapeExamples(s),
	}
}

func ScrapeOrigins(s *goquery.Selection) []string {
	origins := []string{}
	s.Find("span.d i span.d[title]").Each(func(i int, s *goquery.Selection) {
		origin, _ := s.Attr("title")
		origins = append(origins, origin)
	})
	return origins
}

func ScrapeNotes(s *goquery.Selection) []string {
	notes := []string{}
	s.Clone().Find("span[title]").First().Remove().End().End().Find("span.d i span.d[title]").Remove().End().Find("span.d[title]").Each(func(i int, s *goquery.Selection) {
		note, _ := s.Attr("title")
		notes = append(notes, note)
	})
	return notes
}

func ScrapeExamples(s *goquery.Selection) []string {
	examples := []string{}
	s.Find("span.h i").Each(func(i int, s *goquery.Selection) {
		examples = append(examples, s.Text())
	})
	return examples
}
<commit_msg>Added another parameter to compare if a word is a verb.<commit_after>package main

import (
	"net/http"
	"strings"

	"github.com/PuerkitoBio/goquery"
)

func ScrapeWord(word string) *Entry {
	return Scrape("http://lema.rae.es/drae/srv/search?val="+Escape(word), word)
}

func Scrape(url string, word string) *Entry {
	r1, err := http.Get(url)

	if err != nil {
		panic(err)
	}

	r2 := Solve(r1)
	defer r2.Body.Close()

	doc, err := goquery.NewDocumentFromResponse(r2)
	if err != nil {
		panic(err)
	}

	//Look for definitions.
	nodes := doc.Find("body").Children().Filter("div")
	//If no definitions were found, there is probably a list of links to definitions.
	if nodes.Length() == 0 {
		//Choose the link for the word that is not a verb.
		doc.Find("li").EachWithBreak(func(i int, s *goquery.Selection) bool {
			url, _ = s.Find("a").Attr("href")
			if !strings.HasSuffix(s.Find("b").First().Text(), "r"
				|| !strings.HasSuffix(s.Find("b").First().Text(), "rse") {
				return false
			}
			return true
		})
		return Scrape("http://lema.rae.es/drae/srv/"+url, word)
	}
	etymology := strings.TrimSpace(nodes.First().Find("span.a").Text())
	defs := []*Definition{}
	vars := []*Variation{}

	nodes.Each(func(k int, s *goquery.Selection) {
		delimiter := s.Children().Filter("p:not([class])").Eq(1)
		delimiter.NextAll().EachWithBreak(func(i int, s *goquery.Selection) bool {
			//Skip empty P tags.
			if s.Children().Length() == 0 {
				return true
			}
			//Skip morfologies.
			if s.Has("a[title='MORFOLOGÍA.']").Length() > 0 {
				return true
			}
			//Break when the first variation is reached
			if s.HasClass("p") {
				return false
			}
			defs = append(defs, ScrapeDefinition(s))
			return true
		})

		delimiters := s.Find(".p span.k").Parent()
		delimiters.Each(func(i int, v *goquery.Selection) {
			vars = append(vars, &Variation{Variation: strings.TrimSpace(v.Text())})
			i = len(vars) - 1
			v.NextAll().EachWithBreak(func(j int, s *goquery.Selection) bool {
				//Done with this variation.
				if s.HasClass("p") {
					return false
				}
				vars[i].Definitions = append(vars[i].Definitions, ScrapeDefinition(s))
				return true
			})
		})
	})

	entry := &Entry{
		Word:        word,
		Etymology:   etymology,
		Definitions: defs,
		Variations:  vars,
	}

	return entry
}

func ScrapeDefinition(s *goquery.Selection) *Definition {
	category, _ := s.Find("span[title]").First().Attr("title")

	return &Definition{
		Category:   category,
		Definition: strings.TrimSpace(s.Find("span.b").Clone().Children().Not("a").Remove().End().End().Text()),
		Origin:     ScrapeOrigins(s),
		Notes:      ScrapeNotes(s),
		Examples:   ScrapeExamples(s),
	}
}

func ScrapeOrigins(s *goquery.Selection) []string {
	origins := []string{}
	s.Find("span.d i span.d[title]").Each(func(i int, s *goquery.Selection) {
		origin, _ := s.Attr("title")
		origins = append(origins, origin)
	})
	return origins
}

func ScrapeNotes(s *goquery.Selection) []string {
	notes := []string{}
	s.Clone().Find("span[title]").First().Remove().End().End().Find("span.d i span.d[title]").Remove().End().Find("span.d[title]").Each(func(i int, s *goquery.Selection) {
		note, _ := s.Attr("title")
		notes = append(notes, note)
	})
	return notes
}

func ScrapeExamples(s *goquery.Selection) []string {
	examples := []string{}
	s.Find("span.h i").Each(func(i int, s *goquery.Selection) {
		examples = append(examples, s.Text())
	})
	return examples
}
<|endoftext|>


<commit_before>package main

// GOPATH=`pwd` go build reverse-gol.go speed_packed.go ga.go board-standard.go transitions.go db.go && ./reverse-gol

import (
	"fmt"
	"time"
	"math/rand"
)

func main_orig() {
	l := NewBoardIterator(board_width, board_height)
	for i := 0; i < 65; i++ {
		l.Iterate(1)
		fmt.Print("\x0c", l) // Clear screen and print field.
		time.Sleep(time.Second / 30)
	}
}

func main_test_random() {
	for pct := float32(0.1); pct < 1.0; pct += 0.1 {
		l := NewBoardIterator(board_width, board_height)
		l.current.UniformRandom(pct + 0.01)

		//fmt.Print("\x0c", l)
		fmt.Print(l.current, "\n")
		l.Iterate(5) // 65 max reasonable steps for glider
		fmt.Print(l.current, "\n")
	}
}

func main_timer() {
	const glider = `
--X
X-X
-XX`

	start := time.Now()

	for iter := 0; iter < 1000; iter++ {
		l := NewBoardIterator(board_width, board_height)
		l.current.LoadString(glider[1:])

		l.Iterate(65)

		if iter == 0 {
			fmt.Print(l.current, "\n")
		}
	}

	elapsed := time.Since(start)
	fmt.Printf("1000 iterations took %s\n", elapsed)
}

func main_loader() {
	for pct := float32(0.1); pct < 1.0; pct += 0.1 {
		l := NewBoardIterator(board_width, board_height)
		l.current.UniformRandom(pct + 0.01)

		//fmt.Print("\x0c", l)
		fmt.Print(l.current, "\n")
		l.Iterate(5)
		fmt.Print(l.current, "\n")
	}
}

func main_verify_training_examples() {
	var kaggle LifeProblemSet

	problem_offset := 50

	id_list := []int{}
	for id := problem_offset; id < problem_offset+10; id++ {
		id_list = append(id_list, id)
	}
	kaggle.load_csv(true, id_list)
	//fmt.Println(kaggle.problem[107].start)
	//fmt.Println(kaggle.problem[107].end)

	image := NewImageSet(10, 11) // 10 rows of 11 images each, formatted 'appropriately'

	for _, id := range id_list {
		bs_start := NewBoardStats(board_width, board_height)
		kaggle.problem[id].start.AddToStats(bs_start)

		bs_end := NewBoardStats(board_width, board_height)
		kaggle.problem[id].end.AddToStats(bs_end)

		//image.DrawStats(r,c, bs)
		image.DrawStatsNext(bs_start)
		image.DrawStats(image.row_current, image.cols-1, bs_end)

		l := NewBoardIterator(board_width, board_height)
		l.current.CopyFrom(kaggle.problem[id].start)

		//fmt.Printf("Training Example[%d].steps=%d\n", id, steps)

		for i := 0; i < kaggle.problem[id].steps; i++ {
			l.Iterate(1) // Just 1 step per image for now

			bs_now := NewBoardStats(board_width, board_height)
			l.current.AddToStats(bs_now)
			image.DrawStatsNext(bs_now)
		}
		image.DrawStatsNext(bs_end) // For ease of comparison..

		if mismatch := l.current.CompareTo(kaggle.problem[id].end, nil); mismatch>0 {
			fmt.Printf("** Training Example[%d] FAIL - by %d\n", id, mismatch)
		}

		image.DrawStatsCRLF()
	}

	image.save("images/train.png")
}

func main_visualize_density() {
	image := NewImageSet(10, 11) // 10 rows of 11 images each, formatted 'appropriately'

	for pct:=float32(0.1); pct<0.99; pct+=0.1 {
		var bs []*BoardStats
		for j:=0; j<10; j++ {
			bs = append(bs, NewBoardStats(board_width, board_height))
		}

		for i:=0; i<1000; i++ {
			l := NewBoardIterator(board_width, board_height)
			l.current.UniformRandom(pct)
			l.current.AddToStats(bs[0])
			for j:=1; j<len(bs); j++ {
				l.Iterate(1)
				l.current.AddToStats(bs[j])
			}
		}
		for j:=0; j<len(bs); j++ {
			image.DrawStatsNext(bs[j])
		}
		image.DrawStatsCRLF()
	}

	image.save("images/density.png")
}

func main_population_score() {
	image := NewImageSet(10, 12) // 10 rows of 12 images each, formatted 'appropriately'

	var kaggle LifeProblemSet
	id := 58
	kaggle.load_csv(true, []int{id})

	problem := kaggle.problem[id]

	// Now ensure that the transition_collection is valid for this step size
	kaggle.load_transition_collection(problem.steps)

	// This is the TRUE starting place : for reference
	bs_start := NewBoardStats(board_width, board_height)
	problem.start.AddToStats(bs_start)

	// This is the TRUE ending place : for reference
	bs_end := NewBoardStats(board_width, board_height)
	problem.end.AddToStats(bs_end)

	// Create a population of potential boards
	pop_size := 1000
	pop := NewPopulation(pop_size, problem.steps, problem.end, &kaggle.transition_collection[problem.steps])
	for i:=0; i<pop_size; i++ {
		// Create a candidate starting point
		// NB:  We can only work from the problem.end
		pop.individual[i].start.CopyFrom(problem.end)
		//pop.individual[i].start.UniformRandom(0.32)
	}

	p_temp := NewPopulation(pop_size, problem.steps, problem.end, &kaggle.transition_collection[problem.steps])

	l := NewBoardIterator(board_width, board_height)

	iter_max := 1000
	for iter:=0; iter<iter_max; iter++ {
		disp_row := (0 == (iter) % (iter_max/10))

		if disp_row {
			// for ease of comparison
			image.DrawStatsNext(bs_start)
		}

		// Evaluate fitness of every individual in pop
		//for i:=0; i<pop_size; i++ {
		for i, individual := range pop.individual {
			l.current.CopyFrom(individual.start)

			//l.Iterate(5)

			mismatch_from_true_start:=999
			if i<5 && disp_row {
				diff     := NewBoard_BoolPacked(board_width, board_height)
				mismatch_from_true_start = l.current.CompareTo(problem.start, diff)

				bs_trial := NewBoardStats(board_width, board_height)
				l.current.AddToStats(bs_trial)
				//diff.AddToStats(bs_trial)

				//fmt.Printf("\n%3d.%2d : Mismatch from true start = %d\n", iter, i, mismatch_from_true_start)
				bs_trial.MisMatchBy(mismatch_from_true_start)

				image.DrawStatsNext(bs_trial)
			}

			l.Iterate(problem.steps)

			// This is 'allowed' since we know the end result, and can store the diff
			mismatch_from_true_end := l.current.CompareTo(problem.end, individual.diff)

			// This is a lower factor pressure, but good to have too
			count_on := individual.start.CompareTo(board_empty, nil)

			//individual.fitness = -mismatch_from_true_end
			//individual.fitness = -mismatch_from_true_end*4 -count_on*1
			individual.fitness = -mismatch_from_true_end*problem.steps -count_on*1

			if i<5 && disp_row {
				bs_result := NewBoardStats(board_width, board_height)
				//l.current.AddToStats(bs_result)
				individual.diff.AddToStats(bs_result)

				fmt.Printf("%4d.%3d : Mismatch vs true {start,end} = {%3d,%3d}\n", iter, i, mismatch_from_true_start, mismatch_from_true_end) // , individual.start
				bs_result.MisMatchBy(mismatch_from_true_end)

				image.DrawStatsNext(bs_result)
			}

			if mismatch_from_true_end==0 {
				//fmt.Printf("%4d.%3d : Mismatch vs true {start,end} = {%3d,%3d} :: PERFECTION!\n", iter, i, mismatch_from_true_start, mismatch_from_true_end)
			}
		}

		if disp_row {
			//image.DrawStatsNext(bs_end) // For ease of comparison..
			image.DrawStats(image.row_current, image.cols-1, bs_end)

			image.DrawStatsCRLF()
		}

		/*
		best_individual := pop.BestIndividual()
		fmt.Printf("%4d.best: Mismatch vs true {start,end} = {???,%3d}\n", iter, best_individual.fitness)
		fmt.Print(best_individual.start)

		if best_individual.fitness>=0 {
			// We have solved it, really
			//fmt.Print(best_individual.start)
			//break
		}
		*/

		p_temp.GenerationAfter(pop)
		pop, p_temp = p_temp, pop // Switcheroo to advance to next population
	}

	//image.DrawStats(image.row_current, image.cols-1, bs_end)

	image.save("images/score_mutated.png")
}

func main_create_stats(steps int) {
	var transitions TransitionCollectionMap

	transitions.TrainingCSV_to_stats("data/train.csv", steps)
	// No flipping stats (#steps, count_of_training_examples, unique end-point-patches_raw, unique end-point-patches_ud-or-lr, unique end-point-patches_ud*lr ):
	// 1  9866 648k 471k 454k
	// 2 10042 620k 450k 434k
	// 3  9947 589k 427k 411k
	// 4 10089 565k 410k 394k
	// 5  9956 534k 387k 374k

	transitions.SaveCSV(fmt.Sprintf(TransitionCollectionFileStrFmt, steps))
}

func main_create_stats_all() {
	for _,i := range( []int{1,2,3,4,5} ) {
		main_create_stats(i)
	}
	/*
[andrewsm@square reverse-gol]$ ls -l stats/
total 66520
-rw-rw-r--. 1 andrewsm andrewsm 12129025 Feb 27 01:09 transition-1.csv
-rw-rw-r--. 1 andrewsm andrewsm 13549146 Feb 27 01:09 transition-2.csv
-rw-rw-r--. 1 andrewsm andrewsm 14002885 Feb 27 01:09 transition-3.csv
-rw-rw-r--. 1 andrewsm andrewsm 14329277 Feb 27 01:10 transition-4.csv
-rw-rw-r--. 1 andrewsm andrewsm 14098291 Feb 27 01:10 transition-5.csv
	 */
}

func main_read_stats(steps int) {
	var transitions TransitionCollectionList

	transitions.LoadCSV(fmt.Sprintf(TransitionCollectionFileStrFmt, steps))
}

const currently_running_version int = 1002

func main() {
	//rand.Seed(time.Now().UnixNano())
	rand.Seed(1)

	//main_timer()
	//main_visualize_density()

	//main_verify_training_examples()

	//main_create_stats(1)
	//main_create_stats_all()
	//main_read_stats(1)

	//main_population_score()

	if false {
		//test_open_db()
		//return

		//create_list_of_problems_in_db() // NB: This sets up the 'problems' table to want answers...

		//probs := list_of_interesting_problems_from_db(1,5,true) // training
		//fmt.Println(probs)

		//reset_all_currently_processing(-1)

		// To force DB to solve the 2 training problems : 50 and 54 :
		// UPDATE problems SET solution_count=-2 WHERE id=-50 OR id=-54
		//steps:=5 // This is delta
		//problem_count_requested:=2
		//training_only := true

		steps:=4 // This is delta
		//problem_count_requested:=10000 // This may be truncated, if there are less available ids (some may be processing already)
		problem_count_requested:=2000 // This may be truncated, if there are less available ids (some may be processing already)
		training_only := false
		pick_problems_from_db_and_solve_them(steps, problem_count_requested, training_only)
	}

	if true {
		// create submission
		fname := fmt.Sprintf("data/submission_mdda_%s.csv", time.Now().Format("2006-01-02_03-04"))
		//fmt.Println(fname)
		create_submission(fname)
	}

	//fmt.Printf("Random #%3d\n", rand.Intn(1000))
}

<commit_msg>Set for relaunching twice at 2:30pm<commit_after>package main

// GOPATH=`pwd` go build reverse-gol.go speed_packed.go ga.go board-standard.go transitions.go db.go && ./reverse-gol

import (
	"fmt"
	"time"
	"math/rand"
)

func main_orig() {
	l := NewBoardIterator(board_width, board_height)
	for i := 0; i < 65; i++ {
		l.Iterate(1)
		fmt.Print("\x0c", l) // Clear screen and print field.
		time.Sleep(time.Second / 30)
	}
}

func main_test_random() {
	for pct := float32(0.1); pct < 1.0; pct += 0.1 {
		l := NewBoardIterator(board_width, board_height)
		l.current.UniformRandom(pct + 0.01)

		//fmt.Print("\x0c", l)
		fmt.Print(l.current, "\n")
		l.Iterate(5) // 65 max reasonable steps for glider
		fmt.Print(l.current, "\n")
	}
}

func main_timer() {
	const glider = `
--X
X-X
-XX`

	start := time.Now()

	for iter := 0; iter < 1000; iter++ {
		l := NewBoardIterator(board_width, board_height)
		l.current.LoadString(glider[1:])

		l.Iterate(65)

		if iter == 0 {
			fmt.Print(l.current, "\n")
		}
	}

	elapsed := time.Since(start)
	fmt.Printf("1000 iterations took %s\n", elapsed)
}

func main_loader() {
	for pct := float32(0.1); pct < 1.0; pct += 0.1 {
		l := NewBoardIterator(board_width, board_height)
		l.current.UniformRandom(pct + 0.01)

		//fmt.Print("\x0c", l)
		fmt.Print(l.current, "\n")
		l.Iterate(5)
		fmt.Print(l.current, "\n")
	}
}

func main_verify_training_examples() {
	var kaggle LifeProblemSet

	problem_offset := 50

	id_list := []int{}
	for id := problem_offset; id < problem_offset+10; id++ {
		id_list = append(id_list, id)
	}
	kaggle.load_csv(true, id_list)
	//fmt.Println(kaggle.problem[107].start)
	//fmt.Println(kaggle.problem[107].end)

	image := NewImageSet(10, 11) // 10 rows of 11 images each, formatted 'appropriately'

	for _, id := range id_list {
		bs_start := NewBoardStats(board_width, board_height)
		kaggle.problem[id].start.AddToStats(bs_start)

		bs_end := NewBoardStats(board_width, board_height)
		kaggle.problem[id].end.AddToStats(bs_end)

		//image.DrawStats(r,c, bs)
		image.DrawStatsNext(bs_start)
		image.DrawStats(image.row_current, image.cols-1, bs_end)

		l := NewBoardIterator(board_width, board_height)
		l.current.CopyFrom(kaggle.problem[id].start)

		//fmt.Printf("Training Example[%d].steps=%d\n", id, steps)

		for i := 0; i < kaggle.problem[id].steps; i++ {
			l.Iterate(1) // Just 1 step per image for now

			bs_now := NewBoardStats(board_width, board_height)
			l.current.AddToStats(bs_now)
			image.DrawStatsNext(bs_now)
		}
		image.DrawStatsNext(bs_end) // For ease of comparison..

		if mismatch := l.current.CompareTo(kaggle.problem[id].end, nil); mismatch>0 {
			fmt.Printf("** Training Example[%d] FAIL - by %d\n", id, mismatch)
		}

		image.DrawStatsCRLF()
	}

	image.save("images/train.png")
}

func main_visualize_density() {
	image := NewImageSet(10, 11) // 10 rows of 11 images each, formatted 'appropriately'

	for pct:=float32(0.1); pct<0.99; pct+=0.1 {
		var bs []*BoardStats
		for j:=0; j<10; j++ {
			bs = append(bs, NewBoardStats(board_width, board_height))
		}

		for i:=0; i<1000; i++ {
			l := NewBoardIterator(board_width, board_height)
			l.current.UniformRandom(pct)
			l.current.AddToStats(bs[0])
			for j:=1; j<len(bs); j++ {
				l.Iterate(1)
				l.current.AddToStats(bs[j])
			}
		}
		for j:=0; j<len(bs); j++ {
			image.DrawStatsNext(bs[j])
		}
		image.DrawStatsCRLF()
	}

	image.save("images/density.png")
}

func main_population_score() {
	image := NewImageSet(10, 12) // 10 rows of 12 images each, formatted 'appropriately'

	var kaggle LifeProblemSet
	id := 58
	kaggle.load_csv(true, []int{id})

	problem := kaggle.problem[id]

	// Now ensure that the transition_collection is valid for this step size
	kaggle.load_transition_collection(problem.steps)

	// This is the TRUE starting place : for reference
	bs_start := NewBoardStats(board_width, board_height)
	problem.start.AddToStats(bs_start)

	// This is the TRUE ending place : for reference
	bs_end := NewBoardStats(board_width, board_height)
	problem.end.AddToStats(bs_end)

	// Create a population of potential boards
	pop_size := 1000
	pop := NewPopulation(pop_size, problem.steps, problem.end, &kaggle.transition_collection[problem.steps])
	for i:=0; i<pop_size; i++ {
		// Create a candidate starting point
		// NB:  We can only work from the problem.end
		pop.individual[i].start.CopyFrom(problem.end)
		//pop.individual[i].start.UniformRandom(0.32)
	}

	p_temp := NewPopulation(pop_size, problem.steps, problem.end, &kaggle.transition_collection[problem.steps])

	l := NewBoardIterator(board_width, board_height)

	iter_max := 1000
	for iter:=0; iter<iter_max; iter++ {
		disp_row := (0 == (iter) % (iter_max/10))

		if disp_row {
			// for ease of comparison
			image.DrawStatsNext(bs_start)
		}

		// Evaluate fitness of every individual in pop
		//for i:=0; i<pop_size; i++ {
		for i, individual := range pop.individual {
			l.current.CopyFrom(individual.start)

			//l.Iterate(5)

			mismatch_from_true_start:=999
			if i<5 && disp_row {
				diff     := NewBoard_BoolPacked(board_width, board_height)
				mismatch_from_true_start = l.current.CompareTo(problem.start, diff)

				bs_trial := NewBoardStats(board_width, board_height)
				l.current.AddToStats(bs_trial)
				//diff.AddToStats(bs_trial)

				//fmt.Printf("\n%3d.%2d : Mismatch from true start = %d\n", iter, i, mismatch_from_true_start)
				bs_trial.MisMatchBy(mismatch_from_true_start)

				image.DrawStatsNext(bs_trial)
			}

			l.Iterate(problem.steps)

			// This is 'allowed' since we know the end result, and can store the diff
			mismatch_from_true_end := l.current.CompareTo(problem.end, individual.diff)

			// This is a lower factor pressure, but good to have too
			count_on := individual.start.CompareTo(board_empty, nil)

			//individual.fitness = -mismatch_from_true_end
			//individual.fitness = -mismatch_from_true_end*4 -count_on*1
			individual.fitness = -mismatch_from_true_end*problem.steps -count_on*1

			if i<5 && disp_row {
				bs_result := NewBoardStats(board_width, board_height)
				//l.current.AddToStats(bs_result)
				individual.diff.AddToStats(bs_result)

				fmt.Printf("%4d.%3d : Mismatch vs true {start,end} = {%3d,%3d}\n", iter, i, mismatch_from_true_start, mismatch_from_true_end) // , individual.start
				bs_result.MisMatchBy(mismatch_from_true_end)

				image.DrawStatsNext(bs_result)
			}

			if mismatch_from_true_end==0 {
				//fmt.Printf("%4d.%3d : Mismatch vs true {start,end} = {%3d,%3d} :: PERFECTION!\n", iter, i, mismatch_from_true_start, mismatch_from_true_end)
			}
		}

		if disp_row {
			//image.DrawStatsNext(bs_end) // For ease of comparison..
			image.DrawStats(image.row_current, image.cols-1, bs_end)

			image.DrawStatsCRLF()
		}

		/*
		best_individual := pop.BestIndividual()
		fmt.Printf("%4d.best: Mismatch vs true {start,end} = {???,%3d}\n", iter, best_individual.fitness)
		fmt.Print(best_individual.start)

		if best_individual.fitness>=0 {
			// We have solved it, really
			//fmt.Print(best_individual.start)
			//break
		}
		*/

		p_temp.GenerationAfter(pop)
		pop, p_temp = p_temp, pop // Switcheroo to advance to next population
	}

	//image.DrawStats(image.row_current, image.cols-1, bs_end)

	image.save("images/score_mutated.png")
}

func main_create_stats(steps int) {
	var transitions TransitionCollectionMap

	transitions.TrainingCSV_to_stats("data/train.csv", steps)
	// No flipping stats (#steps, count_of_training_examples, unique end-point-patches_raw, unique end-point-patches_ud-or-lr, unique end-point-patches_ud*lr ):
	// 1  9866 648k 471k 454k
	// 2 10042 620k 450k 434k
	// 3  9947 589k 427k 411k
	// 4 10089 565k 410k 394k
	// 5  9956 534k 387k 374k

	transitions.SaveCSV(fmt.Sprintf(TransitionCollectionFileStrFmt, steps))
}

func main_create_stats_all() {
	for _,i := range( []int{1,2,3,4,5} ) {
		main_create_stats(i)
	}
	/*
[andrewsm@square reverse-gol]$ ls -l stats/
total 66520
-rw-rw-r--. 1 andrewsm andrewsm 12129025 Feb 27 01:09 transition-1.csv
-rw-rw-r--. 1 andrewsm andrewsm 13549146 Feb 27 01:09 transition-2.csv
-rw-rw-r--. 1 andrewsm andrewsm 14002885 Feb 27 01:09 transition-3.csv
-rw-rw-r--. 1 andrewsm andrewsm 14329277 Feb 27 01:10 transition-4.csv
-rw-rw-r--. 1 andrewsm andrewsm 14098291 Feb 27 01:10 transition-5.csv
	 */
}

func main_read_stats(steps int) {
	var transitions TransitionCollectionList

	transitions.LoadCSV(fmt.Sprintf(TransitionCollectionFileStrFmt, steps))
}

const currently_running_version int = 1002

func main() {
	//rand.Seed(time.Now().UnixNano())
	rand.Seed(1)

	//main_timer()
	//main_visualize_density()

	//main_verify_training_examples()

	//main_create_stats(1)
	//main_create_stats_all()
	//main_read_stats(1)

	//main_population_score()

	if true {
		//test_open_db()
		//return

		//create_list_of_problems_in_db() // NB: This sets up the 'problems' table to want answers...

		//probs := list_of_interesting_problems_from_db(1,5,true) // training
		//fmt.Println(probs)

		//reset_all_currently_processing(-1)

		// To force DB to solve the 2 training problems : 50 and 54 :
		// UPDATE problems SET solution_count=-2 WHERE id=-50 OR id=-54
		//steps:=5 // This is delta
		//problem_count_requested:=2
		//training_only := true

		steps:=4 // This is delta
		//problem_count_requested:=10000 // This may be truncated, if there are less available ids (some may be processing already)
		problem_count_requested:=2000 // This may be truncated, if there are less available ids (some may be processing already)
		training_only := false
		pick_problems_from_db_and_solve_them(steps, problem_count_requested, training_only)
	}

	if false {
		// create submission
		fname := fmt.Sprintf("data/submission_mdda_%s.csv", time.Now().Format("2006-01-02_03-04"))
		//fmt.Println(fname)
		create_submission(fname)
	}

	//fmt.Printf("Random #%3d\n", rand.Intn(1000))
}

<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/HewlettPackard/oneview-golang/ov"
	"github.com/HewlettPackard/oneview-golang/utils"
	"os"
)

func main() {

	var (
		ClientOV       *ov.OVClient
		new_name       = "TestVolume"
		name_to_update = "UpdatedName"
	)

	ovc := ClientOV.NewOVClient(
		os.Getenv("ONEVIEW_OV_USER"),
		os.Getenv("ONEVIEW_OV_PASSWORD"),
		os.Getenv("ONEVIEW_OV_DOMAIN"),
		os.Getenv("ONEVIEW_OV_ENDPOINT"),
		false,
		500,
		"*")

	// Create storage volume with name <new_name>
	properties := &ov.Properties{new_name, utils.NewNstring("/rest/storage-pools/AAA05D5E-BDB5-4FBB-8E65-A8D400A6A8AF"), 107374741824, "Thin"}

	storageVolume := ov.StorageVolumeV3{TemplateURI: utils.NewNstring("/rest/storage-volume-templates/c93ef008-d8f0-40a5-b2d1-a8d400a6a8b7"), Properties: properties, IsPermanent: false}

	err := ovc.CreateStorageVolume(storageVolume)
	if err != nil {
		fmt.Println("Could not create the volume", err)
	}

	// Update the given storage volume
	update_vol, _ := ovc.GetStorageVolumeByName(new_name)

	updated_storage_volume := ov.StorageVolumeV3{
		ProvisioningTypeForUpdate: update_vol.ProvisioningTypeForUpdate,
		IsPermanent:               update_vol.IsPermanent,
		IsShareable:               update_vol.IsShareable,
		Name:                      name_to_update,
		ProvisionedCapacity:       "107374182400",
		DeviceSpecificAttributes:  update_vol.DeviceSpecificAttributes,
		URI:                       update_vol.URI,
		ETAG:                      update_vol.ETAG,
		Description:               "empty",
	}

	err = ovc.UpdateStorageVolume(updated_storage_volume)
	if err != nil {
		fmt.Println("Could not update the volume", err)
	}

	// Get All the volumes present
	fmt.Println("\nGetting all the storage volumes present in the system: \n")
	sort := "name:desc"
	vol_list, err := ovc.GetStorageVolumes("", sort)
	if err != nil {
		fmt.Println("Error Getting the storage volumes ", err)
	}
	for i := 0; i < len(vol_list.Members); i++ {
		fmt.Println(vol_list.Members[i].Name)
	}

	// Get volume by name
	fmt.Println("\nGetting details of volume with name: ", name_to_update)
	vol_by_name, _ := ovc.GetStorageVolumeByName(name_to_update)
	fmt.Println(vol_by_name)

	// Delete the created volume
	fmt.Println("\nDeleting the volume with name : UpdatedName")
	err = ovc.DeleteStorageVolume(name_to_update)
	if err != nil {
		fmt.Println("Delete Unsuccessful", err)
	}
}
<commit_msg>change variable name as per comments<commit_after>package main

import (
	"fmt"
	"github.com/HewlettPackard/oneview-golang/ov"
	"github.com/HewlettPackard/oneview-golang/utils"
	"os"
)

func main() {

	var (
		ClientOV       *ov.OVClient
		new_volume     = "TestVolume"
		name_to_update = "UpdatedName"
	)

	ovc := ClientOV.NewOVClient(
		os.Getenv("ONEVIEW_OV_USER"),
		os.Getenv("ONEVIEW_OV_PASSWORD"),
		os.Getenv("ONEVIEW_OV_DOMAIN"),
		os.Getenv("ONEVIEW_OV_ENDPOINT"),
		false,
		500,
		"*")

	// Create storage volume with name <new_name>
	properties := &ov.Properties{new_name, utils.NewNstring("/rest/storage-pools/AAA05D5E-BDB5-4FBB-8E65-A8D400A6A8AF"), 107374741824, "Thin"}

	storageVolume := ov.StorageVolumeV3{TemplateURI: utils.NewNstring("/rest/storage-volume-templates/c93ef008-d8f0-40a5-b2d1-a8d400a6a8b7"), Properties: properties, IsPermanent: false}

	err := ovc.CreateStorageVolume(storageVolume)
	if err != nil {
		fmt.Println("Could not create the volume", err)
	}

	// Update the given storage volume
	update_vol, _ := ovc.GetStorageVolumeByName(new_name)

	updated_storage_volume := ov.StorageVolumeV3{
		ProvisioningTypeForUpdate: update_vol.ProvisioningTypeForUpdate,
		IsPermanent:               update_vol.IsPermanent,
		IsShareable:               update_vol.IsShareable,
		Name:                      name_to_update,
		ProvisionedCapacity:       "107374182400",
		DeviceSpecificAttributes:  update_vol.DeviceSpecificAttributes,
		URI:                       update_vol.URI,
		ETAG:                      update_vol.ETAG,
		Description:               "empty",
	}

	err = ovc.UpdateStorageVolume(updated_storage_volume)
	if err != nil {
		fmt.Println("Could not update the volume", err)
	}

	// Get All the volumes present
	fmt.Println("\nGetting all the storage volumes present in the system: \n")
	sort := "name:desc"
	vol_list, err := ovc.GetStorageVolumes("", sort)
	if err != nil {
		fmt.Println("Error Getting the storage volumes ", err)
	}
	for i := 0; i < len(vol_list.Members); i++ {
		fmt.Println(vol_list.Members[i].Name)
	}

	// Get volume by name
	fmt.Println("\nGetting details of volume with name: ", name_to_update)
	vol_by_name, _ := ovc.GetStorageVolumeByName(name_to_update)
	fmt.Println(vol_by_name)

	// Delete the created volume
	fmt.Println("\nDeleting the volume with name : UpdatedName")
	err = ovc.DeleteStorageVolume(name_to_update)
	if err != nil {
		fmt.Println("Delete Unsuccessful", err)
	}
}
<|endoftext|>