<commit_before>package generate

var modelTpl = `package {{.PackageName}}

// Code auto-generated; DO NOT EDIT

import (
	"errors"
	{{ if .Generate "JoinSQL" -}}
	"fmt"
	{{- end }}

	"github.com/Nivl/go-rest-tools/network/http/httperr"
	"github.com/Nivl/go-rest-tools/storage/db"
	uuid "github.com/satori/go.uuid"
)

{{ if .Generate "JoinSQL" -}}
// Join{{.OptionalName}}SQL returns a string ready to be embed in a JOIN query
func Join{{.OptionalName}}SQL(prefix string) string {
	fields := []string{ {{.FieldsAsArray}} }
	output := ""

	for i, field := range fields {
		if i != 0 {
			output += ", "
		}

		fullName := fmt.Sprintf("%s.%s", prefix, field)
		output += fmt.Sprintf("%s \"%s\"", fullName, fullName)
	}

	return output
}
{{- end }}

{{ if .Generate "Get" -}}
// Get{{.OptionalName}}ByID finds and returns an active {{.ModelNameLC}} by ID
func Get{{.OptionalName}}ByID(q db.DB, id string) (*{{.ModelName}}, error) {
	{{.ModelVar}} := &{{.ModelName}}{}
	stmt := "SELECT * from {{.TableName}} WHERE id=$1 and deleted_at IS NULL LIMIT 1"
	err := db.Get(q, {{.ModelVar}}, stmt, id)
	// We want to return nil if a {{.ModelNameLC}} is not found
	if {{.ModelVar}}.ID == "" {
		return nil, err
	}
	return {{.ModelVar}}, err
}
{{- end }}

{{ if .Generate "Exists" -}}
// {{.OptionalName}}Exists checks if a {{.ModelNameLC}} exists for a specific ID
func {{.OptionalName}}Exists(q db.DB, id string) (bool, error) {
	exists := false
	stmt := "SELECT exists(SELECT 1 FROM {{.TableName}} WHERE id=$1 and deleted_at IS NULL)"
	err := db.Get(q, &exists, stmt, id)
	return exists, err
}
{{- end }}

{{ if .Generate "Save" -}}
// Save creates or updates the article depending on the value of the id using
// a transaction
func ({{.ModelVar}} *{{.ModelName}}) Save(q db.DB) error {
	if {{.ModelVar}}.ID == "" {
		return {{.ModelVar}}.Create(q)
	}

	return {{.ModelVar}}.Update(q)
}
{{- end }}

{{ if .Generate "Create" -}}
// Create persists a {{.ModelNameLC}} in the database
func ({{.ModelVar}} *{{.ModelName}}) Create(q db.DB) error {
	if {{.ModelVar}}.ID != "" {
		return errors.New("cannot persist a {{.ModelNameLC}} that already has an ID")
	}

	return {{.ModelVar}}.doCreate(q)
}
{{- end }}

{{ if .Generate "doCreate" -}}
// doCreate persists a {{.ModelNameLC}} in the database using a Node
func ({{.ModelVar}} *{{.ModelName}}) doCreate(q db.DB) error {
	if {{.ModelVar}} == nil {
		return errors.New("{{.ModelNameLC}} not instanced")
	}

	{{.ModelVar}}.ID = uuid.NewV4().String()
	{{.ModelVar}}.UpdatedAt = db.Now()
	if {{.ModelVar}}.CreatedAt == nil {
		{{.ModelVar}}.CreatedAt = db.Now()
	}

	stmt := "{{.CreateStmt}}"
	_, err := q.NamedExec(stmt, {{.ModelVar}})

  return httperr.NewFromSQL(err)
}
{{- end }}

{{ if .Generate "Update" -}}
// Update updates most of the fields of a persisted {{.ModelNameLC}}
// Excluded fields are id, created_at, deleted_at, etc.
func ({{.ModelVar}} *{{.ModelName}}) Update(q db.DB) error {
	if {{.ModelVar}}.ID == "" {
		return errors.New("cannot update a non-persisted {{.ModelNameLC}}")
	}

	return {{.ModelVar}}.doUpdate(q)
}
{{- end }}

{{ if .Generate "doUpdate" -}}
// doUpdate updates a {{.ModelNameLC}} in the database
func ({{.ModelVar}} *{{.ModelName}}) doUpdate(q db.DB) error {
	if {{.ModelVar}}.ID == "" {
		return errors.New("cannot update a non-persisted {{.ModelNameLC}}")
	}

	{{.ModelVar}}.UpdatedAt = db.Now()

	stmt := "{{.UpdateStmt}}"
	_, err := q.NamedExec(stmt, {{.ModelVar}})

	return httperr.NewFromSQL(err)
}
{{- end }}

{{ if .Generate "Delete" -}}
// Delete removes a {{.ModelNameLC}} from the database
func ({{.ModelVar}} *{{.ModelName}}) Delete(q db.DB) error {
	if {{.ModelVar}} == nil {
		return errors.New("{{.ModelNameLC}} not instanced")
	}

	if {{.ModelVar}}.ID == "" {
		return errors.New("{{.ModelNameLC}} has not been saved")
	}

	stmt := "DELETE FROM {{.TableName}} WHERE id=$1"
	_, err := q.Exec(stmt, {{.ModelVar}}.ID)

	return err
}
{{- end }}

{{ if .Generate "GetID" -}}
// GetID returns the ID field
func ({{.ModelVar}} *{{.ModelName}}) GetID() string {
	return {{.ModelVar}}.ID
}
{{- end }}

{{ if .Generate "SetID" -}}
// SetID sets the ID field
func ({{.ModelVar}} *{{.ModelName}}) SetID(id string) {
	{{.ModelVar}}.ID = id
}
{{- end }}

{{ if .Generate "IsZero" -}}
// IsZero checks if the object is either nil or don't have an ID
func ({{.ModelVar}} *{{.ModelName}}) IsZero() bool {
	return {{.ModelVar}} == nil || {{.ModelVar}}.ID == ""
}
{{- end }}`
<commit_msg>feat: Add GetAnyByID() to also retreive deleted resources<commit_after>package generate

var modelTpl = `package {{.PackageName}}

// Code auto-generated; DO NOT EDIT

import (
	"errors"
	{{ if .Generate "JoinSQL" -}}
	"fmt"
	{{- end }}

	"github.com/Nivl/go-rest-tools/network/http/httperr"
	"github.com/Nivl/go-rest-tools/storage/db"
	uuid "github.com/satori/go.uuid"
)

{{ if .Generate "JoinSQL" -}}
// Join{{.OptionalName}}SQL returns a string ready to be embed in a JOIN query
func Join{{.OptionalName}}SQL(prefix string) string {
	fields := []string{ {{.FieldsAsArray}} }
	output := ""

	for i, field := range fields {
		if i != 0 {
			output += ", "
		}

		fullName := fmt.Sprintf("%s.%s", prefix, field)
		output += fmt.Sprintf("%s \"%s\"", fullName, fullName)
	}

	return output
}
{{- end }}

{{ if .Generate "Get" -}}
// Get{{.OptionalName}}ByID finds and returns an active {{.ModelNameLC}} by ID
// Deleted object are not returned
func Get{{.OptionalName}}ByID(q db.DB, id string) (*{{.ModelName}}, error) {
	{{.ModelVar}} := &{{.ModelName}}{}
	stmt := "SELECT * from {{.TableName}} WHERE id=$1 and deleted_at IS NULL LIMIT 1"
	err := db.Get(q, {{.ModelVar}}, stmt, id)
	// We want to return nil if a {{.ModelNameLC}} is not found
	if {{.ModelVar}}.ID == "" {
		return nil, err
	}
	return {{.ModelVar}}, err
}
{{- end }}

{{ if .Generate "GetAny" -}}
// GetAny{{.OptionalName}}ByID finds and returns an {{.ModelNameLC}} by ID.
// Deleted object are returned
func GetAny{{.OptionalName}}ByID(q db.DB, id string) (*{{.ModelName}}, error) {
	{{.ModelVar}} := &{{.ModelName}}{}
	stmt := "SELECT * from {{.TableName}} WHERE id=$1 LIMIT 1"
	err := db.Get(q, {{.ModelVar}}, stmt, id)
	// We want to return nil if a {{.ModelNameLC}} is not found
	if {{.ModelVar}}.ID == "" {
		return nil, err
	}
	return {{.ModelVar}}, err
}
{{- end }}

{{ if .Generate "Exists" -}}
// {{.OptionalName}}Exists checks if a {{.ModelNameLC}} exists for a specific ID
func {{.OptionalName}}Exists(q db.DB, id string) (bool, error) {
	exists := false
	stmt := "SELECT exists(SELECT 1 FROM {{.TableName}} WHERE id=$1 and deleted_at IS NULL)"
	err := db.Get(q, &exists, stmt, id)
	return exists, err
}
{{- end }}

{{ if .Generate "Save" -}}
// Save creates or updates the article depending on the value of the id using
// a transaction
func ({{.ModelVar}} *{{.ModelName}}) Save(q db.DB) error {
	if {{.ModelVar}}.ID == "" {
		return {{.ModelVar}}.Create(q)
	}

	return {{.ModelVar}}.Update(q)
}
{{- end }}

{{ if .Generate "Create" -}}
// Create persists a {{.ModelNameLC}} in the database
func ({{.ModelVar}} *{{.ModelName}}) Create(q db.DB) error {
	if {{.ModelVar}}.ID != "" {
		return errors.New("cannot persist a {{.ModelNameLC}} that already has an ID")
	}

	return {{.ModelVar}}.doCreate(q)
}
{{- end }}

{{ if .Generate "doCreate" -}}
// doCreate persists a {{.ModelNameLC}} in the database using a Node
func ({{.ModelVar}} *{{.ModelName}}) doCreate(q db.DB) error {
	if {{.ModelVar}} == nil {
		return errors.New("{{.ModelNameLC}} not instanced")
	}

	{{.ModelVar}}.ID = uuid.NewV4().String()
	{{.ModelVar}}.UpdatedAt = db.Now()
	if {{.ModelVar}}.CreatedAt == nil {
		{{.ModelVar}}.CreatedAt = db.Now()
	}

	stmt := "{{.CreateStmt}}"
	_, err := q.NamedExec(stmt, {{.ModelVar}})

  return httperr.NewFromSQL(err)
}
{{- end }}

{{ if .Generate "Update" -}}
// Update updates most of the fields of a persisted {{.ModelNameLC}}
// Excluded fields are id, created_at, deleted_at, etc.
func ({{.ModelVar}} *{{.ModelName}}) Update(q db.DB) error {
	if {{.ModelVar}}.ID == "" {
		return errors.New("cannot update a non-persisted {{.ModelNameLC}}")
	}

	return {{.ModelVar}}.doUpdate(q)
}
{{- end }}

{{ if .Generate "doUpdate" -}}
// doUpdate updates a {{.ModelNameLC}} in the database
func ({{.ModelVar}} *{{.ModelName}}) doUpdate(q db.DB) error {
	if {{.ModelVar}}.ID == "" {
		return errors.New("cannot update a non-persisted {{.ModelNameLC}}")
	}

	{{.ModelVar}}.UpdatedAt = db.Now()

	stmt := "{{.UpdateStmt}}"
	_, err := q.NamedExec(stmt, {{.ModelVar}})

	return httperr.NewFromSQL(err)
}
{{- end }}

{{ if .Generate "Delete" -}}
// Delete removes a {{.ModelNameLC}} from the database
func ({{.ModelVar}} *{{.ModelName}}) Delete(q db.DB) error {
	if {{.ModelVar}} == nil {
		return errors.New("{{.ModelNameLC}} not instanced")
	}

	if {{.ModelVar}}.ID == "" {
		return errors.New("{{.ModelNameLC}} has not been saved")
	}

	stmt := "DELETE FROM {{.TableName}} WHERE id=$1"
	_, err := q.Exec(stmt, {{.ModelVar}}.ID)

	return err
}
{{- end }}

{{ if .Generate "GetID" -}}
// GetID returns the ID field
func ({{.ModelVar}} *{{.ModelName}}) GetID() string {
	return {{.ModelVar}}.ID
}
{{- end }}

{{ if .Generate "SetID" -}}
// SetID sets the ID field
func ({{.ModelVar}} *{{.ModelName}}) SetID(id string) {
	{{.ModelVar}}.ID = id
}
{{- end }}

{{ if .Generate "IsZero" -}}
// IsZero checks if the object is either nil or don't have an ID
func ({{.ModelVar}} *{{.ModelName}}) IsZero() bool {
	return {{.ModelVar}} == nil || {{.ModelVar}}.ID == ""
}
{{- end }}`
<|endoftext|>


<commit_before>package cassandra

import (
	"fmt"
	"math"
)

const QueryFmtRead = "SELECT ts, data FROM %s WHERE key IN ? AND ts < ?"
const QueryFmtWrite = "INSERT INTO %s (key, ts, data) values(?,?,?) USING TTL ?"

// TTLTables stores table definitions keyed by their TTL
type TTLTables map[uint32]Table

type Table struct {
	Name       string
	QueryRead  string
	QueryWrite string
	WindowSize uint32
	TTL        uint32
}

// GetTTLTables returns table definitions for the given specifications (ttls is in seconds)
func GetTTLTables(ttls []uint32, windowFactor int, nameFormat string) TTLTables {
	tables := make(TTLTables)
	for _, ttl := range ttls {
		tables[ttl] = GetTable(ttl, windowFactor, nameFormat)
	}
	return tables
}

func GetTable(ttl uint32, windowFactor int, nameFormat string) Table {
	/*
	 * The purpose of this is to bucket metrics of similar TTLs in well-tuned tables.
	 * We first calculate the preFactorWindow, which is the largest power of 2 that's
	 * smaller or equal to the TTL in hours.
	 * The table gets named after the preFactorWindow corresponding to the given TTL,
	 * assuring that we have a correctly tuned table for each given TTL.
	 * Then we divide the preFactorWindow by the windowFactor to obtain the window size
	 * in hours. This lets us control the amount of sstables used for each table/TTL.
	 * For example with a window factor of 20 we want to group the metrics like this:
	 *
	 * generated with: https://gist.github.com/replay/69ad7cfd523edfa552cd12851fa74c58
	 *
	 * +------------------------+---------------+---------------------+----------+
	 * |              TTL hours |    table_name | window_size (hours) | sstables |
	 * +------------------------+---------------+---------------------+----------+
	 * |         0 <= hours < 1 |     metrics_0 |                   1 |    0 - 2 |
	 * |         1 <= hours < 2 |     metrics_1 |                   1 |    1 - 3 |
	 * |         2 <= hours < 4 |     metrics_2 |                   1 |    2 - 5 |
	 * |         4 <= hours < 8 |     metrics_4 |                   1 |    4 - 9 |
	 * |        8 <= hours < 16 |     metrics_8 |                   1 |   8 - 17 |
	 * |       16 <= hours < 32 |    metrics_16 |                   1 |  16 - 33 |
	 * |       32 <= hours < 64 |    metrics_32 |                   2 |  16 - 33 |
	 * |      64 <= hours < 128 |    metrics_64 |                   4 |  16 - 33 |
	 * |     128 <= hours < 256 |   metrics_128 |                   7 |  19 - 38 |
	 * |     256 <= hours < 512 |   metrics_256 |                  13 |  20 - 41 |
	 * |    512 <= hours < 1024 |   metrics_512 |                  26 |  20 - 41 |
	 * |   1024 <= hours < 2048 |  metrics_1024 |                  52 |  20 - 41 |
	 * |   2048 <= hours < 4096 |  metrics_2048 |                 103 |  20 - 41 |
	 * |   4096 <= hours < 8192 |  metrics_4096 |                 205 |  20 - 41 |
	 * |  8192 <= hours < 16384 |  metrics_8192 |                 410 |  20 - 41 |
	 * | 16384 <= hours < 32768 | metrics_16384 |                 820 |  20 - 41 |
	 * | 32768 <= hours < 65536 | metrics_32768 |                1639 |  20 - 41 |
	 * +------------------------+---------------+---------------------+----------+
	 */

	// preFactorWindow is the largest power of 2 that's <= ttl
	// for example ttl of 1024 hours -> preFactorWindow of 1024,
	// but ttl of 1000 or 1023.99 hours -> prefactorWindow of 512
	preFactorWindow := uint32(math.Exp2(math.Floor(math.Log2(ttlUnits(ttl)))))
	tableName := fmt.Sprintf(nameFormat, preFactorWindow)
	windowSize := preFactorWindow/uint32(windowFactor) + 1
	return Table{
		Name:       tableName,
		QueryRead:  fmt.Sprintf(QueryFmtRead, tableName),
		QueryWrite: fmt.Sprintf(QueryFmtWrite, tableName),
		WindowSize: windowSize,
		TTL:        ttl,
	}
}

func ttlUnits(ttl uint32) float64 {
	// convert ttl to hours
	return float64(ttl) / (60 * 60)
}
<commit_msg>table format typo<commit_after>package cassandra

import (
	"fmt"
	"math"
)

const QueryFmtRead = "SELECT ts, data FROM %s WHERE key IN ? AND ts < ?"
const QueryFmtWrite = "INSERT INTO %s (key, ts, data) values(?,?,?) USING TTL ?"

// TTLTables stores table definitions keyed by their TTL
type TTLTables map[uint32]Table

type Table struct {
	Name       string
	QueryRead  string
	QueryWrite string
	WindowSize uint32
	TTL        uint32
}

// GetTTLTables returns table definitions for the given specifications (ttls is in seconds)
func GetTTLTables(ttls []uint32, windowFactor int, nameFormat string) TTLTables {
	tables := make(TTLTables)
	for _, ttl := range ttls {
		tables[ttl] = GetTable(ttl, windowFactor, nameFormat)
	}
	return tables
}

func GetTable(ttl uint32, windowFactor int, nameFormat string) Table {
	/*
	 * The purpose of this is to bucket metrics of similar TTLs in well-tuned tables.
	 * We first calculate the preFactorWindow, which is the largest power of 2 that's
	 * smaller or equal to the TTL in hours.
	 * The table gets named after the preFactorWindow corresponding to the given TTL,
	 * assuring that we have a correctly tuned table for each given TTL.
	 * Then we divide the preFactorWindow by the windowFactor to obtain the window size
	 * in hours. This lets us control the amount of sstables used for each table/TTL.
	 * For example with a window factor of 20 we want to group the metrics like this:
	 *
	 * generated with: https://gist.github.com/replay/69ad7cfd523edfa552cd12851fa74c58
	 *
	 * +------------------------+---------------+---------------------+----------+
	 * |              TTL hours |   table_name | window_size (hours) | sstables |
	 * +------------------------+---------------+---------------------+----------+
	 * |         0 <= hours < 1 |     metric_0 |                   1 |    0 - 2 |
	 * |         1 <= hours < 2 |     metric_1 |                   1 |    1 - 3 |
	 * |         2 <= hours < 4 |     metric_2 |                   1 |    2 - 5 |
	 * |         4 <= hours < 8 |     metric_4 |                   1 |    4 - 9 |
	 * |        8 <= hours < 16 |     metric_8 |                   1 |   8 - 17 |
	 * |       16 <= hours < 32 |    metric_16 |                   1 |  16 - 33 |
	 * |       32 <= hours < 64 |    metric_32 |                   2 |  16 - 33 |
	 * |      64 <= hours < 128 |    metric_64 |                   4 |  16 - 33 |
	 * |     128 <= hours < 256 |   metric_128 |                   7 |  19 - 38 |
	 * |     256 <= hours < 512 |   metric_256 |                  13 |  20 - 41 |
	 * |    512 <= hours < 1024 |   metric_512 |                  26 |  20 - 41 |
	 * |   1024 <= hours < 2048 |  metric_1024 |                  52 |  20 - 41 |
	 * |   2048 <= hours < 4096 |  metric_2048 |                 103 |  20 - 41 |
	 * |   4096 <= hours < 8192 |  metric_4096 |                 205 |  20 - 41 |
	 * |  8192 <= hours < 16384 |  metric_8192 |                 410 |  20 - 41 |
	 * | 16384 <= hours < 32768 | metric_16384 |                 820 |  20 - 41 |
	 * | 32768 <= hours < 65536 | metric_32768 |                1639 |  20 - 41 |
	 * +------------------------+---------------+---------------------+----------+
	 */

	// preFactorWindow is the largest power of 2 that's <= ttl
	// for example ttl of 1024 hours -> preFactorWindow of 1024,
	// but ttl of 1000 or 1023.99 hours -> prefactorWindow of 512
	preFactorWindow := uint32(math.Exp2(math.Floor(math.Log2(ttlUnits(ttl)))))
	tableName := fmt.Sprintf(nameFormat, preFactorWindow)
	windowSize := preFactorWindow/uint32(windowFactor) + 1
	return Table{
		Name:       tableName,
		QueryRead:  fmt.Sprintf(QueryFmtRead, tableName),
		QueryWrite: fmt.Sprintf(QueryFmtWrite, tableName),
		WindowSize: windowSize,
		TTL:        ttl,
	}
}

func ttlUnits(ttl uint32) float64 {
	// convert ttl to hours
	return float64(ttl) / (60 * 60)
}
<|endoftext|>


<commit_before>//Author: Gary Connelly
//date 22/09/2017

package main

import (
	"fmt"

)
//main methof

func main() {
	//sum:= 0;
	for i := 0; i < 100; i++ {
   	// sum += i

	 if i % 3 == 0 {
		fmt.Println("fizz")
	 } else if i % 5 == 0 {
		fmt.Println("buzz")
	 } else if i % 15 == 0  {
		fmt.Println("fizzbuzz")
	 } else {
		fmt.Println(i)
	 }//end if else

  }

}<commit_msg>clean up and comments<commit_after>//Author: Gary Connelly
//date 22/09/2017

//adapted from https://data-representation.github.io/notes/go.html

package main

import (
	"fmt"

)
//main methof

func main() {

	for i := 0; i < 100; i++ {

	// if else conditions for fizz and buzz
	 if i % 3 == 0 {
		fmt.Println("fizz")
	 } else if i % 5 == 0 {
		fmt.Println("buzz")
	 } else if i % 15 == 0  {
		fmt.Println("fizzbuzz")
	 } else {
		fmt.Println(i)
	 }//end if else

  }//end for

}//end main method<|endoftext|>


<commit_before>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package main

import (
	"fmt"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/google/syzkaller/pkg/build"
	"github.com/google/syzkaller/pkg/compiler"
	"github.com/google/syzkaller/pkg/osutil"
	"github.com/google/syzkaller/sys/targets"
)

type linux struct{}

func (*linux) prepare(sourcedir string, build bool, arches []*Arch) error {
	if build {
		// Run 'make mrproper', otherwise out-of-tree build fails.
		// However, it takes unreasonable amount of time,
		// so first check few files and if they are missing hope for best.
		for _, a := range arches {
			arch := a.target.KernelArch
			if osutil.IsExist(filepath.Join(sourcedir, ".config")) ||
				osutil.IsExist(filepath.Join(sourcedir, "init/main.o")) ||
				osutil.IsExist(filepath.Join(sourcedir, "include/config")) ||
				osutil.IsExist(filepath.Join(sourcedir, "include/generated/compile.h")) ||
				osutil.IsExist(filepath.Join(sourcedir, "arch", arch, "include", "generated")) {
				fmt.Printf("make mrproper ARCH=%v\n", arch)
				out, err := osutil.RunCmd(time.Hour, sourcedir, "make", "mrproper", "ARCH="+arch,
					"-j", fmt.Sprint(runtime.NumCPU()))
				if err != nil {
					return fmt.Errorf("make mrproper failed: %v\n%s", err, out)
				}
			}
		}
	} else {
		if len(arches) > 1 {
			return fmt.Errorf("more than 1 arch is invalid without -build")
		}
	}
	return nil
}

func (*linux) prepareArch(arch *Arch) error {
	// Kernel misses these headers on some arches.
	// So we create empty stubs in buildDir/syzkaller and add -IbuildDir/syzkaller
	// as the last flag so it won't override real kernel headers.
	for hdr, data := range map[string]string{
		// This is the only compiler header kernel uses,
		// need to provide it since we use -nostdinc below.
		"stdarg.h": `
#pragma once
#define va_list __builtin_va_list
#define va_start __builtin_va_start
#define va_end __builtin_va_end
#define va_arg __builtin_va_arg
#define va_copy __builtin_va_copy
#define __va_copy __builtin_va_copy
`,
		"asm/a.out.h":    "",
		"asm/prctl.h":    "",
		"asm/mce.h":      "",
		"uapi/asm/msr.h": "",
	} {
		fullPath := filepath.Join(arch.buildDir, "syzkaller", hdr)
		if err := osutil.MkdirAll(filepath.Dir(fullPath)); err != nil {
			return err
		}
		if err := osutil.WriteFile(fullPath, []byte(data)); err != nil {
			return nil
		}
	}
	if !arch.build {
		return nil
	}
	kernelDir := arch.sourceDir
	makeArgs := build.LinuxMakeArgs(arch.target, "", "", arch.buildDir)
	out, err := osutil.RunCmd(time.Hour, kernelDir, "make", append(makeArgs, "defconfig")...)
	if err != nil {
		return fmt.Errorf("make defconfig failed: %v\n%s", err, out)
	}
	_, err = osutil.RunCmd(time.Minute, arch.buildDir, filepath.Join(kernelDir, "scripts", "config"),
		// powerpc arch is configured to be big-endian by default, but we want little-endian powerpc.
		// Since all of our archs are little-endian for now, we just blindly switch it.
		"-d", "CPU_BIG_ENDIAN", "-e", "CPU_LITTLE_ENDIAN",
		// Without CONFIG_NETFILTER kernel does not build.
		"-e", "NETFILTER",
		// include/net/mptcp.h is the only header in kernel that guards some
		// of the consts with own config, so we need to enable CONFIG_MPTCP.
		"-e", "MPTCP",
		// security/smack/smack.h requires this to build.
		"-e", "SECURITY",
		"-e", "SECURITY_SMACK",
		// include/net/nl802154.h does not define some consts without this.
		"-e", "IEEE802154", "-e", "IEEE802154_NL802154_EXPERIMENTAL",
	)
	if err != nil {
		return err
	}
	out, err = osutil.RunCmd(time.Hour, kernelDir, "make", append(makeArgs, "olddefconfig")...)
	if err != nil {
		return fmt.Errorf("make olddefconfig failed: %v\n%s", err, out)
	}
	out, err = osutil.RunCmd(time.Hour, kernelDir, "make", append(makeArgs, "init/main.o")...)
	if err != nil {
		return fmt.Errorf("make failed: %v\n%s", err, out)
	}
	return nil
}

func (*linux) processFile(arch *Arch, info *compiler.ConstInfo) (map[string]uint64, map[string]bool, error) {
	if strings.HasSuffix(info.File, "_kvm.txt") &&
		(arch.target.Arch == targets.ARM || arch.target.Arch == targets.RiscV64) {
		// Hack: KVM is not supported on ARM anymore. We may want some more official support
		// for marking descriptions arch-specific, but so far this combination is the only
		// one. For riscv64, KVM is not supported yet but might be in the future.
		// Note: syz-sysgen also ignores this file for arm and riscv64.
		return nil, nil, nil
	}
	headerArch := arch.target.KernelHeaderArch
	sourceDir := arch.sourceDir
	buildDir := arch.buildDir
	args := []string{
		// This makes the build completely hermetic, only kernel headers are used.
		"-nostdinc",
		"-w", "-fmessage-length=0",
		"-O3", // required to get expected values for some __builtin_constant_p
		"-I.",
		"-D__KERNEL__",
		"-DKBUILD_MODNAME=\"-\"",
		"-I" + sourceDir + "/arch/" + headerArch + "/include",
		"-I" + buildDir + "/arch/" + headerArch + "/include/generated/uapi",
		"-I" + buildDir + "/arch/" + headerArch + "/include/generated",
		"-I" + sourceDir + "/arch/" + headerArch + "/include/asm/mach-malta",
		"-I" + sourceDir + "/arch/" + headerArch + "/include/asm/mach-generic",
		"-I" + buildDir + "/include",
		"-I" + sourceDir + "/include",
		"-I" + sourceDir + "/arch/" + headerArch + "/include/uapi",
		"-I" + buildDir + "/arch/" + headerArch + "/include/generated/uapi",
		"-I" + sourceDir + "/include/uapi",
		"-I" + buildDir + "/include/generated/uapi",
		"-I" + sourceDir,
		"-I" + sourceDir + "/include/linux",
		"-I" + buildDir + "/syzkaller",
		"-include", sourceDir + "/include/linux/kconfig.h",
	}
	args = append(args, arch.target.CFlags...)
	for _, incdir := range info.Incdirs {
		args = append(args, "-I"+sourceDir+"/"+incdir)
	}
	if arch.includeDirs != "" {
		for _, dir := range strings.Split(arch.includeDirs, ",") {
			args = append(args, "-I"+dir)
		}
	}
	params := &extractParams{
		AddSource:      "#include <asm/unistd.h>",
		ExtractFromELF: true,
		TargetEndian:   arch.target.HostEndian,
	}
	cc := arch.target.CCompiler
	res, undeclared, err := extract(info, cc, args, params)
	if err != nil {
		return nil, nil, err
	}
	if arch.target.PtrSize == 4 {
		// mmap syscall on i386/arm is translated to old_mmap and has different signature.
		// As a workaround fix it up to mmap2, which has signature that we expect.
		// pkg/csource has the same hack.
		const mmap = "__NR_mmap"
		const mmap2 = "__NR_mmap2"
		if res[mmap] != 0 || undeclared[mmap] {
			if res[mmap2] == 0 {
				return nil, nil, fmt.Errorf("%v is missing", mmap2)
			}
			res[mmap] = res[mmap2]
			delete(undeclared, mmap)
		}
	}
	return res, undeclared, nil
}
<commit_msg>sys/syz-extract: fix linux extract<commit_after>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package main

import (
	"fmt"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/google/syzkaller/pkg/build"
	"github.com/google/syzkaller/pkg/compiler"
	"github.com/google/syzkaller/pkg/osutil"
	"github.com/google/syzkaller/sys/targets"
)

type linux struct{}

func (*linux) prepare(sourcedir string, build bool, arches []*Arch) error {
	if build {
		// Run 'make mrproper', otherwise out-of-tree build fails.
		// However, it takes unreasonable amount of time,
		// so first check few files and if they are missing hope for best.
		for _, a := range arches {
			arch := a.target.KernelArch
			if osutil.IsExist(filepath.Join(sourcedir, ".config")) ||
				osutil.IsExist(filepath.Join(sourcedir, "init/main.o")) ||
				osutil.IsExist(filepath.Join(sourcedir, "include/config")) ||
				osutil.IsExist(filepath.Join(sourcedir, "include/generated/compile.h")) ||
				osutil.IsExist(filepath.Join(sourcedir, "arch", arch, "include", "generated")) {
				fmt.Printf("make mrproper ARCH=%v\n", arch)
				out, err := osutil.RunCmd(time.Hour, sourcedir, "make", "mrproper", "ARCH="+arch,
					"-j", fmt.Sprint(runtime.NumCPU()))
				if err != nil {
					return fmt.Errorf("make mrproper failed: %v\n%s", err, out)
				}
			}
		}
	} else {
		if len(arches) > 1 {
			return fmt.Errorf("more than 1 arch is invalid without -build")
		}
	}
	return nil
}

func (*linux) prepareArch(arch *Arch) error {
	// Kernel misses these headers on some arches.
	// So we create empty stubs in buildDir/syzkaller and add -IbuildDir/syzkaller
	// as the last flag so it won't override real kernel headers.
	for hdr, data := range map[string]string{
		// This is the only compiler header kernel uses,
		// need to provide it since we use -nostdinc below.
		"stdarg.h": `
#pragma once
#define va_list __builtin_va_list
#define va_start __builtin_va_start
#define va_end __builtin_va_end
#define va_arg __builtin_va_arg
#define va_copy __builtin_va_copy
#define __va_copy __builtin_va_copy
`,
		"asm/a.out.h":    "",
		"asm/prctl.h":    "",
		"asm/mce.h":      "",
		"asm/msr.h":      "",
		"uapi/asm/msr.h": "",
	} {
		fullPath := filepath.Join(arch.buildDir, "syzkaller", hdr)
		if err := osutil.MkdirAll(filepath.Dir(fullPath)); err != nil {
			return err
		}
		if err := osutil.WriteFile(fullPath, []byte(data)); err != nil {
			return nil
		}
	}
	if !arch.build {
		return nil
	}
	kernelDir := arch.sourceDir
	makeArgs := build.LinuxMakeArgs(arch.target, "", "", arch.buildDir)
	out, err := osutil.RunCmd(time.Hour, kernelDir, "make", append(makeArgs, "defconfig")...)
	if err != nil {
		return fmt.Errorf("make defconfig failed: %v\n%s", err, out)
	}
	_, err = osutil.RunCmd(time.Minute, arch.buildDir, filepath.Join(kernelDir, "scripts", "config"),
		// powerpc arch is configured to be big-endian by default, but we want little-endian powerpc.
		// Since all of our archs are little-endian for now, we just blindly switch it.
		"-d", "CPU_BIG_ENDIAN", "-e", "CPU_LITTLE_ENDIAN",
		// Without CONFIG_NETFILTER kernel does not build.
		"-e", "NETFILTER",
		// include/net/mptcp.h is the only header in kernel that guards some
		// of the consts with own config, so we need to enable CONFIG_MPTCP.
		"-e", "MPTCP",
		// security/smack/smack.h requires this to build.
		"-e", "SECURITY",
		"-e", "SECURITY_SMACK",
		// include/net/nl802154.h does not define some consts without this.
		"-e", "IEEE802154", "-e", "IEEE802154_NL802154_EXPERIMENTAL",
	)
	if err != nil {
		return err
	}
	out, err = osutil.RunCmd(time.Hour, kernelDir, "make", append(makeArgs, "olddefconfig")...)
	if err != nil {
		return fmt.Errorf("make olddefconfig failed: %v\n%s", err, out)
	}
	out, err = osutil.RunCmd(time.Hour, kernelDir, "make", append(makeArgs, "init/main.o")...)
	if err != nil {
		return fmt.Errorf("make failed: %v\n%s", err, out)
	}
	return nil
}

func (*linux) processFile(arch *Arch, info *compiler.ConstInfo) (map[string]uint64, map[string]bool, error) {
	if strings.HasSuffix(info.File, "_kvm.txt") &&
		(arch.target.Arch == targets.ARM || arch.target.Arch == targets.RiscV64) {
		// Hack: KVM is not supported on ARM anymore. We may want some more official support
		// for marking descriptions arch-specific, but so far this combination is the only
		// one. For riscv64, KVM is not supported yet but might be in the future.
		// Note: syz-sysgen also ignores this file for arm and riscv64.
		return nil, nil, nil
	}
	headerArch := arch.target.KernelHeaderArch
	sourceDir := arch.sourceDir
	buildDir := arch.buildDir
	args := []string{
		// This makes the build completely hermetic, only kernel headers are used.
		"-nostdinc",
		"-w", "-fmessage-length=0",
		"-O3", // required to get expected values for some __builtin_constant_p
		"-I.",
		"-D__KERNEL__",
		"-DKBUILD_MODNAME=\"-\"",
		"-I" + sourceDir + "/arch/" + headerArch + "/include",
		"-I" + buildDir + "/arch/" + headerArch + "/include/generated/uapi",
		"-I" + buildDir + "/arch/" + headerArch + "/include/generated",
		"-I" + sourceDir + "/arch/" + headerArch + "/include/asm/mach-malta",
		"-I" + sourceDir + "/arch/" + headerArch + "/include/asm/mach-generic",
		"-I" + buildDir + "/include",
		"-I" + sourceDir + "/include",
		"-I" + sourceDir + "/arch/" + headerArch + "/include/uapi",
		"-I" + buildDir + "/arch/" + headerArch + "/include/generated/uapi",
		"-I" + sourceDir + "/include/uapi",
		"-I" + buildDir + "/include/generated/uapi",
		"-I" + sourceDir,
		"-I" + sourceDir + "/include/linux",
		"-I" + buildDir + "/syzkaller",
		"-include", sourceDir + "/include/linux/kconfig.h",
	}
	args = append(args, arch.target.CFlags...)
	for _, incdir := range info.Incdirs {
		args = append(args, "-I"+sourceDir+"/"+incdir)
	}
	if arch.includeDirs != "" {
		for _, dir := range strings.Split(arch.includeDirs, ",") {
			args = append(args, "-I"+dir)
		}
	}
	params := &extractParams{
		AddSource:      "#include <asm/unistd.h>",
		ExtractFromELF: true,
		TargetEndian:   arch.target.HostEndian,
	}
	cc := arch.target.CCompiler
	res, undeclared, err := extract(info, cc, args, params)
	if err != nil {
		return nil, nil, err
	}
	if arch.target.PtrSize == 4 {
		// mmap syscall on i386/arm is translated to old_mmap and has different signature.
		// As a workaround fix it up to mmap2, which has signature that we expect.
		// pkg/csource has the same hack.
		const mmap = "__NR_mmap"
		const mmap2 = "__NR_mmap2"
		if res[mmap] != 0 || undeclared[mmap] {
			if res[mmap2] == 0 {
				return nil, nil, fmt.Errorf("%v is missing", mmap2)
			}
			res[mmap] = res[mmap2]
			delete(undeclared, mmap)
		}
	}
	return res, undeclared, nil
}
<|endoftext|>


<commit_before>package main

import (
	"testing"
	"time"
)

type request struct {
	id         string
	cacheable  bool
	alreadyRan bool
	run        func() (interface{}, error)
	setResult  func(interface{}, error)
}

func (r *request) ID() string {
	return r.id
}

func (r *request) Run() (interface{}, error) {
	if r.alreadyRan {
		panic("Run after Run or SetResult")
	}
	r.alreadyRan = true
	return r.run()
}

func (r *request) Cacheable() bool {
	return r.cacheable
}

func (r *request) SetResult(result interface{}, err error) {
	if r.alreadyRan {
		panic("SetResult after Run or SetResult")
	}
	r.alreadyRan = true
	r.setResult(result, err)
}

func okSetResult(result interface{}, err error) {}

func fatalRun(t *testing.T, msg string) func() (interface{}, error) {
	return func() (interface{}, error) {
		t.Fatal(msg)
		return nil, nil
	}
}

func fatalSetResult(t *testing.T, msg string) func(interface{}, error) {
	return func(_ interface{}, _ error) {
		t.Fatal(msg)
	}
}

func TestNewRequesterAndStop(t *testing.T) {
	var requester Requester = NewRequester(10, 10)
	defer requester.Stop()
	if requester == nil {
		t.Errorf("the returned requester is nil")
	}
}

func TestAddRequestRunsARequest(t *testing.T) {
	var requester = NewRequester(10, 10)
	defer requester.Stop()
	var ran = make(chan struct{})
	r := &request{
		id:        "a",
		cacheable: true,
		run: func() (interface{}, error) {
			close(ran)
			return "result1", nil
		},
		setResult: okSetResult,
	}
	requester.AddRequest(r)
	<-ran
}

func TestNonCacheableRequests(t *testing.T) {
	var requester = NewRequester(10, 10)
	defer requester.Stop()
	var expected1, expected2 = "foo", "bar"
	var setted = make(chan struct{})
	r := &request{
		id:        "a",
		cacheable: false,
		run: func() (interface{}, error) {
			return expected1, nil
		},
		setResult: nil,
	}
	requester.AddRequest(r)
	r = &request{
		id:        "a",
		cacheable: false,
		run: func() (interface{}, error) {
			defer close(setted)
			return expected2, nil
		},
		setResult: nil,
	}
	requester.AddRequest(r)
	<-setted
}
func TestStopWithQueueFromForum(t *testing.T) {
	t.Parallel()
	var (
		requester    = NewRequester(1000, 2)
		r1Started    = make(chan struct{})
		r1Continue   = make(chan struct{})
		r1Finished   = make(chan struct{})
		r2SetResult  = make(chan struct{})
		r4Started    = make(chan struct{})
		r4Continue   = make(chan struct{})
		r4Finished   = make(chan struct{})
		stopStarted  = make(chan struct{})
		stopFinished = make(chan struct{})
	)
	r1 := &request{ // Should Run
		id:        "a",
		cacheable: true,
		run: func() (interface{}, error) {
			close(r1Started)
			<-r1Continue
			defer close(r1Finished)
			return "result1", nil
		},
		setResult: fatalSetResult(t, "r4 should be runned not SetResulted"),
	}

	r2 := &request{
		id:        "a",
		cacheable: true,
		run:       fatalRun(t, "r3 should not be Ran as it should be cached"),
		setResult: func(_ interface{}, _ error) {
			close(r2SetResult)
		},
	}

	r3 := &request{
		id:        "a",
		cacheable: false,
		run:       fatalRun(t, "r3 should not be Ran as it should be cached"),
		setResult: fatalSetResult(t, "r3 should not be set resulted after Stop"),
	}

	r4 := &request{
		id:        "b",
		cacheable: false,
		run: func() (interface{}, error) {
			close(r4Started)
			<-r4Continue
			defer close(r4Finished)
			return "result4", nil
		},
		setResult: fatalSetResult(t, "r4 should be runned not SetResulted"),
	}

	r5 := &request{
		id:        "b",
		cacheable: false,
		run:       fatalRun(t, "r5 should've not run as it should be cached"),
		setResult: fatalSetResult(t, "r5 should've not been setResulted as it's non cacheable"),
	}

	r6 := &request{
		id:        "c",
		cacheable: true,
		run:       fatalRun(t, "r6 should not be Ran after Stop"),
		setResult: fatalSetResult(t, "r6 should not be SetResulted,  after Stop"),
	}

	r7 := &request{
		id:        "d",
		cacheable: true,
		run:       fatalRun(t, "r7 should not be Run after Stop"),
		setResult: fatalSetResult(t, "r7 should not be SetResulted after Stop"),
	}

	go requester.AddRequest(r1)
	go requester.AddRequest(r4)
	<-r1Started
	<-r4Started
	go requester.AddRequest(r2)
	go requester.AddRequest(r5)
	time.Sleep(100 * time.Millisecond)
	go func() {
		close(stopStarted)
		requester.Stop()
		close(stopFinished)
	}()
	<-stopStarted
	time.Sleep(100 * time.Millisecond)
	go requester.AddRequest(r6)
	go requester.AddRequest(r3)
	time.Sleep(100 * time.Millisecond)
	close(r1Continue)
	close(r4Continue)
	<-r1Finished
	<-r4Finished
	<-r2SetResult
	go requester.AddRequest(r7)
	time.Sleep(200 * time.Millisecond)
}
<commit_msg>task04: fix the messages for TestStopWithQueueFromForum<commit_after>package main

import (
	"testing"
	"time"
)

type request struct {
	id         string
	cacheable  bool
	alreadyRan bool
	run        func() (interface{}, error)
	setResult  func(interface{}, error)
}

func (r *request) ID() string {
	return r.id
}

func (r *request) Run() (interface{}, error) {
	if r.alreadyRan {
		panic("Run after Run or SetResult")
	}
	r.alreadyRan = true
	return r.run()
}

func (r *request) Cacheable() bool {
	return r.cacheable
}

func (r *request) SetResult(result interface{}, err error) {
	if r.alreadyRan {
		panic("SetResult after Run or SetResult")
	}
	r.alreadyRan = true
	r.setResult(result, err)
}

func okSetResult(result interface{}, err error) {}

func fatalRun(t *testing.T, msg string) func() (interface{}, error) {
	return func() (interface{}, error) {
		t.Fatal(msg)
		return nil, nil
	}
}

func fatalSetResult(t *testing.T, msg string) func(interface{}, error) {
	return func(_ interface{}, _ error) {
		t.Fatal(msg)
	}
}

func TestNewRequesterAndStop(t *testing.T) {
	var requester Requester = NewRequester(10, 10)
	defer requester.Stop()
	if requester == nil {
		t.Errorf("the returned requester is nil")
	}
}

func TestAddRequestRunsARequest(t *testing.T) {
	var requester = NewRequester(10, 10)
	defer requester.Stop()
	var ran = make(chan struct{})
	r := &request{
		id:        "a",
		cacheable: true,
		run: func() (interface{}, error) {
			close(ran)
			return "result1", nil
		},
		setResult: okSetResult,
	}
	requester.AddRequest(r)
	<-ran
}

func TestNonCacheableRequests(t *testing.T) {
	var requester = NewRequester(10, 10)
	defer requester.Stop()
	var expected1, expected2 = "foo", "bar"
	var setted = make(chan struct{})
	r := &request{
		id:        "a",
		cacheable: false,
		run: func() (interface{}, error) {
			return expected1, nil
		},
		setResult: nil,
	}
	requester.AddRequest(r)
	r = &request{
		id:        "a",
		cacheable: false,
		run: func() (interface{}, error) {
			defer close(setted)
			return expected2, nil
		},
		setResult: nil,
	}
	requester.AddRequest(r)
	<-setted
}

func TestStopWithQueueFromForum(t *testing.T) {
	t.Parallel()
	var (
		requester    = NewRequester(1000, 2)
		r1Started    = make(chan struct{})
		r1Continue   = make(chan struct{})
		r1Finished   = make(chan struct{})
		r2SetResult  = make(chan struct{})
		r4Started    = make(chan struct{})
		r4Continue   = make(chan struct{})
		r4Finished   = make(chan struct{})
		stopStarted  = make(chan struct{})
		stopFinished = make(chan struct{})
	)
	r1 := &request{ // Should Run
		id:        "a",
		cacheable: true,
		run: func() (interface{}, error) {
			close(r1Started)
			<-r1Continue
			defer close(r1Finished)
			return "result1", nil
		},
		setResult: fatalSetResult(t, "r4 should be runned not SetResulted"),
	}

	r2 := &request{
		id:        "a",
		cacheable: true,
		run:       fatalRun(t, "r2 should not be runned as it should be cached"),
		setResult: func(_ interface{}, _ error) {
			close(r2SetResult)
		},
	}

	r3 := &request{
		id:        "a",
		cacheable: false,
		run:       fatalRun(t, "r3 should not be runned as it should be cached"),
		setResult: fatalSetResult(t, "r3 should not be SetResulterd, as it was added after Stop"),
	}

	r4 := &request{
		id:        "b",
		cacheable: false,
		run: func() (interface{}, error) {
			close(r4Started)
			<-r4Continue
			defer close(r4Finished)
			return "result4", nil
		},
		setResult: fatalSetResult(t, "r4 should be runned not SetResulted"),
	}

	r5 := &request{
		id:        "b",
		cacheable: false,
		run:       fatalRun(t, "r5 should've not Run after Stop"),
		setResult: fatalSetResult(t, "r5 should've not been SetResulted as it's non cacheable"),
	}

	r6 := &request{
		id:        "c",
		cacheable: true,
		run:       fatalRun(t, "r6 should not be Run after Stop"),
		setResult: fatalSetResult(t, "r6 should not be SetResulted,  after Stop"),
	}

	r7 := &request{
		id:        "d",
		cacheable: true,
		run:       fatalRun(t, "r7 should not be Run after Stop"),
		setResult: fatalSetResult(t, "r7 should not be SetResulted, after Stop"),
	}

	go requester.AddRequest(r1)
	go requester.AddRequest(r4)
	<-r1Started
	<-r4Started
	go requester.AddRequest(r2)
	go requester.AddRequest(r5)
	time.Sleep(100 * time.Millisecond)
	go func() {
		close(stopStarted)
		requester.Stop()
		close(stopFinished)
	}()
	<-stopStarted
	time.Sleep(100 * time.Millisecond)
	go requester.AddRequest(r6)
	go requester.AddRequest(r3)
	time.Sleep(100 * time.Millisecond)
	close(r1Continue)
	close(r4Continue)
	<-r1Finished
	<-r4Finished
	<-r2SetResult
	go requester.AddRequest(r7)
	time.Sleep(200 * time.Millisecond)
}
<|endoftext|>


<commit_before>/*
Copyright 2017 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package jsonnet

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
)

type ImportedData struct {
	err       error
	foundHere string
	content   string
}

type Importer interface {
	Import(codeDir string, importedPath string) *ImportedData
}

type ImportCacheValue struct {
	data *ImportedData

	// nil if we have only imported it via importstr
	asCode potentialValue
}

type importCacheKey struct {
	dir          string
	importedPath string
}

type importCacheMap map[importCacheKey]ImportCacheValue

type ImportCache struct {
	cache    importCacheMap
	importer Importer
}

func MakeImportCache(importer Importer) *ImportCache {
	return &ImportCache{importer: importer, cache: make(importCacheMap)}
}

func (cache *ImportCache) importData(key importCacheKey) *ImportCacheValue {
	if value, ok := cache.cache[key]; ok {
		return &value
	}
	data := cache.importer.Import(key.dir, key.importedPath)
	val := ImportCacheValue{
		data: data,
	}
	cache.cache[key] = val
	return &val
}

func (cache *ImportCache) ImportString(codeDir, importedPath string, e *evaluator) (*valueString, error) {
	data := cache.importData(importCacheKey{codeDir, importedPath})
	if data.data.err != nil {
		return nil, e.Error(data.data.err.Error())
	}
	return makeValueString(data.data.content), nil
}

func codeToPV(e *evaluator, filename string, code string) potentialValue {
	node, err := snippetToAST(filename, code)
	if err != nil {
		// TODO(sbarzowski) we should wrap (static) error here
		// within a RuntimeError. Because whether we get this error or not
		// actually depends on what happens in Runtime (whether import gets
		// evaluated).
		// The same thinking applies to external variables.
		return makeErrorThunk(err)
	}
	return makeThunk(makeInitialEnv(filename, e.i.baseStd), node)
}

func (cache *ImportCache) ImportCode(codeDir, importedPath string, e *evaluator) (value, error) {
	cached := cache.importData(importCacheKey{codeDir, importedPath})
	if cached.data.err != nil {
		return nil, e.Error(cached.data.err.Error())
	}
	if cached.asCode == nil {
		cached.asCode = codeToPV(e, cached.data.foundHere, cached.data.content)
	}
	return e.evaluate(cached.asCode)
}

// Concrete importers
// -------------------------------------

type FileImporter struct {
	// TODO(sbarzowski) fill it in
	JPaths []string
}

func tryPath(dir, importedPath string) (found bool, content []byte, foundHere string, err error) {
	var absPath string
	if path.IsAbs(importedPath) {
		absPath = importedPath
	} else {
		absPath = path.Join(dir, importedPath)
	}
	content, err = ioutil.ReadFile(absPath)
	if os.IsNotExist(err) {
		return false, nil, "", nil
	}
	return true, content, absPath, err
}

func (importer *FileImporter) Import(dir, importedPath string) *ImportedData {
	found, content, foundHere, err := tryPath(dir, importedPath)
	if err != nil {
		return &ImportedData{err: err}
	}

	for i := len(importer.JPaths) - 1; !found && i >= 0; i-- {
		found, content, foundHere, err = tryPath(importer.JPaths[i], importedPath)
		if err != nil {
			return &ImportedData{err: err}
		}
	}

	if !found {
		return &ImportedData{
			err: fmt.Errorf("Couldn't open import %#v: No match locally or in the Jsonnet library paths.", importedPath),
		}
	}
	return &ImportedData{content: string(content), foundHere: foundHere}
}

type MemoryImporter struct {
	Data map[string]string
}

func (importer *MemoryImporter) Import(dir, importedPath string) *ImportedData {
	if content, ok := importer.Data[importedPath]; ok {
		return &ImportedData{content: content, foundHere: importedPath}
	}
	return &ImportedData{err: fmt.Errorf("Import not available %v", importedPath)}
}
<commit_msg>Make fields of ImportedData public<commit_after>/*
Copyright 2017 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package jsonnet

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
)

type ImportedData struct {
	Err       error
	FoundHere string
	Content   string
}

type Importer interface {
	Import(codeDir string, importedPath string) *ImportedData
}

type ImportCacheValue struct {
	data *ImportedData

	// nil if we have only imported it via importstr
	asCode potentialValue
}

type importCacheKey struct {
	dir          string
	importedPath string
}

type importCacheMap map[importCacheKey]ImportCacheValue

type ImportCache struct {
	cache    importCacheMap
	importer Importer
}

func MakeImportCache(importer Importer) *ImportCache {
	return &ImportCache{importer: importer, cache: make(importCacheMap)}
}

func (cache *ImportCache) importData(key importCacheKey) *ImportCacheValue {
	if value, ok := cache.cache[key]; ok {
		return &value
	}
	data := cache.importer.Import(key.dir, key.importedPath)
	val := ImportCacheValue{
		data: data,
	}
	cache.cache[key] = val
	return &val
}

func (cache *ImportCache) ImportString(codeDir, importedPath string, e *evaluator) (*valueString, error) {
	data := cache.importData(importCacheKey{codeDir, importedPath})
	if data.data.Err != nil {
		return nil, e.Error(data.data.Err.Error())
	}
	return makeValueString(data.data.Content), nil
}

func codeToPV(e *evaluator, filename string, code string) potentialValue {
	node, err := snippetToAST(filename, code)
	if err != nil {
		// TODO(sbarzowski) we should wrap (static) error here
		// within a RuntimeError. Because whether we get this error or not
		// actually depends on what happens in Runtime (whether import gets
		// evaluated).
		// The same thinking applies to external variables.
		return makeErrorThunk(err)
	}
	return makeThunk(makeInitialEnv(filename, e.i.baseStd), node)
}

func (cache *ImportCache) ImportCode(codeDir, importedPath string, e *evaluator) (value, error) {
	cached := cache.importData(importCacheKey{codeDir, importedPath})
	if cached.data.Err != nil {
		return nil, e.Error(cached.data.Err.Error())
	}
	if cached.asCode == nil {
		cached.asCode = codeToPV(e, cached.data.FoundHere, cached.data.Content)
	}
	return e.evaluate(cached.asCode)
}

// Concrete importers
// -------------------------------------

type FileImporter struct {
	// TODO(sbarzowski) fill it in
	JPaths []string
}

func tryPath(dir, importedPath string) (found bool, content []byte, foundHere string, err error) {
	var absPath string
	if path.IsAbs(importedPath) {
		absPath = importedPath
	} else {
		absPath = path.Join(dir, importedPath)
	}
	content, err = ioutil.ReadFile(absPath)
	if os.IsNotExist(err) {
		return false, nil, "", nil
	}
	return true, content, absPath, err
}

func (importer *FileImporter) Import(dir, importedPath string) *ImportedData {
	found, content, foundHere, err := tryPath(dir, importedPath)
	if err != nil {
		return &ImportedData{Err: err}
	}

	for i := len(importer.JPaths) - 1; !found && i >= 0; i-- {
		found, content, foundHere, err = tryPath(importer.JPaths[i], importedPath)
		if err != nil {
			return &ImportedData{Err: err}
		}
	}

	if !found {
		return &ImportedData{
			Err: fmt.Errorf("Couldn't open import %#v: No match locally or in the Jsonnet library paths.", importedPath),
		}
	}
	return &ImportedData{Content: string(content), FoundHere: foundHere}
}

type MemoryImporter struct {
	Data map[string]string
}

func (importer *MemoryImporter) Import(dir, importedPath string) *ImportedData {
	if content, ok := importer.Data[importedPath]; ok {
		return &ImportedData{Content: content, FoundHere: importedPath}
	}
	return &ImportedData{Err: fmt.Errorf("Import not available %v", importedPath)}
}
<|endoftext|>


<commit_before>package imaging

import (
	"bytes"
	"io"
	"math"
	"os"
	"time"

	// this is the only outside pixur package dependency.  Avoid depending too much on schema.
	"pixur.org/pixur/be/status"

	// unfortuantely, most of my machines are stuck on 6.8.x.x
	// TODO: update this to v2 when possible.
	"gopkg.in/gographics/imagick.v1/imagick"
)

const (
	// WEBM magic header
	ebmlHeader = "\x1a\x45\xdf\xa3"
)

const (
	DefaultGifFormat  ImageFormat = "GIF"
	DefaultJpegFormat ImageFormat = "JPEG"
	DefaultPngFormat  ImageFormat = "PNG"
)

const (
	thumbnailSquareSize = 160
)

const gifTicksPerSecond = 100

func init() {
	imagick.Initialize()
	// never call Terminate
}

// a list of all formats: mw.QueryFormats("*")

// ImageFormat is the string format of the image
type ImageFormat string

// IsGif returns true if the type of this image is a GIF.
func (f ImageFormat) IsGif() bool {
	return f == "GIF" || f == "GIF87"
}

// IsJpeg returns true if the type of this image is a JPEG.
func (f ImageFormat) IsJpeg() bool {
	return f == "JPE" || f == "JPEG" || f == "JPG"
}

// IsPng returns true if the type of this image is a PNG.
func (f ImageFormat) IsPng() bool {
	// These are the types imagick says it supports: PNG PNG00 PNG24 PNG32 PNG48 PNG64 PNG8.  I am
	// only including the ones I know.
	return f == "PNG" || f == "PNG24" || f == "PNG32" || f == "PNG8"
}

type PixurImage2 interface {
	Format() ImageFormat
	Dimensions() (width, height uint, sts status.S)
	// In the future, this could also include a histogram
	Duration() (*time.Duration, status.S)

	Thumbnail() (PixurImage2, status.S)

	Write(io.Writer) status.S

	Close()
}

var _ PixurImage2 = (*pixurImage2)(nil)

type pixurImage2 struct {
	mw *imagick.MagickWand
}

func (pi *pixurImage2) Write(w io.Writer) status.S {

	switch w := w.(type) {
	case *os.File:
		if err := pi.mw.WriteImageFile(w); err != nil {
			return status.InternalError(err, "can't write image file")
		}
	default:
		panic("uh oh")
	}

	return nil
}

func (pi *pixurImage2) Thumbnail() (PixurImage2, status.S) {
	w, h, sts := pi.Dimensions()
	if sts != nil {
		return nil, sts
	}
	var neww, newh uint
	var x, y int
	if w > h {
		neww = h
		newh = h
		x = int((w - neww) / 2)
		y = 0
	} else {
		neww = w
		newh = h
		x = 0
		y = int((h - newh) / 2)
	}
	newmw := pi.mw.Clone()
	newmw.ResetIterator()
	destroy := true
	defer func() {
		if destroy {
			newmw.Destroy()
		}
	}()

	if err := newmw.CropImage(newh, neww, x, y); err != nil {
		return nil, status.InternalError(err, "unable to crop thumbnail")
	}

	side := uint(thumbnailSquareSize)
	newmw.TransformImageColorspace(imagick.COLORSPACE_LAB)
	if err := newmw.ResizeImage(side, side, imagick.FILTER_LANCZOS2_SHARP, 1); err != nil {
		return nil, status.InternalError(err, "unable to resize thumbnail")
	}
	newmw.TransformImageColorspace(imagick.COLORSPACE_SRGB)

	for _, p := range newmw.GetImageProfiles("*") {
		switch p {
		case "icc":
			fallthrough // usually big
		default:
			newmw.RemoveImageProfile(p)
		}
	}

	for _, p := range newmw.GetImageProperties("*") {
		// I found these by looking through all properties of images I had and seeing which ones
		// ImageMagick reads (rather than informitively sets)
		switch p {
		case "png:bit-depth-written":
		case "png:IHDR.color-type-orig":
		case "png:IHDR.bit-depth-orig":

		case "jpeg:colorspace":
			fallthrough // we only want srgb
		case "jpeg:sampling-factor":
			fallthrough // we set this later
		case "date:modify":
			fallthrough // don't care
		case "png:tIME":
			fallthrough // don't care
		default:
			if err := newmw.DeleteImageProperty(p); err != nil {
				return nil, status.InternalError(err, "unable to delete property ", p)
			}
		}
	}

	format := pi.Format()
	switch {
	case format.IsGif():
	case format.IsPng():

	default:
		if err := newmw.SetImageFormat(string(DefaultJpegFormat)); err != nil {
			return nil, status.InternalError(err, "unable to set format")
		}
		fallthrough
	case format.IsJpeg():
		newmw.SetImageCompressionQuality(90)
		if err := newmw.SetOption("jpeg:sampling-factor", "1x1,1x1,1x1"); err != nil {
			return nil, status.InternalError(err, "unable to set sampling factor")
		}
	}

	// TODO:trim profiles (keep colorspace?), apply orientation,
	newpi := &pixurImage2{
		mw: newmw,
	}
	destroy = false
	return newpi, nil
}

func (pi *pixurImage2) Format() ImageFormat {
	return ImageFormat(pi.mw.GetImageFormat())
}

func (pi *pixurImage2) Dimensions() (uint, uint, status.S) {
	w, h, _, _, err := pi.mw.GetImagePage()
	if err != nil {
		return 0, 0, status.InternalError(err, "unable to get image dimensions")
	}
	return w, h, nil
}

func (pi *pixurImage2) Duration() (*time.Duration, status.S) {
	if !pi.Format().IsGif() {
		return nil, nil
	}
	if tps := pi.mw.GetImageTicksPerSecond(); tps != gifTicksPerSecond {
		return nil, status.InternalErrorf(nil, "Wrong ticks per second %v", tps)
	}
	switch pi.mw.GetNumberImages() {
	case 1:
		return nil, nil
	case 0:
		return nil, status.InvalidArgument(nil, "no images")
	}
	var d time.Duration
	pi.mw.ResetIterator()
	const tickDuration = time.Second / gifTicksPerSecond
	const delayTicksMax = math.MaxInt64 / int64(tickDuration)
	for {
		delayTicks := int64(pi.mw.GetImageDelay())
		if delayTicks < 0 || delayTicks > delayTicksMax {
			return nil, status.InvalidArgument(nil, "delayTicks would overflow", delayTicks)
		}
		// this should always be positive
		delayTickDuration := time.Duration(delayTicks) * tickDuration
		if d += delayTickDuration; d < 0 {
			return nil, status.InvalidArgument(nil, "duration overflow", d)
		}

		if !pi.mw.NextImage() {
			break
		}
	}

	return &d, nil
}

func (pi *pixurImage2) Close() {
	if pi.mw != nil {
		pi.mw.Destroy()
		pi.mw = nil
	}
}

func ReadImage2(r io.Reader) (PixurImage2, status.S) {
	mw := imagick.NewMagickWand()
	destroy := true
	defer func() {
		if destroy {
			mw.Destroy()
		}
	}()
	switch r := r.(type) {
	case *os.File:
		if err := mw.ReadImageFile(r); err != nil {
			return nil, status.InvalidArgument(err, "unable to decode image")
		}
	default:
		var b bytes.Buffer
		if _, err := io.Copy(&b, r); err != nil {
			return nil, status.InvalidArgument(err, "unable to copy image")
		}
		if err := mw.ReadImageBlob(b.Bytes()); err != nil {
			return nil, status.InvalidArgument(err, "unable to decode image")
		}
	}
	pi := &pixurImage2{mw: mw}
	destroy = false
	return pi, nil
}
<commit_msg>be/imaging: fix some bugs with thumbnail creation<commit_after>package imaging

import (
	"bytes"
	"io"
	"math"
	"os"
	"time"

	// this is the only outside pixur package dependency.  Avoid depending too much on schema.
	"pixur.org/pixur/be/status"

	// unfortuantely, most of my machines are stuck on 6.8.x.x
	// TODO: update this to v2 when possible.
	"gopkg.in/gographics/imagick.v1/imagick"
)

const (
	// WEBM magic header
	ebmlHeader = "\x1a\x45\xdf\xa3"
)

const (
	DefaultGifFormat  ImageFormat = "GIF"
	DefaultJpegFormat ImageFormat = "JPEG"
	DefaultPngFormat  ImageFormat = "PNG"
)

const (
	thumbnailSquareSize = 160
)

const gifTicksPerSecond = 100

func init() {
	imagick.Initialize()
	// never call Terminate
}

// a list of all formats: mw.QueryFormats("*")

// ImageFormat is the string format of the image
type ImageFormat string

// IsGif returns true if the type of this image is a GIF.
func (f ImageFormat) IsGif() bool {
	return f == "GIF" || f == "GIF87"
}

// IsJpeg returns true if the type of this image is a JPEG.
func (f ImageFormat) IsJpeg() bool {
	return f == "JPE" || f == "JPEG" || f == "JPG"
}

// IsPng returns true if the type of this image is a PNG.
func (f ImageFormat) IsPng() bool {
	// These are the types imagick says it supports: PNG PNG00 PNG24 PNG32 PNG48 PNG64 PNG8.  I am
	// only including the ones I know.
	return f == "PNG" || f == "PNG24" || f == "PNG32" || f == "PNG8"
}

type PixurImage2 interface {
	Format() ImageFormat
	Dimensions() (width, height uint, sts status.S)
	// In the future, this could also include a histogram
	Duration() (*time.Duration, status.S)

	Thumbnail() (PixurImage2, status.S)

	Write(io.Writer) status.S

	Close()
}

var _ PixurImage2 = (*pixurImage2)(nil)

type pixurImage2 struct {
	mw *imagick.MagickWand
}

func (pi *pixurImage2) Write(w io.Writer) status.S {

	switch w := w.(type) {
	case *os.File:
		if err := pi.mw.WriteImageFile(w); err != nil {
			return status.InternalError(err, "can't write image file")
		}
	default:
		panic("uh oh")
	}

	return nil
}

func (pi *pixurImage2) Thumbnail() (PixurImage2, status.S) {
	w, h, sts := pi.Dimensions()
	if sts != nil {
		return nil, sts
	}
	var neww, newh uint
	var x, y int
	if w > h {
		neww = h
		newh = h
		x = int((w - neww) / 2)
		y = 0
	} else {
		neww = w
		newh = w
		x = 0
		y = int((h - newh) / 2)
	}
	newmw := pi.mw.Clone()
	newmw.ResetIterator()
	// Some PNGs have an `oFFs` section that messes with the crop
	if err := newmw.SetImagePage(w, h, 0, 0); err != nil {
		return nil, status.InternalError(err, "unable to repage image")
	}

	destroy := true
	defer func() {
		if destroy {
			newmw.Destroy()
		}
	}()

	if err := newmw.CropImage(neww, newh, x, y); err != nil {
		return nil, status.InternalError(err, "unable to crop thumbnail")
	}

	side := uint(thumbnailSquareSize)
	newmw.TransformImageColorspace(imagick.COLORSPACE_LAB)
	if err := newmw.ResizeImage(side, side, imagick.FILTER_LANCZOS2_SHARP, 1); err != nil {
		return nil, status.InternalError(err, "unable to resize thumbnail")
	}
	newmw.TransformImageColorspace(imagick.COLORSPACE_SRGB)

	for _, p := range newmw.GetImageProfiles("*") {
		switch p {
		case "icc":
			fallthrough // usually big, and we always convert to srgb
		default:
			newmw.RemoveImageProfile(p)
		}
	}

	for _, p := range newmw.GetImageProperties("*") {
		// I found these by looking through all properties of images I had and seeing which ones
		// ImageMagick reads (rather than informatively sets)
		switch p {
		case "png:bit-depth-written":
		case "png:IHDR.color-type-orig":
		case "png:IHDR.bit-depth-orig":

		case "jpeg:colorspace":
			fallthrough // we only want srgb
		case "jpeg:sampling-factor":
			fallthrough // we set this later
		case "date:modify":
			fallthrough // don't care
		case "png:tIME":
			fallthrough // don't care
		default:
			if err := newmw.DeleteImageProperty(p); err != nil {
				return nil, status.InternalError(err, "unable to delete property ", p)
			}
		}
	}

	// None of the images I found had options, but delete them anyways
	for _, o := range newmw.GetOptions("*") {
		if err := newmw.DeleteOption(o); err != nil {
			return nil, status.InternalError(err, "unable to delete option ", o)
		}
	}

	for _, a := range newmw.GetImageArtifacts("*") {
		if err := newmw.DeleteImageArtifact(a); err != nil {
			return nil, status.InternalError(err, "unable to delete artifact ", a)
		}
	}

	format := pi.Format()
	switch {
	case format.IsGif():
	case format.IsPng():
		if err := newmw.SetOption("png:exclude-chunks", "all"); err != nil {
			return nil, status.InternalError(err, "unable to exclude png chunks")
		}
		// TODO: maybe do this for JPEG?
		if err := newmw.SetOption("png:bit-depth", "8"); err != nil {
			return nil, status.InternalError(err, "unable to set png bit depth")
		}

	default:
		if err := newmw.SetImageFormat(string(DefaultJpegFormat)); err != nil {
			return nil, status.InternalError(err, "unable to set format")
		}
		fallthrough
	case format.IsJpeg():
		newmw.SetImageCompressionQuality(90)
		if err := newmw.SetOption("jpeg:sampling-factor", "1x1,1x1,1x1"); err != nil {
			return nil, status.InternalError(err, "unable to set sampling factor")
		}
	}

	// TODO:trim profiles (keep colorspace?), apply orientation,
	newpi := &pixurImage2{
		mw: newmw,
	}
	destroy = false
	return newpi, nil
}

func (pi *pixurImage2) Format() ImageFormat {
	return ImageFormat(pi.mw.GetImageFormat())
}

func (pi *pixurImage2) Dimensions() (uint, uint, status.S) {
	/*
		w, h, _, _, err := pi.mw.GetImagePage()
		if err != nil {
			return 0, 0, status.InternalError(err, "unable to get image dimensions")
		}*/

	return pi.mw.GetImageWidth(), pi.mw.GetImageHeight(), nil
}

func (pi *pixurImage2) Duration() (*time.Duration, status.S) {
	if !pi.Format().IsGif() {
		return nil, nil
	}
	if tps := pi.mw.GetImageTicksPerSecond(); tps != gifTicksPerSecond {
		return nil, status.InternalErrorf(nil, "Wrong ticks per second %v", tps)
	}
	switch pi.mw.GetNumberImages() {
	case 1:
		return nil, nil
	case 0:
		return nil, status.InvalidArgument(nil, "no images")
	}
	var d time.Duration
	pi.mw.ResetIterator()
	const tickDuration = time.Second / gifTicksPerSecond
	const delayTicksMax = math.MaxInt64 / int64(tickDuration)
	for {
		delayTicks := int64(pi.mw.GetImageDelay())
		if delayTicks < 0 || delayTicks > delayTicksMax {
			return nil, status.InvalidArgument(nil, "delayTicks would overflow", delayTicks)
		}
		// this should always be positive
		delayTickDuration := time.Duration(delayTicks) * tickDuration
		if d += delayTickDuration; d < 0 {
			return nil, status.InvalidArgument(nil, "duration overflow", d)
		}

		if !pi.mw.NextImage() {
			break
		}
	}

	return &d, nil
}

func (pi *pixurImage2) Close() {
	if pi.mw != nil {
		pi.mw.Destroy()
		pi.mw = nil
	}
}

func ReadImage2(r io.Reader) (PixurImage2, status.S) {
	mw := imagick.NewMagickWand()
	destroy := true
	defer func() {
		if destroy {
			mw.Destroy()
		}
	}()
	switch r := r.(type) {
	case *os.File:
		if err := mw.ReadImageFile(r); err != nil {
			return nil, status.InvalidArgument(err, "unable to decode image")
		}
	default:
		var b bytes.Buffer
		if _, err := io.Copy(&b, r); err != nil {
			return nil, status.InvalidArgument(err, "unable to copy image")
		}
		if err := mw.ReadImageBlob(b.Bytes()); err != nil {
			return nil, status.InvalidArgument(err, "unable to decode image")
		}
	}
	pi := &pixurImage2{mw: mw}
	destroy = false
	return pi, nil
}
<|endoftext|>


<commit_before>package main

import (
    "bufio"
    "encoding/json"
    "fmt"
    "html/template"
    "io/ioutil"
    "net/http"
    "os"
    "regexp"
    "strings"
    "time"

    "github.com/sirupsen/logrus"
)

// Build - Basic struct returned by zuul api in json format
type Build struct {
    Status string `json:"result"`
    URL    string `json:"log_url"`
}

func writeHTLMReport(success []string, failed []string, hash string, output string) {

    var reportTemplate = template.Must(template.New("report").Parse(`
    <!DOCTYPE html>

    <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css"
          integrity="sha384-ggOyR0iXCbMQv3Xipma34MD+dH/1fQ784/j6cY/iJTQUOhcWr7x9JvoRxT2MZw1T"
          crossorigin="anonymous">

Report latest containers

Latest hash: {{.Hash}}

Date: {{.Today.Format "01-02-2006 15:04"}}

Failing: {{.Failing}}

Passing: {{.Passing}}

        Container Name
        Status

    {{range .Items}}
        {{ if eq .ContainerStatus "SUCCESS" }}

        {{ else }}

        {{ end }}
        {{.ContainerName}}
        {{.ContainerStatus}}

    {{end}}

    `))

    type Containers struct {
        ContainerName string
        ContainerStatus string
    }

    type Data struct {
        Items []*Containers
        Hash string
        Today time.Time
        Passing int
        Failing int
    }

    data := new(Data)
    data.Hash = hash
    data.Today = time.Now()
    data.Passing = len(success)
    data.Failing = len(failed)

    for _, container := range(success) {
        item := new(Containers)
        item.ContainerName = container
        item.ContainerStatus = "SUCCESS"
        data.Items = append(data.Items, item)
    }
    for _, container := range(failed) {
        item := new(Containers)
        item.ContainerName = container
        item.ContainerStatus = "FAILURE"
        data.Items = append(data.Items, item)
    }

    file, err := os.Create(output)
    if err != nil {
        logrus.Errorln("Failed to create HTML report file")
    }
    writer := bufio.NewWriter(file)
    if err := reportTemplate.Execute(writer, data); err != nil {
        writer.Flush()
        logrus.Fatal(err)
    }
    writer.Flush()
}

func getLatestGoodBuildURL(jobName string, opts *globalOptions) string {

    url := fmt.Sprintf("%sbuilds?job_name=%s", opts.zuulAPI, jobName)
    response, err := http.Get(url)
    var builds []Build

    if err != nil {
        logrus.Errorln("The HTTP request failed with error ", err)
    } else {
        data, _ := ioutil.ReadAll(response.Body)
        if err := json.Unmarshal(data, &builds); err != nil {
            logrus.Errorln("The unmarshal failed with error ", err)
        }
    }
    for _, build := range builds {
        if build.Status == "SUCCESS" {
            return build.URL
        }
    }
    return ""
}

func _fetch_(URL string) (string, error) {
    request, err := http.NewRequest(http.MethodGet, URL, nil)
    if err != nil {
        logrus.Errorln("The HTTP request failed with error ", err)
        return "", err
    }

    client := &http.Client{}
    response, err := client.Do(request)
    if err != nil {
        logrus.Errorln("Failed to get response from the HTTP request")
        return "", err
    }
    if response.StatusCode == 404 {
        msg := fmt.Sprintf("URL %s not found", URL)
        logrus.Errorln(msg)
        return "", fmt.Errorf(msg)
    }

    data, err := ioutil.ReadAll(response.Body)
    if err != nil {
        logrus.Errorln("Failed to read Body content")
        return "", err
    }
    return string(data), nil
}

func _fetch(URL string) (string, error) {
    response, err := http.Get(URL)
    if err != nil {
        logrus.Errorln("The HTTP request failed with error ", err)
        return "", err
    }
    if response.StatusCode == 404 {
        msg := fmt.Sprintf("URL not found: %s", URL)
        logrus.Errorln(msg)
        return "", fmt.Errorf(msg)
    }
    data, err := ioutil.ReadAll(response.Body)
    if err != nil {
        logrus.Errorln("Failed to read Body content")
        return "", err
    }
    return string(data), nil
}

func fetchLogs(URL string) string {
    // There are three different places where we can get the list of
    // containers...
    path := fmt.Sprintf("%slogs/containers-successfully-built.log", URL)
    data, _ := _fetch(path)

    if data == "" {
        path := fmt.Sprintf("%slogs/containers-built.log", URL)
        data , _ = _fetch(path)
    }

    if data == "" {
        path := fmt.Sprintf("%sjob-output.txt", URL)
        data, _ = _fetch(path)
    }
    logrus.Info("Fetching logs in ", path)
    return data
}

func repoExists(repoName string, repositories []Container) bool {
    for _, container := range repositories {
        if container.Name == repoName {
            return true
        }
    }
    return false
}

func getCurrentTripleoRepo(api string) string {

    type DlrnApiResponse struct {
        PromoteName   string `json:"promote_name"`
        RepoHash      string `json:"repo_hash"`
        AggregateHash string `json:"aggregate_hash"`
        CommitHash    string `json:"commit_hash"`
    }

    var returnApi []DlrnApiResponse

    apiEndpoint := "api/promotions?promote_name=current-tripleo&limit=1"
    apiUrl := fmt.Sprintf("%s/%s", api, apiEndpoint)

    response, err := http.Get(apiUrl)

    if err != nil {
        logrus.Errorln("The HTTP request failed with error ", err)
    } else {
        data, _ := ioutil.ReadAll(response.Body)
        if err := json.Unmarshal(data, &returnApi); err != nil {
            logrus.Errorln("The unmarshal failed with error ", err)
        }
    }
    if len(returnApi) > 0 {
        if returnApi[0].AggregateHash != "" {
            return returnApi[0].AggregateHash
        }
        return returnApi[0].RepoHash
    }

    return ""
}

func parseLog(data string) [][2]string {
    // r, _ := regexp.Compile(`(?m)item=docker push.*\/([\w-]+):([\w]+_[\w]+)`)
    r, _ := regexp.Compile(`(?m)primary \|\s(\w+)\: digest\:.*\n.*\n.*Tag w\/ arch suffix and push image: .*\/([\w-]+)`)
    var result [][2]string

    for _, matches := range r.FindAllStringSubmatch(data, -1) {
        if strings.Contains(matches[1], "x86_64") {
            result = append(result, [2]string{matches[2], matches[1][:len(matches[1])-7]})
        }
        // result = append(result, [2]string{matches[2], matches[1]})
    }
    if len(result) == 0 {
        r, _ := regexp.Compile(`(?m)\/([\w-]+)\s+([\w_]+)`)

        for _, matches := range r.FindAllStringSubmatch(data, -1) {
            result = append(result, [2]string{matches[1], matches[2]})
        }
    }
    return result
}
<commit_msg>Fix fetchLogs function<commit_after>package main

import (
    "bufio"
    "encoding/json"
    "fmt"
    "html/template"
    "io/ioutil"
    "net/http"
    "os"
    "regexp"
    "strings"
    "time"

    "github.com/sirupsen/logrus"
)

// Build - Basic struct returned by zuul api in json format
type Build struct {
    Status string `json:"result"`
    URL    string `json:"log_url"`
}

func writeHTLMReport(success []string, failed []string, hash string, output string) {

    var reportTemplate = template.Must(template.New("report").Parse(`
    <!DOCTYPE html>

    <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.3.1/css/bootstrap.min.css"
          integrity="sha384-ggOyR0iXCbMQv3Xipma34MD+dH/1fQ784/j6cY/iJTQUOhcWr7x9JvoRxT2MZw1T"
          crossorigin="anonymous">

Report latest containers

Latest hash: {{.Hash}}

Date: {{.Today.Format "01-02-2006 15:04"}}

Failing: {{.Failing}}

Passing: {{.Passing}}

        Container Name
        Status

    {{range .Items}}
        {{ if eq .ContainerStatus "SUCCESS" }}

        {{ else }}

        {{ end }}
        {{.ContainerName}}
        {{.ContainerStatus}}

    {{end}}

    `))

    type Containers struct {
        ContainerName string
        ContainerStatus string
    }

    type Data struct {
        Items []*Containers
        Hash string
        Today time.Time
        Passing int
        Failing int
    }

    data := new(Data)
    data.Hash = hash
    data.Today = time.Now()
    data.Passing = len(success)
    data.Failing = len(failed)

    for _, container := range(success) {
        item := new(Containers)
        item.ContainerName = container
        item.ContainerStatus = "SUCCESS"
        data.Items = append(data.Items, item)
    }
    for _, container := range(failed) {
        item := new(Containers)
        item.ContainerName = container
        item.ContainerStatus = "FAILURE"
        data.Items = append(data.Items, item)
    }

    file, err := os.Create(output)
    if err != nil {
        logrus.Errorln("Failed to create HTML report file")
    }
    writer := bufio.NewWriter(file)
    if err := reportTemplate.Execute(writer, data); err != nil {
        writer.Flush()
        logrus.Fatal(err)
    }
    writer.Flush()
}

func getLatestGoodBuildURL(jobName string, opts *globalOptions) string {

    url := fmt.Sprintf("%sbuilds?job_name=%s", opts.zuulAPI, jobName)
    response, err := http.Get(url)
    var builds []Build

    if err != nil {
        logrus.Errorln("The HTTP request failed with error ", err)
    } else {
        data, _ := ioutil.ReadAll(response.Body)
        if err := json.Unmarshal(data, &builds); err != nil {
            logrus.Errorln("The unmarshal failed with error ", err)
        }
    }
    for _, build := range builds {
        if build.Status == "SUCCESS" {
            return build.URL
        }
    }
    return ""
}

func _fetch_(URL string) (string, error) {
    request, err := http.NewRequest(http.MethodGet, URL, nil)
    if err != nil {
        logrus.Errorln("The HTTP request failed with error ", err)
        return "", err
    }

    client := &http.Client{}
    response, err := client.Do(request)
    if err != nil {
        logrus.Errorln("Failed to get response from the HTTP request")
        return "", err
    }
    if response.StatusCode == 404 {
        msg := fmt.Sprintf("URL %s not found", URL)
        logrus.Errorln(msg)
        return "", fmt.Errorf(msg)
    }

    data, err := ioutil.ReadAll(response.Body)
    if err != nil {
        logrus.Errorln("Failed to read Body content")
        return "", err
    }
    return string(data), nil
}

func _fetch(URL string) (string, error) {
    response, err := http.Get(URL)
    if err != nil {
        logrus.Errorln("The HTTP request failed with error ", err)
        return "", err
    }
    if response.StatusCode == 404 {
        msg := fmt.Sprintf("URL not found: %s", URL)
        logrus.Errorln(msg)
        return "", fmt.Errorf(msg)
    }
    data, err := ioutil.ReadAll(response.Body)
    if err != nil {
        logrus.Errorln("Failed to read Body content")
        return "", err
    }
    return string(data), nil
}

func fetchLogs(URL string) string {
    // There are three different places where we can get the list of
    // containers...
    path := fmt.Sprintf("%slogs/containers-successfully-built.log", URL)
    data, _ := _fetch(path)

    if data == "" {
        path = fmt.Sprintf("%slogs/containers-built.log", URL)
        data , _ = _fetch(path)
    }

    if data == "" {
        path = fmt.Sprintf("%sjob-output.txt", URL)
        data, _ = _fetch(path)
    }
    logrus.Info("Fetching logs in ", path)
    return data
}

func repoExists(repoName string, repositories []Container) bool {
    for _, container := range repositories {
        if container.Name == repoName {
            return true
        }
    }
    return false
}

func getCurrentTripleoRepo(api string) string {

    type DlrnApiResponse struct {
        PromoteName   string `json:"promote_name"`
        RepoHash      string `json:"repo_hash"`
        AggregateHash string `json:"aggregate_hash"`
        CommitHash    string `json:"commit_hash"`
    }

    var returnApi []DlrnApiResponse

    apiEndpoint := "api/promotions?promote_name=current-tripleo&limit=1"
    apiUrl := fmt.Sprintf("%s/%s", api, apiEndpoint)

    response, err := http.Get(apiUrl)

    if err != nil {
        logrus.Errorln("The HTTP request failed with error ", err)
    } else {
        data, _ := ioutil.ReadAll(response.Body)
        if err := json.Unmarshal(data, &returnApi); err != nil {
            logrus.Errorln("The unmarshal failed with error ", err)
        }
    }
    if len(returnApi) > 0 {
        if returnApi[0].AggregateHash != "" {
            return returnApi[0].AggregateHash
        }
        return returnApi[0].RepoHash
    }

    return ""
}

func parseLog(data string) [][2]string {
    // r, _ := regexp.Compile(`(?m)item=docker push.*\/([\w-]+):([\w]+_[\w]+)`)
    r, _ := regexp.Compile(`(?m)primary \|\s(\w+)\: digest\:.*\n.*\n.*Tag w\/ arch suffix and push image: .*\/([\w-]+)`)
    var result [][2]string

    for _, matches := range r.FindAllStringSubmatch(data, -1) {
        if strings.Contains(matches[1], "x86_64") {
            result = append(result, [2]string{matches[2], matches[1][:len(matches[1])-7]})
        }
        // result = append(result, [2]string{matches[2], matches[1]})
    }
    if len(result) == 0 {
        r, _ := regexp.Compile(`(?m)\/([\w-]+)\s+([\w_]+)`)

        for _, matches := range r.FindAllStringSubmatch(data, -1) {
            result = append(result, [2]string{matches[1], matches[2]})
        }
    }
    return result
}
<|endoftext|>


<commit_before>// Copyright 2016 The Gosl Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package msh

import (
	"math"
	"testing"

	"github.com/cpmech/gosl/chk"
	"github.com/cpmech/gosl/io"
	"github.com/cpmech/gosl/num"
	"github.com/cpmech/gosl/utl"
)

func Test_shp01(tst *testing.T) {

	//verbose()
	chk.PrintTitle("shp01")

	r := []float64{0, 0, 0}

	verb := true
	for name, _ := range Functions {

		io.Pfyel("--------------------------------- %-6s---------------------------------\n", name)

		// check S
		tol := 1e-17
		if name == "tri10" {
			tol = 1e-14
		}
		checkShape(tst, name, tol, verb)

		// check dSdR
		tol = 1e-14
		if name == "lin5" || name == "lin4" || name == "tri10" || name == "qua12" || name == "qua16" {
			tol = 1e-10
		}
		if name == "tri15" {
			tol = 1e-9
		}
		checkDerivs(tst, name, r, tol, verb)

		io.PfGreen("OK\n")
	}
}

// checkShape checks that shape functions result in 1.0 @ nodes
func checkShape(tst *testing.T, shape string, tol float64, verbose bool) {

	// information
	fcn := Functions[shape]
	ndim := GeomNdim[shape]
	nverts := NumVerts[shape]
	coords := NatCoords[shape]

	// allocate slices
	S := make([]float64, nverts)
	dSdR := utl.Alloc(nverts, ndim)

	// loop over all vertices
	errS := 0.0
	r := []float64{0, 0, 0}
	for n := 0; n < nverts; n++ {

		// natural coordinates @ vertex
		for i := 0; i < ndim; i++ {
			r[i] = coords[i][n]
		}

		// compute function
		fcn(S, dSdR, r, false)

		// check
		if verbose {
			for _, val := range S {
				if math.Abs(val) < 1e-15 {
					val = 0
				}
				io.Pf("%3v", val)
			}
			io.Pf("\n")
		}
		for m := 0; m < nverts; m++ {
			if n == m {
				errS += math.Abs(S[m] - 1.0)
			} else {
				errS += math.Abs(S[m])
			}
		}
	}

	// error
	if errS > tol {
		tst.Errorf("%s failed with err = %g\n", shape, errS)
		return
	}
}

// checkDerivs checks dSdR derivatives of shape structures
func checkDerivs(tst *testing.T, shape string, r []float64, tol float64, verbose bool) {

	// information
	fcn := Functions[shape]
	ndim := GeomNdim[shape]
	nverts := NumVerts[shape]

	// allocate slices
	S := make([]float64, nverts)
	dSdR := utl.Alloc(nverts, ndim)

	// auxiliary
	r_tmp := make([]float64, len(r))
	S_tmp := make([]float64, nverts)

	// analytical
	fcn(S, dSdR, r, true)

	// numerical
	for n := 0; n < nverts; n++ {
		for i := 0; i < ndim; i++ {
			dSndRi, _ := num.DerivCentral(func(t float64, args ...interface{}) (Sn float64) {
				copy(r_tmp, r)
				r_tmp[i] = t
				fcn(S_tmp, nil, r_tmp, false)
				Sn = S_tmp[n]
				return
			}, r[i], 1e-1)
			if verbose {
				io.Pfgrey2("  dS%ddR%d @ %5.2f = %v (num: %v)\n", n, i, r, dSdR[n][i], dSndRi)
			}
			if math.Abs(dSdR[n][i]-dSndRi) > tol {
				tst.Errorf("nurbs dS%ddR%d failed with err = %g\n", n, i, math.Abs(dSdR[n][i]-dSndRi))
				return
			}
		}
	}
}
<commit_msg>Use chk deriv instead of num deriv in gm/msh<commit_after>// Copyright 2016 The Gosl Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package msh

import (
	"math"
	"testing"

	"github.com/cpmech/gosl/chk"
	"github.com/cpmech/gosl/io"
	"github.com/cpmech/gosl/utl"
)

func Test_shp01(tst *testing.T) {

	//verbose()
	chk.PrintTitle("shp01")

	r := []float64{0, 0, 0}

	for name, _ := range Functions {

		io.Pfyel("--------------------------------- %-6s---------------------------------\n", name)

		// check S
		tol := 1e-17
		if name == "tri10" {
			tol = 1e-14
		}
		checkShape(tst, name, tol, chk.Verbose)

		// check dSdR
		tol = 1e-14
		if name == "lin5" || name == "lin4" || name == "tri10" || name == "qua12" || name == "qua16" {
			tol = 1e-10
		}
		if name == "tri15" {
			tol = 1e-9
		}
		checkDerivs(tst, name, r, tol, chk.Verbose)

		io.PfGreen("OK\n")
	}
}

// checkShape checks that shape functions result in 1.0 @ nodes
func checkShape(tst *testing.T, shape string, tol float64, verbose bool) {

	// information
	fcn := Functions[shape]
	ndim := GeomNdim[shape]
	nverts := NumVerts[shape]
	coords := NatCoords[shape]

	// allocate slices
	S := make([]float64, nverts)
	dSdR := utl.Alloc(nverts, ndim)

	// loop over all vertices
	errS := 0.0
	r := []float64{0, 0, 0}
	for n := 0; n < nverts; n++ {

		// natural coordinates @ vertex
		for i := 0; i < ndim; i++ {
			r[i] = coords[i][n]
		}

		// compute function
		fcn(S, dSdR, r, false)

		// check
		if verbose {
			for _, val := range S {
				if math.Abs(val) < 1e-15 {
					val = 0
				}
				io.Pf("%3v", val)
			}
			io.Pf("\n")
		}
		for m := 0; m < nverts; m++ {
			if n == m {
				errS += math.Abs(S[m] - 1.0)
			} else {
				errS += math.Abs(S[m])
			}
		}
	}

	// error
	if errS > tol {
		tst.Errorf("%s failed with err = %g\n", shape, errS)
		return
	}
}

// checkDerivs checks dSdR derivatives of shape structures
func checkDerivs(tst *testing.T, shape string, r []float64, tol float64, verbose bool) {

	// information
	fcn := Functions[shape]
	ndim := GeomNdim[shape]
	nverts := NumVerts[shape]

	// allocate slices
	S := make([]float64, nverts)
	dSdR := utl.Alloc(nverts, ndim)

	// analytical
	fcn(S, dSdR, r, true)

	chk.DerivVecVec(tst, "dSdR", tol, dSdR, r[:ndim], 1e-1, verbose, func(f, x []float64) error {
		fcn(f, nil, x, false)
		return nil
	})
}
<|endoftext|>


<commit_before>// Copyright 2011 The Walk Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package walk

import "syscall"

import . "github.com/lxn/go-winapi"

const tabPageWindowClass = `\o/ Walk_TabPage_Class \o/`

var tabPageBackgroundBrush Brush

func init() {
	MustRegisterWindowClass(tabPageWindowClass)

	tabPageBackgroundBrush, _ = NewSystemColorBrush(COLOR_WINDOW)
}

type TabPage struct {
	ContainerBase
	title         string
	tabWidget     *TabWidget
	titleProperty *Property
}

func NewTabPage() (*TabPage, error) {
	tp := &TabPage{}

	if err := InitWidget(
		tp,
		nil,
		tabPageWindowClass,
		WS_POPUP,
		WS_EX_CONTROLPARENT); err != nil {
		return nil, err
	}

	tp.children = newWidgetList(tp)

	tp.SetBackground(tabPageBackgroundBrush)

	tp.titleProperty = NewProperty(
		"Title",
		func() interface{} {
			return tp.Title()
		},
		func(v interface{}) error {
			return tp.SetTitle(v.(string))
		},
		nil)

	tp.MustRegisterProperties(tp.titleProperty)

	return tp, nil
}

func (tp *TabPage) Dispose() {
	b := tp.Background()
	if b != nil {
		b.Dispose()
		tp.SetBackground(nil)
	}

	tp.ContainerBase.Dispose()
}

func (tp *TabPage) Enabled() bool {
	if tp.tabWidget != nil {
		return tp.tabWidget.Enabled() && tp.enabled
	}

	return tp.enabled
}

func (tp *TabPage) Font() *Font {
	if tp.font != nil {
		return tp.font
	} else if tp.tabWidget != nil {
		return tp.tabWidget.Font()
	}

	return defaultFont
}

func (tp *TabPage) Title() string {
	return tp.title
}

func (tp *TabPage) SetTitle(value string) error {
	tp.title = value

	if tp.tabWidget == nil {
		return nil
	}

	return tp.tabWidget.onPageChanged(tp)
}

func (tp *TabPage) tcItem() *TCITEM {
	text := syscall.StringToUTF16(tp.Title())

	item := &TCITEM{
		Mask:       TCIF_TEXT,
		PszText:    &text[0],
		CchTextMax: int32(len(text)),
	}

	return item
}
<commit_msg>TabPage: Do not dispose of shared background brush<commit_after>// Copyright 2011 The Walk Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package walk

import "syscall"

import . "github.com/lxn/go-winapi"

const tabPageWindowClass = `\o/ Walk_TabPage_Class \o/`

var tabPageBackgroundBrush Brush

func init() {
	MustRegisterWindowClass(tabPageWindowClass)

	tabPageBackgroundBrush, _ = NewSystemColorBrush(COLOR_WINDOW)
}

type TabPage struct {
	ContainerBase
	title         string
	tabWidget     *TabWidget
	titleProperty *Property
}

func NewTabPage() (*TabPage, error) {
	tp := &TabPage{}

	if err := InitWidget(
		tp,
		nil,
		tabPageWindowClass,
		WS_POPUP,
		WS_EX_CONTROLPARENT); err != nil {
		return nil, err
	}

	tp.children = newWidgetList(tp)

	tp.SetBackground(tabPageBackgroundBrush)

	tp.titleProperty = NewProperty(
		"Title",
		func() interface{} {
			return tp.Title()
		},
		func(v interface{}) error {
			return tp.SetTitle(v.(string))
		},
		nil)

	tp.MustRegisterProperties(tp.titleProperty)

	return tp, nil
}

func (tp *TabPage) Enabled() bool {
	if tp.tabWidget != nil {
		return tp.tabWidget.Enabled() && tp.enabled
	}

	return tp.enabled
}

func (tp *TabPage) Font() *Font {
	if tp.font != nil {
		return tp.font
	} else if tp.tabWidget != nil {
		return tp.tabWidget.Font()
	}

	return defaultFont
}

func (tp *TabPage) Title() string {
	return tp.title
}

func (tp *TabPage) SetTitle(value string) error {
	tp.title = value

	if tp.tabWidget == nil {
		return nil
	}

	return tp.tabWidget.onPageChanged(tp)
}

func (tp *TabPage) tcItem() *TCITEM {
	text := syscall.StringToUTF16(tp.Title())

	item := &TCITEM{
		Mask:       TCIF_TEXT,
		PszText:    &text[0],
		CchTextMax: int32(len(text)),
	}

	return item
}
<|endoftext|>


<commit_before>// Copyright 2013 Yang Hong. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Program mailthem implements a simple SMTP mail deliver tool
package main

import (
	"bufio"
	"bytes"
	"code.google.com/p/gopass"
	"encoding/base64"
	"fmt"
	"io"
	"io/ioutil"
	"net/smtp"
	"os"
	"path"
	"strings"
)

// SInfo student information
type SInfo struct {
	snum string
	name string
	addr string
}

var boundary = "f46d043c813270fc6b04c2d223da"

// getAuthInfo get smtp authentication username and password
// from command line using gopass
func getAuthInfo() (string, string, error) {
	reader := bufio.NewReader(os.Stdin)

	fmt.Print("Enter username: ")
	username, _ := reader.ReadString('\n')

	password, passerr := gopass.GetPass("Enter password: ")
	return strings.Trim(username, "\n"), password, passerr
}

// readStudentInfo reads student email addresses and name strings
// from the contact file
func readStudentInfo() []SInfo {
	// read email addresses from a file
	fmt.Print("Directory with address list: ")
	addrListPath, err := bufio.NewReader(os.Stdin).ReadString('\n')
	addrListFile, err := os.Open(strings.Trim(addrListPath, "\n"))
	if err != nil {
		panic(err)
	}

	addrListBuf := bufio.NewReader(addrListFile)

	// 50 students for initial set
	var sl []SInfo = make([]SInfo, 0, 50)
	for {
		line, err := addrListBuf.ReadString('\n')
		if err != nil && err != io.EOF {
			panic(err)
		}
		entry := strings.Split(strings.Trim(line, "\n"), " ")

		// make sure the line indeed has 3 parts
		if len(entry) == 3 {
			sl = append(sl, SInfo{entry[0], entry[1], entry[2]})
		}
		// the last line will return io.EOF
		if err == io.EOF {
			break
		}
	}
	return sl
}

func readAttachInfo() (filename string, content []byte) {
	fmt.Print("Attachment filename: ")

	filename, err := bufio.NewReader(os.Stdin).ReadString('\n')
	if err != nil {
		panic(err)
	}

	content, err = ioutil.ReadFile(strings.Trim(filename, "\n"))
	if err != nil {
		panic(err)
	}
	return filename, content
}

func readMailBody() (body []byte) {
	fmt.Print("Body file path: ")

	filename, err := bufio.NewReader(os.Stdin).ReadString('\n')
	if err != nil {
		panic(err)
	}

	body, err = ioutil.ReadFile(strings.Trim(filename, "\n"))
	if err != nil {
		panic(err)
	}
	return body
}

// makeMessage generates message contents for each student
func makeMessage(each SInfo, header map[string]string, body []byte, attachName string, content []byte) (msg string) {
	var buf bytes.Buffer // A Buffer needs no initialization.

	for k, v := range header {
		buf.WriteString(k)
		buf.WriteString(": ")
		buf.WriteString(v)
		buf.WriteString("\r\n")
	}

	if attachName != "" {
		buf.WriteString("Content-Type: multipart/mixed; boundary=" + boundary + "\r\n")
		buf.WriteString("--" + boundary + "\r\n")
	}

	buf.WriteString("Content-Type: text/plain; charset=\"utf-8\"\r\n")

	// start of body
	buf.Write(body)
	buf.Write([]byte("\r\n"))
	// buf.Write([]byte("Dear all,\n\r"))
	// buf.Write([]byte("  This is Distributed Computing course website and password required when downloading materials.\n\r"))
	// buf.Write([]byte("  http://zhiyuan.sjtu.edu.cn/courses/476 password：Lorenzo\n\r"))

	// buf.WriteString("\n\r\n\r----------\n\r")
	// buf.WriteString("Heng\n\r")

	// start of attachment
	buf.WriteString("--" + boundary + "\r\n")
	buf.WriteString("Content-Type: application/octet-stream\n")
	buf.WriteString("Content-Transfer-Encoding: base64\n")
	buf.WriteString("Content-Disposition: attachment; filename=\"" + path.Base(strings.Trim(attachName, "\n")) + "\"\n\n")
	b := make([]byte, base64.StdEncoding.EncodedLen(len(content)))
	base64.StdEncoding.Encode(b, content)
	buf.Write(b)
	buf.WriteString("\n--" + boundary + "\r\n")

	msg, _ = buf.ReadString(byte(0))
	fmt.Println(msg)
	return msg
}

func main() {
	username, password, err := getAuthInfo()
	if err != nil {
		fmt.Println(err)
	}

	sl := readStudentInfo()

	attach, attachContent := readAttachInfo()

	body := readMailBody()

	fmt.Println(sl)
	// to avoid server recognition, use gmail as default
	auth := smtp.PlainAuth("", username, password, "smtp.gmail.com")

	// fill message with student lab grades
	for _, each := range sl {
		title := "DC course website"
		header := make(map[string]string)
		header["From"] = username
		header["To"] = each.addr
		header["Subject"] = title
		header["MIME-Version"] = "1.0"
		// header["Content-Transfer-Encoding"] = "base64"

		msgstr := []byte(makeMessage(each, header, body, attach, attachContent))

		err = smtp.SendMail("smtp.gmail.com:587", auth, username, []string{each.addr}, msgstr)
		if err != nil {
			fmt.Println(err)
		} else {
			fmt.Printf("send ok %s\n", each.addr)
		}
	}
}
<commit_msg>refactore readstring call; add multiple attachments<commit_after>// Copyright 2013 Yang Hong. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Program mailthem implements a simple SMTP mail deliver tool
package main

import (
	"bufio"
	"bytes"
	"code.google.com/p/gopass"
	"encoding/base64"
	"fmt"
	"io"
	"io/ioutil"
	"net/smtp"
	"os"
	"path"
	"strings"
)

// SInfo student information
type SInfo struct {
	snum string
	name string
	addr string
}

// borrowed from github.com/scorredoira/email
var boundary = "f46d043c813270fc6b04c2d223da"

func oneLiner(prompt string) string {
	reader := bufio.NewReader(os.Stdin)

	fmt.Printf("%s", prompt)
	line, err := reader.ReadString('\n')

	if err != nil {
		fmt.Fprintf(os.Stderr, "read stdin error\n")
		panic(err)
	}

	return strings.Trim(line, "\r\n")
}

// getAuthInfo get smtp authentication username and password
// from command line using gopass
func getAuthInfo() (string, string) {
	username := oneLiner("Enter username: ")

	password, passerr := gopass.GetPass("Enter password: ")
	if passerr != nil {
		fmt.Fprintf(os.Stderr, "get password error\n")
		panic(passerr)
	}

	return username, password
}

// readStudentInfo reads student email addresses and name strings
// from the contact file
func readStudentInfo() []SInfo {
	// read email addresses from a file
	addrListPath := oneLiner("Directory with address list: ")
	addrListFile, err := os.Open(addrListPath)
	if err != nil {
		panic(err)
	}

	addrListBuf := bufio.NewReader(addrListFile)

	// 50 students for initial set
	var sl []SInfo = make([]SInfo, 0, 50)
	for {
		line, err := addrListBuf.ReadString('\n')
		if err != nil && err != io.EOF {
			panic(err)
		}
		entry := strings.Split(strings.Trim(line, "\n"), " ")

		// make sure the line indeed has 3 parts
		if len(entry) == 3 {
			sl = append(sl, SInfo{entry[0], entry[1], entry[2]})
		}
		// the last line will return io.EOF
		if err == io.EOF {
			break
		}
	}
	return sl
}

func readAttachInfo() map[string][]byte {
	attachments := make(map[string][]byte)

	for {
		filename := oneLiner("Attachment filename: ")

		if filename == "" {
			return attachments
		}

		content, err := ioutil.ReadFile(filename)
		if err != nil {
			panic(err)
		}
		attachments[filename] = content
	}
}

func readMailBody() (body []byte) {
	filename := oneLiner("Body file path: ")

	body, err := ioutil.ReadFile(filename)
	if err != nil {
		panic(err)
	}
	return body
}

// makeMessage generates message contents for each student
// Attachment part of this function is copied and modified from
// https://github.com/scorredoira/email
func makeMessage(each SInfo, header map[string]string, body []byte, attachments map[string][]byte) (msg string) {
	var buf bytes.Buffer // A Buffer needs no initialization.

	for k, v := range header {
		buf.WriteString(k)
		buf.WriteString(": ")
		buf.WriteString(v)
		buf.WriteString("\r\n")
	}

	if len(attachments) > 0 {
		buf.WriteString("Content-Type: multipart/mixed; boundary=" + boundary + "\r\n")
		buf.WriteString("--" + boundary + "\r\n")
	}

	buf.WriteString("Content-Type: text/plain; charset=\"utf-8\"\r\n")

	// start of body
	buf.Write(body)
	buf.Write([]byte("\r\n"))

	// start of attachment
	for name, content := range attachments {
		buf.WriteString("--" + boundary + "\r\n")
		buf.WriteString("Content-Type: application/octet-stream\n")
		buf.WriteString("Content-Transfer-Encoding: base64\n")
		buf.WriteString("Content-Disposition: attachment; filename=\"" + path.Base(name) + "\"\n\n")
		b := make([]byte, base64.StdEncoding.EncodedLen(len(content)))
		base64.StdEncoding.Encode(b, content)
		buf.Write(b)
		buf.WriteString("\n--" + boundary + "\r\n")
	}

	msg, _ = buf.ReadString(byte(0))
	fmt.Println(msg)
	return msg
}

func main() {
	username, password := getAuthInfo()

	sl := readStudentInfo()

	attachments := readAttachInfo()

	title := oneLiner("Title: ")

	body := readMailBody()

	fmt.Println(sl)
	// to avoid server recognition, use gmail as default
	auth := smtp.PlainAuth("", username, password, "smtp.gmail.com")

	// fill message with student lab grades
	for _, each := range sl {
		header := make(map[string]string)
		header["From"] = username
		header["To"] = each.addr
		header["Subject"] = title
		header["MIME-Version"] = "1.0"
		// header["Content-Transfer-Encoding"] = "base64"

		msgstr := []byte(makeMessage(each, header, body, attachments))

		if err := smtp.SendMail("smtp.gmail.com:587", auth, username, []string{each.addr}, msgstr); err != nil {
			fmt.Println(err)
		} else {
			fmt.Printf("send ok %s\n", each.addr)
		}
	}
}
<|endoftext|>