<commit_before>
package golog

import (
    "io"
    "io/ioutil"
    "log"
    "os"
    "fmt"
    "strings"
    "time"
    "sync/atomic"
)
const (
	// everything
	LevelTrace int32 = 1

	// Info, Warnings and Errors
	LevelInfo int32 = 2

	// Warning and Errors
	LevelWarn int32 = 4

	// Errors
	LevelError int32 = 8
)

// traceLog provides support to write to log files.
type traceLog struct {
	LogLevel           int32
	Trace              *log.Logger
	Info               *log.Logger
	Warning            *log.Logger
	Error              *log.Logger
	File               *log.Logger
	LogFile            *os.File
}

// log maintains a pointer to a singleton for the logging system.
var logger traceLog

// Called to init the logging system.
func Init(logLevel int32, baseFilePath string) {
	log.SetPrefix("TRACE: ")
	log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile)
	StartFile(logLevel, baseFilePath)
}

// StartFile initializes tracelog and only displays the specified logging level
// and creates a file to capture writes.
func StartFile(logLevel int32, baseFilePath string) {
	baseFilePath = strings.TrimRight(baseFilePath, "/")
	currentDate := time.Now().UTC()
	dateDirectory := time.Now().UTC().Format("2006-01-02")
	dateFile := currentDate.Format("2006-01-02T15-04-05")

	filePath := fmt.Sprintf("%s/%s/", baseFilePath, dateDirectory)
	fileName := strings.Replace(fmt.Sprintf("%s.txt", dateFile), " ", "-", -1)

	err := os.MkdirAll(filePath, os.ModePerm)
	if err != nil {
		log.Fatalf("main : Start : Failed to Create log directory : %s : %s\n", filePath, err)
	}

	logf, err := os.Create(fmt.Sprintf("%s%s", filePath, fileName))
	if err != nil {
		log.Fatalf("main : Start : Failed to Create log file : %s : %s\n", fileName, err)
	}

	turnOnLogging(logLevel, logf)

}

// Stop will release resources and shutdown all processing.
func Stop() error {
	var err error
	if logger.LogFile != nil {
		Trace("main", "Stop", "Closing File")
		err = logger.LogFile.Close()
	}
	return err
}

// LogLevel returns the configured logging level.
func LogLevel() int32 {
	return atomic.LoadInt32(&logger.LogLevel)
}

// turnOnLogging configures the logging writers.
func turnOnLogging(logLevel int32, fileHandle io.Writer) {
	traceHandle := ioutil.Discard
	infoHandle := ioutil.Discard
	warnHandle := ioutil.Discard
	errorHandle := ioutil.Discard

	if logLevel&LevelTrace != 0 {
		traceHandle = os.Stdout
		infoHandle = os.Stdout
		warnHandle = os.Stdout
		errorHandle = os.Stderr
	}

	if logLevel&LevelInfo != 0 {
		infoHandle = os.Stdout
		warnHandle = os.Stdout
		errorHandle = os.Stderr
	}

	if logLevel&LevelWarn != 0 {
		warnHandle = os.Stdout
		errorHandle = os.Stderr
	}

	if logLevel&LevelError != 0 {
		errorHandle = os.Stderr
	}

	if fileHandle != nil {
		if traceHandle == os.Stdout {
			traceHandle = io.MultiWriter(fileHandle, traceHandle)
		}

		if infoHandle == os.Stdout {
			infoHandle = io.MultiWriter(fileHandle, infoHandle)
		}

		if warnHandle == os.Stdout {
			warnHandle = io.MultiWriter(fileHandle, warnHandle)
		}

		if errorHandle == os.Stderr {
			errorHandle = io.MultiWriter(fileHandle, errorHandle)
		}
	}

	logger.Trace = log.New(traceHandle, "TRACE: ", log.Ldate|log.Ltime|log.Lshortfile)
	logger.Info = log.New(infoHandle, "INFO: ", log.Ldate|log.Ltime|log.Lshortfile)
	logger.Warning = log.New(warnHandle, "WARNING: ", log.Ldate|log.Ltime|log.Lshortfile)
	logger.Error = log.New(errorHandle, "ERROR: ", log.Ldate|log.Ltime|log.Lshortfile)

	atomic.StoreInt32(&logger.LogLevel, logLevel)
}

//** TRACE

// Trace writes to the Trace destination
func Trace(format string, a ...interface{}) {
	logger.Trace.Output(2, fmt.Sprintf("%s\n", fmt.Sprintf(format, a...)))
}

//** INFO

// Info writes to the Info destination
func Info(format string, a ...interface{}) {
	logger.Info.Output(2, fmt.Sprintf(fmt.Sprintf(format, a...)))
}

//** WARNING

// Warning writes to the Warning destination
func Warning(format string, a ...interface{}) {
	logger.Warning.Output(2, fmt.Sprintf(fmt.Sprintf(format, a...)))
}

//** ERROR

// Error writes to the Error destination and accepts an err
func Error(format string, a ...interface{}) {
	logger.Error.Output(2, fmt.Sprintf(fmt.Sprintf(format, a...)))
}

//writes to the Error and exit(1)
func Fatal(format string, a ...interface{}) {
	logger.Error.Output(2, fmt.Sprintf(fmt.Sprintf(format, a...)))
	os.Exit(1)
}

<commit_msg>cleanup. changes in the interface<commit_after>
package golog

import (
    "io"
    "io/ioutil"
    "log"
    "os"
    "fmt"
    "strings"
    "time"
    "sync/atomic"
)
const (
	// everything
	LevelTrace int32 = 1

	// Info, Warnings and Errors
	LevelInfo int32 = 2

	// Warning and Errors
	LevelWarn int32 = 4

	// Errors
	LevelError int32 = 8
)

// goLogStruct provides support to write to log files.
type goLogStruct struct {
	LogLevel           int32
	Trace              *log.Logger
	Info               *log.Logger
	Warning            *log.Logger
	Error              *log.Logger
	File               *log.Logger
	LogFile            *os.File
}

// log maintains a pointer to a singleton for the logging system.
var logger goLogStruct

// Called to init the logging system.
func (lS goLogStruct) Init(logLevel int32, baseFilePath string) error {
	log.SetPrefix("TRACE: ")
	log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile)
	err := startFile(logLevel, baseFilePath)
	if(err != nil)
		return err;
	lS = logger
	return
}

// StartFile initializes goLogStruct and only displays the specified logging level
// and creates a file to capture writes.
func startFile(logLevel int32, baseFilePath string) error {
	baseFilePath = strings.TrimRight(baseFilePath, "/")
	currentDate := time.Now().UTC()
	dateDirectory := time.Now().UTC().Format("2006-01-02")
	dateFile := currentDate.Format("2006-01-02T15-04-05")

	filePath := fmt.Sprintf("%s/%s/", baseFilePath, dateDirectory)
	fileName := strings.Replace(fmt.Sprintf("%s.txt", dateFile), " ", "-", -1)

	err := os.MkdirAll(filePath, os.ModePerm)
	if err != nil {
		log.Fatalf("main : Start : Failed to Create log directory : %s : %s\n", filePath, err)
		return err
	}

	logf, err := os.Create(fmt.Sprintf("%s%s", filePath, fileName))
	if err != nil {
		log.Fatalf("main : Start : Failed to Create log file : %s : %s\n", fileName, err)
		return err
	}

	turnOnLogging(logLevel, logf)
	return

}

// Stop will release resources and shutdown all processing.
func Stop() error {
	var err error
	if logger.LogFile != nil {
		Trace("main", "Stop", "Closing File")
		err = logger.LogFile.Close()
	}
	return err
}

// LogLevel returns the configured logging level.
func GetLogLevel() int32 {
	return atomic.LoadInt32(&logger.LogLevel)
}

// turnOnLogging configures the logging writers.
func turnOnLogging(logLevel int32, fileHandle io.Writer) {
	traceHandle := ioutil.Discard
	infoHandle := ioutil.Discard
	warnHandle := ioutil.Discard
	errorHandle := ioutil.Discard

	if logLevel&LevelTrace != 0 {
		traceHandle = os.Stdout
		infoHandle = os.Stdout
		warnHandle = os.Stdout
		errorHandle = os.Stderr
	}

	if logLevel&LevelInfo != 0 {
		infoHandle = os.Stdout
		warnHandle = os.Stdout
		errorHandle = os.Stderr
	}

	if logLevel&LevelWarn != 0 {
		warnHandle = os.Stdout
		errorHandle = os.Stderr
	}

	if logLevel&LevelError != 0 {
		errorHandle = os.Stderr
	}

	if fileHandle != nil {
		if traceHandle == os.Stdout {
			traceHandle = io.MultiWriter(fileHandle, traceHandle)
		}

		if infoHandle == os.Stdout {
			infoHandle = io.MultiWriter(fileHandle, infoHandle)
		}

		if warnHandle == os.Stdout {
			warnHandle = io.MultiWriter(fileHandle, warnHandle)
		}

		if errorHandle == os.Stderr {
			errorHandle = io.MultiWriter(fileHandle, errorHandle)
		}
	}

	logger.Trace = log.New(traceHandle, "TRACE: ", log.Ldate|log.Ltime|log.Lshortfile)
	logger.Info = log.New(infoHandle, "INFO: ", log.Ldate|log.Ltime|log.Lshortfile)
	logger.Warning = log.New(warnHandle, "WARNING: ", log.Ldate|log.Ltime|log.Lshortfile)
	logger.Error = log.New(errorHandle, "ERROR: ", log.Ldate|log.Ltime|log.Lshortfile)

	atomic.StoreInt32(&logger.LogLevel, logLevel)
}

//** TRACE

// Trace writes to the Trace destination
func Trace(format string, a ...interface{}) {
	logger.Trace.Output(2, fmt.Sprintf("%s\n", fmt.Sprintf(format, a...)))
}

//** INFO

// Info writes to the Info destination
func Info(format string, a ...interface{}) {
	logger.Info.Output(2, fmt.Sprintf(fmt.Sprintf(format, a...)))
}

//** WARNING

// Warning writes to the Warning destination
func Warning(format string, a ...interface{}) {
	logger.Warning.Output(2, fmt.Sprintf(fmt.Sprintf(format, a...)))
}

//** ERROR

// Error writes to the Error destination and accepts an err
func Error(format string, a ...interface{}) {
	logger.Error.Output(2, fmt.Sprintf(fmt.Sprintf(format, a...)))
}

//writes to the Error and exit(1)
func Fatal(format string, a ...interface{}) {
	logger.Error.Output(2, fmt.Sprintf(fmt.Sprintf(format, a...)))
	os.Exit(1)
}

<|endoftext|>


<commit_before>package empire

import (
	"database/sql/driver"
	"time"

	"github.com/remind101/empire/pkg/container"
	"gopkg.in/gorp.v1"
)

// JobID represents a unique identifier for a Job.
type JobID string

// Scan implements the sql.Scanner interface.
func (id *JobID) Scan(src interface{}) error {
	if src, ok := src.([]byte); ok {
		*id = JobID(src)
	}

	return nil
}

// Value implements the driver.Value interface.
func (id JobID) Value() (driver.Value, error) {
	return driver.Value(string(id)), nil
}

// Job represents a Job that was submitted to the scheduler.
type Job struct {
	ID JobID `db:"id"`

	AppName        `db:"app_id"`
	ReleaseVersion `db:"release_version"`
	ProcessType    `db:"process_type"`
	Instance       int `db:"instance"`

	Environment Vars  `db:"environment"`
	Image       Image `db:"image"`
	Command     `db:"command"`

	// UpdatedAt indicates when this job last changed state.
	UpdatedAt time.Time `db:"updated_at"`
}

// PreInsert implements a pre insert hook for the db interface
func (j *Job) PreInsert(s gorp.SqlExecutor) error {
	j.UpdatedAt = Now()
	return nil
}

func (j *Job) ContainerName() string {
	return newContainerName(
		j.AppName,
		j.ReleaseVersion,
		j.ProcessType,
		j.Instance,
	)
}

// JobState represents the state of a submitted job.
type JobState struct {
	Job       *Job
	MachineID string
	Name      string
	State     string
}

// Schedule is an interface that represents something that can schedule jobs
// onto the cluster.
type Scheduler interface {
	Schedule(...*Job) error
	Unschedule(...*Job) error
}

type JobsFinder interface {
	JobsList(JobsListQuery) ([]*Job, error)
}

type JobsService interface {
	Scheduler
	JobsFinder
}

type jobsService struct {
	DB
	scheduler container.Scheduler
}

func (s *jobsService) JobsList(q JobsListQuery) ([]*Job, error) {
	return JobsList(s.DB, q)
}

func (s *jobsService) Schedule(jobs ...*Job) error {
	for _, j := range jobs {
		if _, err := Schedule(s.DB, s.scheduler, j); err != nil {
			return err
		}
	}

	return nil
}

func (s *jobsService) Unschedule(jobs ...*Job) error {
	for _, j := range jobs {
		if err := Unschedule(s.DB, s.scheduler, j); err != nil {
			return err
		}
	}

	return nil
}

// JobsCreate inserts the Job into the database.
func JobsCreate(db Inserter, job *Job) (*Job, error) {
	return job, db.Insert(job)
}

// JobsDestroy removes a Job from the database.
func JobsDestroy(db Deleter, job *Job) error {
	_, err := db.Delete(job)
	return err
}

// JobsListQuery is a query object to filter results from JobsRepository List.
type JobsListQuery struct {
	App     AppName
	Release ReleaseVersion
}

// JobsList returns a filtered list of Jobs.
func JobsList(db Queryier, q JobsListQuery) ([]*Job, error) {
	var jobs []*Job
	query := `select * from jobs where (app_id = $1 OR $1 = '') and (release_version = $2 OR $2 = 0)`
	return jobs, db.Select(&jobs, query, string(q.App), int(q.Release))
}

// Schedule schedules to job onto the cluster, then persists it to the database.
func Schedule(db Inserter, s container.Scheduler, j *Job) (*Job, error) {
	container := &container.Container{
		Name:    j.ContainerName(),
		Env:     environment(j.Environment),
		Command: string(j.Command),
		Image: container.Image{
			Repo: string(j.Image.Repo),
			ID:   j.Image.ID,
		},
	}

	// Schedule the job onto the cluster.
	if err := s.Schedule(container); err != nil {
		return nil, err
	}

	return JobsCreate(db, j)
}

func Unschedule(db Deleter, s container.Scheduler, j *Job) error {
	if err := s.Unschedule(j.ContainerName()); err != nil {
		return err
	}

	return JobsDestroy(db, j)
}

type JobStatesFinder interface {
	JobStatesByApp(*App) ([]*JobState, error)
}

type JobStatesService interface {
	JobStatesFinder
}

type jobStatesService struct {
	DB
	JobsService
	scheduler container.Scheduler
}

func (s *jobStatesService) JobStatesByApp(app *App) ([]*JobState, error) {
	// Jobs expected to be running
	jobs, err := s.JobsService.JobsList(JobsListQuery{
		App: app.Name,
	})
	if err != nil {
		return nil, err
	}

	// Job states for all existing jobs
	sjs, err := s.scheduler.ContainerStates()
	if err != nil {
		return nil, err
	}

	// Create a map for easy lookups
	jsm := make(map[string]*container.ContainerState, len(sjs))
	for _, js := range sjs {
		jsm[js.Name] = js
	}

	// Create JobState based on Jobs and container.ContainerStates
	js := make([]*JobState, len(jobs))
	for i, j := range jobs {
		s, ok := jsm[j.ContainerName()]

		machineID := "unknown"
		state := "unknown"
		if ok {
			machineID = s.MachineID
			state = s.State
		}

		js[i] = &JobState{
			Job:       j,
			Name:      j.ContainerName(),
			MachineID: machineID,
			State:     state,
		}
	}

	return js, nil
}
<commit_msg>Add SERVICE_NAME environment variable.<commit_after>package empire

import (
	"database/sql/driver"
	"fmt"
	"time"

	"github.com/remind101/empire/pkg/container"
	"gopkg.in/gorp.v1"
)

// JobID represents a unique identifier for a Job.
type JobID string

// Scan implements the sql.Scanner interface.
func (id *JobID) Scan(src interface{}) error {
	if src, ok := src.([]byte); ok {
		*id = JobID(src)
	}

	return nil
}

// Value implements the driver.Value interface.
func (id JobID) Value() (driver.Value, error) {
	return driver.Value(string(id)), nil
}

// Job represents a Job that was submitted to the scheduler.
type Job struct {
	ID JobID `db:"id"`

	AppName        `db:"app_id"`
	ReleaseVersion `db:"release_version"`
	ProcessType    `db:"process_type"`
	Instance       int `db:"instance"`

	Environment Vars  `db:"environment"`
	Image       Image `db:"image"`
	Command     `db:"command"`

	// UpdatedAt indicates when this job last changed state.
	UpdatedAt time.Time `db:"updated_at"`
}

// PreInsert implements a pre insert hook for the db interface
func (j *Job) PreInsert(s gorp.SqlExecutor) error {
	j.UpdatedAt = Now()
	return nil
}

func (j *Job) ContainerName() string {
	return newContainerName(
		j.AppName,
		j.ReleaseVersion,
		j.ProcessType,
		j.Instance,
	)
}

// JobState represents the state of a submitted job.
type JobState struct {
	Job       *Job
	MachineID string
	Name      string
	State     string
}

// Schedule is an interface that represents something that can schedule jobs
// onto the cluster.
type Scheduler interface {
	Schedule(...*Job) error
	Unschedule(...*Job) error
}

type JobsFinder interface {
	JobsList(JobsListQuery) ([]*Job, error)
}

type JobsService interface {
	Scheduler
	JobsFinder
}

type jobsService struct {
	DB
	scheduler container.Scheduler
}

func (s *jobsService) JobsList(q JobsListQuery) ([]*Job, error) {
	return JobsList(s.DB, q)
}

func (s *jobsService) Schedule(jobs ...*Job) error {
	for _, j := range jobs {
		if _, err := Schedule(s.DB, s.scheduler, j); err != nil {
			return err
		}
	}

	return nil
}

func (s *jobsService) Unschedule(jobs ...*Job) error {
	for _, j := range jobs {
		if err := Unschedule(s.DB, s.scheduler, j); err != nil {
			return err
		}
	}

	return nil
}

// JobsCreate inserts the Job into the database.
func JobsCreate(db Inserter, job *Job) (*Job, error) {
	return job, db.Insert(job)
}

// JobsDestroy removes a Job from the database.
func JobsDestroy(db Deleter, job *Job) error {
	_, err := db.Delete(job)
	return err
}

// JobsListQuery is a query object to filter results from JobsRepository List.
type JobsListQuery struct {
	App     AppName
	Release ReleaseVersion
}

// JobsList returns a filtered list of Jobs.
func JobsList(db Queryier, q JobsListQuery) ([]*Job, error) {
	var jobs []*Job
	query := `select * from jobs where (app_id = $1 OR $1 = '') and (release_version = $2 OR $2 = 0)`
	return jobs, db.Select(&jobs, query, string(q.App), int(q.Release))
}

// Schedule schedules to job onto the cluster, then persists it to the database.
func Schedule(db Inserter, s container.Scheduler, j *Job) (*Job, error) {
	env := environment(j.Environment)
	env["SERVICE_NAME"] = fmt.Sprintf("%s/%s", j.ProcessType, j.AppName)

	container := &container.Container{
		Name:    j.ContainerName(),
		Env:     env,
		Command: string(j.Command),
		Image: container.Image{
			Repo: string(j.Image.Repo),
			ID:   j.Image.ID,
		},
	}

	// Schedule the job onto the cluster.
	if err := s.Schedule(container); err != nil {
		return nil, err
	}

	return JobsCreate(db, j)
}

func Unschedule(db Deleter, s container.Scheduler, j *Job) error {
	if err := s.Unschedule(j.ContainerName()); err != nil {
		return err
	}

	return JobsDestroy(db, j)
}

type JobStatesFinder interface {
	JobStatesByApp(*App) ([]*JobState, error)
}

type JobStatesService interface {
	JobStatesFinder
}

type jobStatesService struct {
	DB
	JobsService
	scheduler container.Scheduler
}

func (s *jobStatesService) JobStatesByApp(app *App) ([]*JobState, error) {
	// Jobs expected to be running
	jobs, err := s.JobsService.JobsList(JobsListQuery{
		App: app.Name,
	})
	if err != nil {
		return nil, err
	}

	// Job states for all existing jobs
	sjs, err := s.scheduler.ContainerStates()
	if err != nil {
		return nil, err
	}

	// Create a map for easy lookups
	jsm := make(map[string]*container.ContainerState, len(sjs))
	for _, js := range sjs {
		jsm[js.Name] = js
	}

	// Create JobState based on Jobs and container.ContainerStates
	js := make([]*JobState, len(jobs))
	for i, j := range jobs {
		s, ok := jsm[j.ContainerName()]

		machineID := "unknown"
		state := "unknown"
		if ok {
			machineID = s.MachineID
			state = s.State
		}

		js[i] = &JobState{
			Job:       j,
			Name:      j.ContainerName(),
			MachineID: machineID,
			State:     state,
		}
	}

	return js, nil
}
<|endoftext|>


<commit_before>package importer

import (
	"crypto/rand"
	"path/filepath"
	"strings"

	"github.com/decaf-emu/huehuetenango/pkg/importer/schema"
	"github.com/decaf-emu/huehuetenango/pkg/models"
	"github.com/decaf-emu/huehuetenango/pkg/repository"
	"github.com/decaf-emu/huehuetenango/pkg/search"
	"github.com/oklog/ulid"
)

type Importer interface {
	ImportZIP(path string) error
	ImportSource(source Source) error
}

type repositoryImporter struct {
	repository repository.Repository
	index      search.Index
}

func NewImporter(repository repository.Repository, index search.Index) Importer {
	return &repositoryImporter{
		repository: repository,
		index:      index,
	}
}

func (i *repositoryImporter) ImportZIP(path string) error {
	source := NewZIPSource(path)
	if err := source.Open(); err != nil {
		return err
	}
	defer source.Close()
	return i.ImportSource(source)
}

func (i *repositoryImporter) ImportSource(source Source) error {
	sourceTitles, err := source.Titles()
	if err != nil {
		return err
	}

	for _, sourceTitle := range sourceTitles {
		if err := i.importTitle(sourceTitle); err != nil {
			return err
		}
	}

	if err := i.updateImportSources(); err != nil {
		return err
	}

	return nil
}

func (i *repositoryImporter) importTitle(sourceTitle *Title) error {
	if sourceTitle.App == nil {
		return nil
	}

	title := new(models.Title)
	sourceTitle.App.FillModel(title)
	if sourceTitle.Meta != nil {
		sourceTitle.Meta.FillModel(title)
	}
	if sourceTitle.COS != nil {
		sourceTitle.COS.FillModel(title)
	}

	if title.ID == models.SystemTitleID {
		if title.LongNameEnglish == "" {
			title.LongNameEnglish = "System Library"
		}
		if title.ShortNameEnglish == "" {
			title.ShortNameEnglish = "System Library"
		}
	}

	if err := i.repository.StoreTitle(title); err != nil {
		return err
	}

	existingRPLs, err := i.repository.ListRPLsByTitle(title.ID)
	if err != nil {
		return err
	}
	for _, existingRPL := range existingRPLs {
		exists := false

		for _, sourceRPL := range sourceTitle.RPLs {
			if existingRPL.Name == sourceRPL.Name {
				exists = true
				break
			}
		}

		if !exists {
			if err := i.repository.RemoveRPL(existingRPL.ID); err != nil {
				return err
			}
		}
	}

	for _, rpl := range sourceTitle.RPLs {
		if err := i.importRPL(title, rpl); err != nil {
			return err
		}
	}

	return i.index.IndexTitle(title)
}

func (i *repositoryImporter) importRPL(title *models.Title, sourceRPL *schema.RPL) error {
	rpl, err := i.repository.FindRPLByName(title.ID, sourceRPL.Name)
	if err != nil {
		return err
	}

	if rpl == nil {
		id, err := ulid.New(ulid.Now(), rand.Reader)
		if err != nil {
			return err
		}
		rpl = &models.RPL{
			ID:      models.RPLID(id.String()),
			TitleID: title.ID,
		}
	}

	sourceRPL.FillModel(rpl)

	if err := i.repository.StoreRPL(rpl); err != nil {
		return err
	}

	if err := i.importRPLExports(title, rpl, sourceRPL.Exports); err != nil {
		return err
	}

	if err := i.importRPLImports(title, rpl, sourceRPL.Imports); err != nil {
		return err
	}

	return nil
}

func (i *repositoryImporter) importRPLExports(title *models.Title, rpl *models.RPL,
	sourceExports *schema.Exports) error {

	if sourceExports == nil {
		existingExports, err := i.repository.ListExportsByRPL(rpl.ID)
		if err != nil {
			return err
		}
		for _, existingExport := range existingExports {
			if err := i.repository.RemoveExport(existingExport.ID); err != nil {
				return err
			}
		}
		return nil
	}

	dataMap := make(map[string]bool, len(sourceExports.Data))
	for _, dataExport := range sourceExports.Data {
		dataMap[dataExport] = true
	}
	functionMap := make(map[string]bool, len(sourceExports.Functions))
	for _, functionExport := range sourceExports.Functions {
		functionMap[functionExport] = true
	}

	existingExports, err := i.repository.ListExportsByRPL(rpl.ID)
	if err != nil {
		return err
	}

	for _, existing := range existingExports {
		if existing.Type == models.DataObject {
			if _, exists := dataMap[existing.Name]; exists {
				delete(dataMap, existing.Name)
			} else {
				i.repository.RemoveExport(existing.ID)
			}
		} else if existing.Type == models.FunctionObject {
			if _, exists := functionMap[existing.Name]; exists {
				delete(functionMap, existing.Name)
			} else {
				i.repository.RemoveExport(existing.ID)
			}
		}
	}

	exports := make([]*models.Export, 0, len(dataMap)+len(functionMap))
	for name := range dataMap {
		model, err := models.NewTempExport(name)
		if err != nil {
			return err
		}
		defer models.ReleaseTempExport(model)

		model.TitleID = rpl.TitleID
		model.RPLID = rpl.ID
		model.Type = models.DataObject
		exports = append(exports, model)
	}
	for name := range functionMap {
		model, err := models.NewTempExport(name)
		if err != nil {
			return err
		}
		defer models.ReleaseTempExport(model)

		model.TitleID = rpl.TitleID
		model.RPLID = rpl.ID
		model.Type = models.FunctionObject
		exports = append(exports, model)
	}
	if err := i.repository.StoreExports(exports); err != nil {
		return err
	}
	return nil
}

func (i *repositoryImporter) importRPLImports(title *models.Title, rpl *models.RPL,
	sourceImports []*schema.Imports) error {

	for _, sourceImport := range sourceImports {
		dataMap := make(map[string]bool, len(sourceImport.Data))
		for _, dataImport := range sourceImport.Data {
			dataMap[dataImport] = true
		}
		functionMap := make(map[string]bool, len(sourceImport.Functions))
		for _, functionImport := range sourceImport.Functions {
			functionMap[functionImport] = true
		}

		existingImports, err := i.repository.ListImportsBySourceName(rpl.ID, sourceImport.Name)
		if err != nil {
			return err
		}

		for _, existing := range existingImports {
			if existing.Type == models.DataObject {
				if _, exists := dataMap[existing.Name]; exists {
					delete(dataMap, existing.Name)
				} else {
					i.repository.RemoveImport(existing.ID)
				}
			} else if existing.Type == models.FunctionObject {
				if _, exists := functionMap[existing.Name]; exists {
					delete(functionMap, existing.Name)
				} else {
					i.repository.RemoveImport(existing.ID)
				}
			}
		}

		imports := make([]*models.Import, 0, len(dataMap)+len(functionMap))
		for name := range dataMap {
			model, err := models.NewTempImport(name)
			if err != nil {
				return err
			}
			defer models.ReleaseTempImport(model)

			model.TitleID = rpl.TitleID
			model.RPLID = rpl.ID
			model.Type = models.DataObject
			model.SourceName = sourceImport.Name
			imports = append(imports, model)
		}
		for name := range functionMap {
			model, err := models.NewTempImport(name)
			if err != nil {
				return err
			}
			defer models.ReleaseTempImport(model)

			model.TitleID = rpl.TitleID
			model.RPLID = rpl.ID
			model.Type = models.FunctionObject
			model.SourceName = sourceImport.Name
			imports = append(imports, model)
		}

		if err := i.repository.StoreImports(imports); err != nil {
			return err
		}
	}

	return nil
}

func (i *repositoryImporter) updateImportSources() error {
	titles, err := i.repository.ListTitles(true)
	if err != nil {
		return err
	}

	systemRPLs, err := i.repository.ListRPLsByTitle(models.SystemTitleID)
	if err != nil {
		return err
	}

	systemDataMap := make(map[string]*models.Export)
	systemFunctionMap := make(map[string]*models.Export)
	for _, rpl := range systemRPLs {
		exports, err := i.repository.ListExportsByRPL(rpl.ID)
		if err != nil {
			return err
		}
		for _, export := range exports {
			key := rpl.Name + "_" + export.Name
			if export.Type == models.DataObject {
				systemDataMap[key] = export
			} else if export.Type == models.FunctionObject {
				systemFunctionMap[key] = export
			}
		}
	}

	for _, title := range titles {
		rpls, err := i.repository.ListRPLsByTitle(title.ID)
		if err != nil {
			return err
		}

		titleDataMap := make(map[string]*models.Export)
		titleFunctionMap := make(map[string]*models.Export)
		for _, rpl := range rpls {
			exports, err := i.repository.ListExportsByRPL(rpl.ID)
			if err != nil {
				return err
			}
			for _, export := range exports {
				key := rpl.Name + "_" + export.Name
				if export.Type == models.DataObject {
					titleDataMap[key] = export
				} else if export.Type == models.FunctionObject {
					titleFunctionMap[key] = export
				}
			}
		}

		for _, rpl := range rpls {
			imports, err := i.repository.ListImportsByRPL(rpl.ID)
			if err != nil {
				return err
			}

			for _, rplImport := range imports {
				var sourceID models.RPLID
				var sourceTitleID models.TitleID
				sourceName := string(rplImport.SourceName)

				lookupSignatures := make([]string, 1)
				lookupSignatures[0] = sourceName + "_" + rplImport.Name

				// most import source names don't contain the .rpl extension
				// but some do, in which case we create another lookup signature
				// without the extension
				baseSourceName := strings.TrimSuffix(sourceName, filepath.Ext(sourceName))
				if baseSourceName != sourceName {
					lookupSignatures = append(lookupSignatures, baseSourceName+"_"+rplImport.Name)
				}

				for _, lookupSignature := range lookupSignatures {
					if rplImport.Type == models.DataObject {
						if match, exists := titleDataMap[lookupSignature]; exists {
							sourceID = match.RPLID
							sourceTitleID = match.TitleID
						} else if match, exists := systemDataMap[lookupSignature]; exists {
							sourceID = match.RPLID
							sourceTitleID = match.TitleID
						}
					} else if rplImport.Type == models.FunctionObject {
						if match, exists := titleFunctionMap[lookupSignature]; exists {
							sourceID = match.RPLID
							sourceTitleID = match.TitleID
						} else if match, exists := systemFunctionMap[lookupSignature]; exists {
							sourceID = match.RPLID
							sourceTitleID = match.TitleID
						}
					}
				}

				if sourceID != "" && sourceTitleID != 0 {
					rplImport.SourceID = sourceID
					rplImport.SourceTitleID = sourceTitleID
				}
			}

			if err := i.repository.StoreImports(imports); err != nil {
				return err
			}
		}
	}

	return nil
}
<commit_msg>Formatting<commit_after>package importer

import (
	"crypto/rand"
	"path/filepath"
	"strings"

	"github.com/decaf-emu/huehuetenango/pkg/importer/schema"
	"github.com/decaf-emu/huehuetenango/pkg/models"
	"github.com/decaf-emu/huehuetenango/pkg/repository"
	"github.com/decaf-emu/huehuetenango/pkg/search"
	"github.com/oklog/ulid"
)

type Importer interface {
	ImportZIP(path string) error
	ImportSource(source Source) error
}

type repositoryImporter struct {
	repository repository.Repository
	index      search.Index
}

func NewImporter(repository repository.Repository, index search.Index) Importer {
	return &repositoryImporter{
		repository: repository,
		index:      index,
	}
}

func (i *repositoryImporter) ImportZIP(path string) error {
	source := NewZIPSource(path)
	if err := source.Open(); err != nil {
		return err
	}
	defer source.Close()
	return i.ImportSource(source)
}

func (i *repositoryImporter) ImportSource(source Source) error {
	sourceTitles, err := source.Titles()
	if err != nil {
		return err
	}

	for _, sourceTitle := range sourceTitles {
		if err := i.importTitle(sourceTitle); err != nil {
			return err
		}
	}

	if err := i.updateImportSources(); err != nil {
		return err
	}

	return nil
}

func (i *repositoryImporter) importTitle(sourceTitle *Title) error {
	if sourceTitle.App == nil {
		return nil
	}

	title := new(models.Title)
	sourceTitle.App.FillModel(title)
	if sourceTitle.Meta != nil {
		sourceTitle.Meta.FillModel(title)
	}
	if sourceTitle.COS != nil {
		sourceTitle.COS.FillModel(title)
	}

	if title.ID == models.SystemTitleID {
		if title.LongNameEnglish == "" {
			title.LongNameEnglish = "System Library"
		}
		if title.ShortNameEnglish == "" {
			title.ShortNameEnglish = "System Library"
		}
	}

	if err := i.repository.StoreTitle(title); err != nil {
		return err
	}

	existingRPLs, err := i.repository.ListRPLsByTitle(title.ID)
	if err != nil {
		return err
	}
	for _, existingRPL := range existingRPLs {
		exists := false

		for _, sourceRPL := range sourceTitle.RPLs {
			if existingRPL.Name == sourceRPL.Name {
				exists = true
				break
			}
		}

		if !exists {
			if err := i.repository.RemoveRPL(existingRPL.ID); err != nil {
				return err
			}
		}
	}

	for _, rpl := range sourceTitle.RPLs {
		if err := i.importRPL(title, rpl); err != nil {
			return err
		}
	}

	return i.index.IndexTitle(title)
}

func (i *repositoryImporter) importRPL(title *models.Title, sourceRPL *schema.RPL) error {
	rpl, err := i.repository.FindRPLByName(title.ID, sourceRPL.Name)
	if err != nil {
		return err
	}

	if rpl == nil {
		id, err := ulid.New(ulid.Now(), rand.Reader)
		if err != nil {
			return err
		}
		rpl = &models.RPL{
			ID:      models.RPLID(id.String()),
			TitleID: title.ID,
		}
	}

	sourceRPL.FillModel(rpl)

	if err := i.repository.StoreRPL(rpl); err != nil {
		return err
	}

	if err := i.importRPLExports(title, rpl, sourceRPL.Exports); err != nil {
		return err
	}

	if err := i.importRPLImports(title, rpl, sourceRPL.Imports); err != nil {
		return err
	}

	return nil
}

func (i *repositoryImporter) importRPLExports(title *models.Title, rpl *models.RPL,
	sourceExports *schema.Exports) error {
	if sourceExports == nil {
		existingExports, err := i.repository.ListExportsByRPL(rpl.ID)
		if err != nil {
			return err
		}
		for _, existingExport := range existingExports {
			if err := i.repository.RemoveExport(existingExport.ID); err != nil {
				return err
			}
		}
		return nil
	}

	dataMap := make(map[string]bool, len(sourceExports.Data))
	for _, dataExport := range sourceExports.Data {
		dataMap[dataExport] = true
	}
	functionMap := make(map[string]bool, len(sourceExports.Functions))
	for _, functionExport := range sourceExports.Functions {
		functionMap[functionExport] = true
	}

	existingExports, err := i.repository.ListExportsByRPL(rpl.ID)
	if err != nil {
		return err
	}

	for _, existing := range existingExports {
		if existing.Type == models.DataObject {
			if _, exists := dataMap[existing.Name]; exists {
				delete(dataMap, existing.Name)
			} else {
				i.repository.RemoveExport(existing.ID)
			}
		} else if existing.Type == models.FunctionObject {
			if _, exists := functionMap[existing.Name]; exists {
				delete(functionMap, existing.Name)
			} else {
				i.repository.RemoveExport(existing.ID)
			}
		}
	}

	exports := make([]*models.Export, 0, len(dataMap)+len(functionMap))
	for name := range dataMap {
		model, err := models.NewTempExport(name)
		if err != nil {
			return err
		}
		defer models.ReleaseTempExport(model)

		model.TitleID = rpl.TitleID
		model.RPLID = rpl.ID
		model.Type = models.DataObject
		exports = append(exports, model)
	}
	for name := range functionMap {
		model, err := models.NewTempExport(name)
		if err != nil {
			return err
		}
		defer models.ReleaseTempExport(model)

		model.TitleID = rpl.TitleID
		model.RPLID = rpl.ID
		model.Type = models.FunctionObject
		exports = append(exports, model)
	}
	if err := i.repository.StoreExports(exports); err != nil {
		return err
	}
	return nil
}

func (i *repositoryImporter) importRPLImports(title *models.Title, rpl *models.RPL,
	sourceImports []*schema.Imports) error {

	for _, sourceImport := range sourceImports {
		dataMap := make(map[string]bool, len(sourceImport.Data))
		for _, dataImport := range sourceImport.Data {
			dataMap[dataImport] = true
		}
		functionMap := make(map[string]bool, len(sourceImport.Functions))
		for _, functionImport := range sourceImport.Functions {
			functionMap[functionImport] = true
		}

		existingImports, err := i.repository.ListImportsBySourceName(rpl.ID, sourceImport.Name)
		if err != nil {
			return err
		}

		for _, existing := range existingImports {
			if existing.Type == models.DataObject {
				if _, exists := dataMap[existing.Name]; exists {
					delete(dataMap, existing.Name)
				} else {
					i.repository.RemoveImport(existing.ID)
				}
			} else if existing.Type == models.FunctionObject {
				if _, exists := functionMap[existing.Name]; exists {
					delete(functionMap, existing.Name)
				} else {
					i.repository.RemoveImport(existing.ID)
				}
			}
		}

		imports := make([]*models.Import, 0, len(dataMap)+len(functionMap))
		for name := range dataMap {
			model, err := models.NewTempImport(name)
			if err != nil {
				return err
			}
			defer models.ReleaseTempImport(model)

			model.TitleID = rpl.TitleID
			model.RPLID = rpl.ID
			model.Type = models.DataObject
			model.SourceName = sourceImport.Name
			imports = append(imports, model)
		}
		for name := range functionMap {
			model, err := models.NewTempImport(name)
			if err != nil {
				return err
			}
			defer models.ReleaseTempImport(model)

			model.TitleID = rpl.TitleID
			model.RPLID = rpl.ID
			model.Type = models.FunctionObject
			model.SourceName = sourceImport.Name
			imports = append(imports, model)
		}

		if err := i.repository.StoreImports(imports); err != nil {
			return err
		}
	}

	return nil
}

func (i *repositoryImporter) updateImportSources() error {
	titles, err := i.repository.ListTitles(true)
	if err != nil {
		return err
	}

	systemRPLs, err := i.repository.ListRPLsByTitle(models.SystemTitleID)
	if err != nil {
		return err
	}

	systemDataMap := make(map[string]*models.Export)
	systemFunctionMap := make(map[string]*models.Export)
	for _, rpl := range systemRPLs {
		exports, err := i.repository.ListExportsByRPL(rpl.ID)
		if err != nil {
			return err
		}
		for _, export := range exports {
			key := rpl.Name + "_" + export.Name
			if export.Type == models.DataObject {
				systemDataMap[key] = export
			} else if export.Type == models.FunctionObject {
				systemFunctionMap[key] = export
			}
		}
	}

	for _, title := range titles {
		rpls, err := i.repository.ListRPLsByTitle(title.ID)
		if err != nil {
			return err
		}

		titleDataMap := make(map[string]*models.Export)
		titleFunctionMap := make(map[string]*models.Export)
		for _, rpl := range rpls {
			exports, err := i.repository.ListExportsByRPL(rpl.ID)
			if err != nil {
				return err
			}
			for _, export := range exports {
				key := rpl.Name + "_" + export.Name
				if export.Type == models.DataObject {
					titleDataMap[key] = export
				} else if export.Type == models.FunctionObject {
					titleFunctionMap[key] = export
				}
			}
		}

		for _, rpl := range rpls {
			imports, err := i.repository.ListImportsByRPL(rpl.ID)
			if err != nil {
				return err
			}

			for _, rplImport := range imports {
				var sourceID models.RPLID
				var sourceTitleID models.TitleID
				sourceName := string(rplImport.SourceName)

				lookupSignatures := make([]string, 1)
				lookupSignatures[0] = sourceName + "_" + rplImport.Name

				// most import source names don't contain the .rpl extension
				// but some do, in which case we create another lookup signature
				// without the extension
				baseSourceName := strings.TrimSuffix(sourceName, filepath.Ext(sourceName))
				if baseSourceName != sourceName {
					lookupSignatures = append(lookupSignatures, baseSourceName+"_"+rplImport.Name)
				}

				for _, lookupSignature := range lookupSignatures {
					if rplImport.Type == models.DataObject {
						if match, exists := titleDataMap[lookupSignature]; exists {
							sourceID = match.RPLID
							sourceTitleID = match.TitleID
						} else if match, exists := systemDataMap[lookupSignature]; exists {
							sourceID = match.RPLID
							sourceTitleID = match.TitleID
						}
					} else if rplImport.Type == models.FunctionObject {
						if match, exists := titleFunctionMap[lookupSignature]; exists {
							sourceID = match.RPLID
							sourceTitleID = match.TitleID
						} else if match, exists := systemFunctionMap[lookupSignature]; exists {
							sourceID = match.RPLID
							sourceTitleID = match.TitleID
						}
					}
				}

				if sourceID != "" && sourceTitleID != 0 {
					rplImport.SourceID = sourceID
					rplImport.SourceTitleID = sourceTitleID
				}
			}

			if err := i.repository.StoreImports(imports); err != nil {
				return err
			}
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/devfeel/dotweb"
	"net/http"
	"strconv"
	"time"
)

func main() {
	//初始化DotServer
	app := dotweb.New()

	//设置dotserver日志目录
	//如果不设置，默认不启用，且默认为当前目录
	app.SetEnabledLog(true)

	//开启development模式
	app.SetDevelopmentMode()

	//设置gzip开关
	//app.SetEnabledGzip(true)

	//设置路由
	InitRoute(app.HttpServer)

	//InitModule(app)

	app.UseRequestLog()
	app.Use(
		NewAccessFmtLog("app"),
	//NewSimpleAuth("admin"),
	)

	//启动 监控服务
	app.SetPProfConfig(true, 8081)

	//全局容器
	app.AppContext.Set("gstring", "gvalue")
	app.AppContext.Set("gint", 1)

	// 开始服务
	port := 8080
	fmt.Println("dotweb.StartServer => " + strconv.Itoa(port))
	err := app.StartServer(port)
	fmt.Println("dotweb.StartServer error => ", err)
}

func Index(ctx *dotweb.HttpContext) {
	ctx.Response.Header().Set("Content-Type", "text/html; charset=utf-8")
	fmt.Println(time.Now(), "Index Handler")
	ctx.WriteString("index => ", ctx.RouterParams)
}

func DefaultError(ctx *dotweb.HttpContext) {
	panic("my panic error!")
}

func Redirect(ctx *dotweb.HttpContext) {
	ctx.Redirect(http.StatusMovedPermanently, "http://www.baidu.com")
}

func InitRoute(server *dotweb.HttpServer) {
	server.Router().GET("/", Index)
	server.Router().GET("/use", Index).Use(NewAccessFmtLog("Router-use"))

	g := server.Group("/group").Use(NewAccessFmtLog("group"))
	g.GET("/", Index)
	g.GET("/use", Index).Use(NewAccessFmtLog("group-use"))
}

func InitModule(dotserver *dotweb.DotWeb) {
	dotserver.RegisterModule(&dotweb.HttpModule{
		OnBeginRequest: func(ctx *dotweb.HttpContext) {
			fmt.Println(time.Now(), "HttpModule BeginRequest1:", ctx.Request.RequestURI)
		},
		OnEndRequest: func(ctx *dotweb.HttpContext) {
			fmt.Println(time.Now(), "HttpModule EndRequest1:", ctx.Request.RequestURI)
		},
	})

	dotserver.RegisterModule(&dotweb.HttpModule{
		OnBeginRequest: func(ctx *dotweb.HttpContext) {
			fmt.Println(time.Now(), "HttpModule BeginRequest2:", ctx.Request.RequestURI)
		},
	})
	dotserver.RegisterModule(&dotweb.HttpModule{
		OnEndRequest: func(ctx *dotweb.HttpContext) {
			fmt.Println(time.Now(), "HttpModule EndRequest3:", ctx.Request.RequestURI)
		},
	})
}

type AccessFmtLog struct {
	dotweb.BaseMiddlware
	Index string
}

func (m *AccessFmtLog) Handle(ctx *dotweb.HttpContext) error {
	fmt.Println(time.Now(), "[AccessFmtLog ", m.Index, "] begin request -> ", ctx.Request.RequestURI)
	err := m.Next(ctx)
	fmt.Println(time.Now(), "[AccessFmtLog ", m.Index, "] finish request ", err, " -> ", ctx.Request.RequestURI)
	return err
}

func NewAccessFmtLog(index string) *AccessFmtLog {
	return &AccessFmtLog{Index: index}
}

type SimpleAuth struct {
	dotweb.BaseMiddlware
	exactToken string
}

func (m *SimpleAuth) Handle(ctx *dotweb.HttpContext) error {
	fmt.Println(time.Now(), "[SimpleAuth] begin request -> ", ctx.Request.RequestURI)
	var err error
	if ctx.QueryString("token") != m.exactToken {
		ctx.Write(http.StatusUnauthorized, []byte("sorry, Unauthorized"))
	} else {
		err = m.Next(ctx)
	}
	fmt.Println(time.Now(), "[SimpleAuth] finish request ", err, " -> ", ctx.Request.RequestURI)
	return err
}

func NewSimpleAuth(exactToken string) *SimpleAuth {
	return &SimpleAuth{exactToken: exactToken}
}
<commit_msg>Version 0.3.13 * HttpContext增加ViewC、WriteStringC、WriteBlobC、WriteJsonC、WriteJsonBlobC方法，以支持传入指定HttpCode * 调整Redirect(code int, targetUrl string) 为 Redirect(code int, targetUrl string) error，增加error返回值 * DotWeb增加SetLogger方法，支持接入第三方Logger，需实现logger.AppLog接口 * DotWeb增加SetConfig方法，支持配置与启动分离 * 新增 example/logger 目录 * 2017-05-16 09:00<commit_after>package main

import (
	"fmt"
	"github.com/devfeel/dotweb"
	"net/http"
	"strconv"
	"time"
)

func main() {
	//初始化DotServer
	app := dotweb.New()

	//设置dotserver日志目录
	//如果不设置，默认不启用，且默认为当前目录
	app.SetEnabledLog(true)

	//开启development模式
	app.SetDevelopmentMode()

	//设置gzip开关
	//app.SetEnabledGzip(true)

	//设置路由
	InitRoute(app.HttpServer)

	//InitModule(app)

	app.UseRequestLog()
	app.Use(
		NewAccessFmtLog("app"),
	//NewSimpleAuth("admin"),
	)

	//启动 监控服务
	app.SetPProfConfig(true, 8081)

	//全局容器
	app.AppContext.Set("gstring", "gvalue")
	app.AppContext.Set("gint", 1)

	// 开始服务
	port := 8080
	fmt.Println("dotweb.StartServer => " + strconv.Itoa(port))
	err := app.StartServer(port)
	fmt.Println("dotweb.StartServer error => ", err)
}

func Index(ctx *dotweb.HttpContext) {
	ctx.Response.Header().Set("Content-Type", "text/html; charset=utf-8")
	//fmt.Println(time.Now(), "Index Handler")
	ctx.WriteString("index  => ", fmt.Sprint(ctx.RouterNode.Middlewares()))
}

func DefaultError(ctx *dotweb.HttpContext) {
	panic("my panic error!")
}

func Redirect(ctx *dotweb.HttpContext) {
	ctx.Redirect(http.StatusMovedPermanently, "http://www.baidu.com")
}

func InitRoute(server *dotweb.HttpServer) {
	server.Router().GET("/", Index)
	server.Router().GET("/use", Index).Use(NewAccessFmtLog("Router-use"))

	g := server.Group("/group").Use(NewAccessFmtLog("group")).Use(NewSimpleAuth("admin"))
	g.GET("/", Index)
	g.GET("/use", Index).Use(NewAccessFmtLog("group-use"))
}

func InitModule(dotserver *dotweb.DotWeb) {
	dotserver.RegisterModule(&dotweb.HttpModule{
		OnBeginRequest: func(ctx *dotweb.HttpContext) {
			fmt.Println(time.Now(), "HttpModule BeginRequest1:", ctx.Request.RequestURI)
		},
		OnEndRequest: func(ctx *dotweb.HttpContext) {
			fmt.Println(time.Now(), "HttpModule EndRequest1:", ctx.Request.RequestURI)
		},
	})

	dotserver.RegisterModule(&dotweb.HttpModule{
		OnBeginRequest: func(ctx *dotweb.HttpContext) {
			fmt.Println(time.Now(), "HttpModule BeginRequest2:", ctx.Request.RequestURI)
		},
	})
	dotserver.RegisterModule(&dotweb.HttpModule{
		OnEndRequest: func(ctx *dotweb.HttpContext) {
			fmt.Println(time.Now(), "HttpModule EndRequest3:", ctx.Request.RequestURI)
		},
	})
}

type AccessFmtLog struct {
	dotweb.BaseMiddlware
	Index string
}

func (m *AccessFmtLog) Handle(ctx *dotweb.HttpContext) error {
	fmt.Println(time.Now(), "[AccessFmtLog ", m.Index, "] begin request -> ", ctx.Request.RequestURI)
	err := m.Next(ctx)
	fmt.Println(time.Now(), "[AccessFmtLog ", m.Index, "] finish request ", err, " -> ", ctx.Request.RequestURI)
	return err
}

func NewAccessFmtLog(index string) *AccessFmtLog {
	return &AccessFmtLog{Index: index}
}

type SimpleAuth struct {
	dotweb.BaseMiddlware
	exactToken string
}

func (m *SimpleAuth) Handle(ctx *dotweb.HttpContext) error {
	fmt.Println(time.Now(), "[SimpleAuth] begin request -> ", ctx.Request.RequestURI)
	var err error
	if ctx.QueryString("token") != m.exactToken {
		ctx.Write(http.StatusUnauthorized, []byte("sorry, Unauthorized"))
	} else {
		err = m.Next(ctx)
	}
	fmt.Println(time.Now(), "[SimpleAuth] finish request ", err, " -> ", ctx.Request.RequestURI)
	return err
}

func NewSimpleAuth(exactToken string) *SimpleAuth {
	return &SimpleAuth{exactToken: exactToken}
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package server

import (
	"encoding/base64"
	"net/http"
	"strings"

	"github.com/containerd/containerd"
	"github.com/containerd/containerd/errdefs"
	containerdimages "github.com/containerd/containerd/images"
	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"golang.org/x/net/context"
	runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"

	containerdresolver "github.com/containerd/cri/pkg/containerd/resolver"
	imagestore "github.com/containerd/cri/pkg/store/image"
	"github.com/containerd/cri/pkg/util"
)

// For image management:
// 1) We have an in-memory metadata index to:
//   a. Maintain ImageID -> RepoTags, ImageID -> RepoDigset relationships; ImageID
//   is the digest of image config, which conforms to oci image spec.
//   b. Cache constant and useful information such as image chainID, config etc.
//   c. An image will be added into the in-memory metadata only when it's successfully
//   pulled and unpacked.
//
// 2) We use containerd image metadata store and content store:
//   a. To resolve image reference (digest/tag) locally. During pulling image, we
//   normalize the image reference provided by user, and put it into image metadata
//   store with resolved descriptor. For the other operations, if image id is provided,
//   we'll access the in-memory metadata index directly; if image reference is
//   provided, we'll normalize it, resolve it in containerd image metadata store
//   to get the image id.
//   b. As the backup of in-memory metadata in 1). During startup, the in-memory
//   metadata could be re-constructed from image metadata store + content store.
//
// Several problems with current approach:
// 1) An entry in containerd image metadata store doesn't mean a "READY" (successfully
// pulled and unpacked) image. E.g. during pulling, the client gets killed. In that case,
// if we saw an image without snapshots or with in-complete contents during startup,
// should we re-pull the image? Or should we remove the entry?
//
// yanxuean: We cann't delete image directly, because we don't know if the image
// is pulled by us. There are resource leakage.
//
// 2) Containerd suggests user to add entry before pulling the image. However if
// an error occurrs during the pulling, should we remove the entry from metadata
// store? Or should we leave it there until next startup (resource leakage)?
//
// 3) The cri plugin only exposes "READY" (successfully pulled and unpacked) images
// to the user, which are maintained in the in-memory metadata index. However, it's
// still possible that someone else removes the content or snapshot by-pass the cri plugin,
// how do we detect that and update the in-memory metadata correspondingly? Always
// check whether corresponding snapshot is ready when reporting image status?
//
// 4) Is the content important if we cached necessary information in-memory
// after we pull the image? How to manage the disk usage of contents? If some
// contents are missing but snapshots are ready, is the image still "READY"?

// PullImage pulls an image with authentication config.
func (c *criService) PullImage(ctx context.Context, r *runtime.PullImageRequest) (*runtime.PullImageResponse, error) {
	imageRef := r.GetImage().GetImage()
	namedRef, err := util.NormalizeImageRef(imageRef)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to parse image reference %q", imageRef)
	}
	ref := namedRef.String()
	if ref != imageRef {
		logrus.Debugf("PullImage using normalized image ref: %q", ref)
	}
	resolver := containerdresolver.NewResolver(containerdresolver.Options{
		Credentials: func(string) (string, string, error) { return ParseAuth(r.GetAuth()) },
		Client:      http.DefaultClient,
		Registry:    c.getResolverOptions(),
	})
	_, desc, err := resolver.Resolve(ctx, ref)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to resolve image %q", ref)
	}
	// We have to check schema1 here, because after `Pull`, schema1
	// image has already been converted.
	isSchema1 := desc.MediaType == containerdimages.MediaTypeDockerSchema1Manifest

	image, err := c.client.Pull(ctx, ref,
		containerd.WithSchema1Conversion,
		containerd.WithResolver(resolver),
		containerd.WithPullSnapshotter(c.config.ContainerdConfig.Snapshotter),
		containerd.WithPullUnpack,
	)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to pull and unpack image %q", ref)
	}

	// Get image information.
	info, err := getImageInfo(ctx, image)
	if err != nil {
		return nil, errors.Wrap(err, "failed to get image information")
	}
	imageID := info.id

	repoDigest, repoTag := getRepoDigestAndTag(namedRef, image.Target().Digest, isSchema1)
	for _, r := range []string{repoTag, repoDigest, imageID} {
		if r == "" {
			continue
		}
		if err := c.createImageReference(ctx, r, image.Target()); err != nil {
			return nil, errors.Wrapf(err, "failed to update image reference %q", r)
		}
	}

	logrus.Debugf("Pulled image %q with image id %q, repo tag %q, repo digest %q", imageRef, imageID,
		repoTag, repoDigest)
	img := imagestore.Image{
		ID:        imageID,
		ChainID:   info.chainID.String(),
		Size:      info.size,
		ImageSpec: info.imagespec,
		Image:     image,
	}
	if repoDigest != "" {
		img.RepoDigests = []string{repoDigest}
	}
	if repoTag != "" {
		img.RepoTags = []string{repoTag}
	}

	if err := c.imageStore.Add(img); err != nil {
		return nil, errors.Wrapf(err, "failed to add image %q into store", img.ID)
	}

	// NOTE(random-liu): the actual state in containerd is the source of truth, even we maintain
	// in-memory image store, it's only for in-memory indexing. The image could be removed
	// by someone else anytime, before/during/after we create the metadata. We should always
	// check the actual state in containerd before using the image or returning status of the
	// image.
	return &runtime.PullImageResponse{ImageRef: img.ID}, nil
}

// ParseAuth parses AuthConfig and returns username and password/secret required by containerd.
func ParseAuth(auth *runtime.AuthConfig) (string, string, error) {
	if auth == nil {
		return "", "", nil
	}
	if auth.Username != "" {
		return auth.Username, auth.Password, nil
	}
	if auth.IdentityToken != "" {
		return "", auth.IdentityToken, nil
	}
	if auth.Auth != "" {
		decLen := base64.StdEncoding.DecodedLen(len(auth.Auth))
		decoded := make([]byte, decLen)
		_, err := base64.StdEncoding.Decode(decoded, []byte(auth.Auth))
		if err != nil {
			return "", "", err
		}
		fields := strings.SplitN(string(decoded), ":", 2)
		if len(fields) != 2 {
			return "", "", errors.Errorf("invalid decoded auth: %q", decoded)
		}
		user, passwd := fields[0], fields[1]
		return user, strings.Trim(passwd, "\x00"), nil
	}
	// TODO(random-liu): Support RegistryToken.
	return "", "", errors.New("invalid auth config")
}

// createImageReference creates image reference inside containerd image store.
// Note that because create and update are not finished in one transaction, there could be race. E.g.
// the image reference is deleted by someone else after create returns already exists, but before update
// happens.
func (c *criService) createImageReference(ctx context.Context, name string, desc imagespec.Descriptor) error {
	img := containerdimages.Image{
		Name:   name,
		Target: desc,
	}
	// TODO(random-liu): Figure out which is the more performant sequence create then update or
	// update then create.
	_, err := c.client.ImageService().Create(ctx, img)
	if err == nil {
		return nil
	}
	if err != nil && !errdefs.IsAlreadyExists(err) {
		return err
	}
	_, err = c.client.ImageService().Update(ctx, img, "target")
	return err
}

func (c *criService) getResolverOptions() map[string][]string {
	options := make(map[string][]string)
	for ns, mirror := range c.config.Mirrors {
		options[ns] = append(options[ns], mirror.Endpoints...)
	}
	return options
}
<commit_msg>Remove useless error-check in createImageReference<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package server

import (
	"encoding/base64"
	"net/http"
	"strings"

	"github.com/containerd/containerd"
	"github.com/containerd/containerd/errdefs"
	containerdimages "github.com/containerd/containerd/images"
	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"golang.org/x/net/context"
	runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"

	containerdresolver "github.com/containerd/cri/pkg/containerd/resolver"
	imagestore "github.com/containerd/cri/pkg/store/image"
	"github.com/containerd/cri/pkg/util"
)

// For image management:
// 1) We have an in-memory metadata index to:
//   a. Maintain ImageID -> RepoTags, ImageID -> RepoDigset relationships; ImageID
//   is the digest of image config, which conforms to oci image spec.
//   b. Cache constant and useful information such as image chainID, config etc.
//   c. An image will be added into the in-memory metadata only when it's successfully
//   pulled and unpacked.
//
// 2) We use containerd image metadata store and content store:
//   a. To resolve image reference (digest/tag) locally. During pulling image, we
//   normalize the image reference provided by user, and put it into image metadata
//   store with resolved descriptor. For the other operations, if image id is provided,
//   we'll access the in-memory metadata index directly; if image reference is
//   provided, we'll normalize it, resolve it in containerd image metadata store
//   to get the image id.
//   b. As the backup of in-memory metadata in 1). During startup, the in-memory
//   metadata could be re-constructed from image metadata store + content store.
//
// Several problems with current approach:
// 1) An entry in containerd image metadata store doesn't mean a "READY" (successfully
// pulled and unpacked) image. E.g. during pulling, the client gets killed. In that case,
// if we saw an image without snapshots or with in-complete contents during startup,
// should we re-pull the image? Or should we remove the entry?
//
// yanxuean: We cann't delete image directly, because we don't know if the image
// is pulled by us. There are resource leakage.
//
// 2) Containerd suggests user to add entry before pulling the image. However if
// an error occurrs during the pulling, should we remove the entry from metadata
// store? Or should we leave it there until next startup (resource leakage)?
//
// 3) The cri plugin only exposes "READY" (successfully pulled and unpacked) images
// to the user, which are maintained in the in-memory metadata index. However, it's
// still possible that someone else removes the content or snapshot by-pass the cri plugin,
// how do we detect that and update the in-memory metadata correspondingly? Always
// check whether corresponding snapshot is ready when reporting image status?
//
// 4) Is the content important if we cached necessary information in-memory
// after we pull the image? How to manage the disk usage of contents? If some
// contents are missing but snapshots are ready, is the image still "READY"?

// PullImage pulls an image with authentication config.
func (c *criService) PullImage(ctx context.Context, r *runtime.PullImageRequest) (*runtime.PullImageResponse, error) {
	imageRef := r.GetImage().GetImage()
	namedRef, err := util.NormalizeImageRef(imageRef)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to parse image reference %q", imageRef)
	}
	ref := namedRef.String()
	if ref != imageRef {
		logrus.Debugf("PullImage using normalized image ref: %q", ref)
	}
	resolver := containerdresolver.NewResolver(containerdresolver.Options{
		Credentials: func(string) (string, string, error) { return ParseAuth(r.GetAuth()) },
		Client:      http.DefaultClient,
		Registry:    c.getResolverOptions(),
	})
	_, desc, err := resolver.Resolve(ctx, ref)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to resolve image %q", ref)
	}
	// We have to check schema1 here, because after `Pull`, schema1
	// image has already been converted.
	isSchema1 := desc.MediaType == containerdimages.MediaTypeDockerSchema1Manifest

	image, err := c.client.Pull(ctx, ref,
		containerd.WithSchema1Conversion,
		containerd.WithResolver(resolver),
		containerd.WithPullSnapshotter(c.config.ContainerdConfig.Snapshotter),
		containerd.WithPullUnpack,
	)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to pull and unpack image %q", ref)
	}

	// Get image information.
	info, err := getImageInfo(ctx, image)
	if err != nil {
		return nil, errors.Wrap(err, "failed to get image information")
	}
	imageID := info.id

	repoDigest, repoTag := getRepoDigestAndTag(namedRef, image.Target().Digest, isSchema1)
	for _, r := range []string{repoTag, repoDigest, imageID} {
		if r == "" {
			continue
		}
		if err := c.createImageReference(ctx, r, image.Target()); err != nil {
			return nil, errors.Wrapf(err, "failed to update image reference %q", r)
		}
	}

	logrus.Debugf("Pulled image %q with image id %q, repo tag %q, repo digest %q", imageRef, imageID,
		repoTag, repoDigest)
	img := imagestore.Image{
		ID:        imageID,
		ChainID:   info.chainID.String(),
		Size:      info.size,
		ImageSpec: info.imagespec,
		Image:     image,
	}
	if repoDigest != "" {
		img.RepoDigests = []string{repoDigest}
	}
	if repoTag != "" {
		img.RepoTags = []string{repoTag}
	}

	if err := c.imageStore.Add(img); err != nil {
		return nil, errors.Wrapf(err, "failed to add image %q into store", img.ID)
	}

	// NOTE(random-liu): the actual state in containerd is the source of truth, even we maintain
	// in-memory image store, it's only for in-memory indexing. The image could be removed
	// by someone else anytime, before/during/after we create the metadata. We should always
	// check the actual state in containerd before using the image or returning status of the
	// image.
	return &runtime.PullImageResponse{ImageRef: img.ID}, nil
}

// ParseAuth parses AuthConfig and returns username and password/secret required by containerd.
func ParseAuth(auth *runtime.AuthConfig) (string, string, error) {
	if auth == nil {
		return "", "", nil
	}
	if auth.Username != "" {
		return auth.Username, auth.Password, nil
	}
	if auth.IdentityToken != "" {
		return "", auth.IdentityToken, nil
	}
	if auth.Auth != "" {
		decLen := base64.StdEncoding.DecodedLen(len(auth.Auth))
		decoded := make([]byte, decLen)
		_, err := base64.StdEncoding.Decode(decoded, []byte(auth.Auth))
		if err != nil {
			return "", "", err
		}
		fields := strings.SplitN(string(decoded), ":", 2)
		if len(fields) != 2 {
			return "", "", errors.Errorf("invalid decoded auth: %q", decoded)
		}
		user, passwd := fields[0], fields[1]
		return user, strings.Trim(passwd, "\x00"), nil
	}
	// TODO(random-liu): Support RegistryToken.
	return "", "", errors.New("invalid auth config")
}

// createImageReference creates image reference inside containerd image store.
// Note that because create and update are not finished in one transaction, there could be race. E.g.
// the image reference is deleted by someone else after create returns already exists, but before update
// happens.
func (c *criService) createImageReference(ctx context.Context, name string, desc imagespec.Descriptor) error {
	img := containerdimages.Image{
		Name:   name,
		Target: desc,
	}
	// TODO(random-liu): Figure out which is the more performant sequence create then update or
	// update then create.
	_, err := c.client.ImageService().Create(ctx, img)
	if err == nil {
		return nil
	}
	if !errdefs.IsAlreadyExists(err) {
		return err
	}
	_, err = c.client.ImageService().Update(ctx, img, "target")
	return err
}

func (c *criService) getResolverOptions() map[string][]string {
	options := make(map[string][]string)
	for ns, mirror := range c.config.Mirrors {
		options[ns] = append(options[ns], mirror.Endpoints...)
	}
	return options
}
<|endoftext|>


<commit_before>package template

import (
	"fmt"
	"time"

	"github.com/docker/go-units"
)

// Metadata contains the information for a template.
type Metadata struct {
	Tag        string
	Repository string

	Created JSONTime
}

// String returns the string slice form of Metadata.
func (m Metadata) String() []string {
	tDelta := time.Now().Sub(time.Time(m.Created))
	return []string{m.Tag, m.Repository, units.HumanDuration(tDelta) + " ago"}
}

// JSONTime is time.Time with JSON marshaling and unmarshaling implementations.
type JSONTime time.Time

const (
	// "Mon, 02 Jan 2006 15:04:05 MST"
	timeFormat = time.RFC1123
)

// NewTime returns a new JSONTime containing the current time.
func NewTime() JSONTime {
	return JSONTime(time.Now())
}

// MarshalJSON marshals JSONTime to JSON.
func (t *JSONTime) MarshalJSON() ([]byte, error) {
	stamp := fmt.Sprintf(`"%s"`, time.Time(*t).Format(timeFormat))

	return []byte(stamp), nil
}

// UnmarshalJSON unmarshals JSON to JSONTime.
func (t *JSONTime) UnmarshalJSON(b []byte) error {
	time, err := time.Parse(timeFormat, string(b)[1:len(b)-1])
	if err != nil {
		return err
	}

	*t = JSONTime(time)

	return nil
}

// String returns the string form of JSONTime.
func (t JSONTime) String() string {
	return fmt.Sprintf("%s", time.Time(t).Format(timeFormat))
}
<commit_msg>pkg/template: change the flaky metadata time format<commit_after>package template

import (
	"fmt"
	"time"

	"github.com/docker/go-units"
)

// Metadata contains the information for a template.
type Metadata struct {
	Tag        string
	Repository string

	Created JSONTime
}

// String returns the string slice form of Metadata.
func (m Metadata) String() []string {
	tDelta := time.Now().Sub(time.Time(m.Created))
	return []string{m.Tag, m.Repository, units.HumanDuration(tDelta) + " ago"}
}

// JSONTime is time.Time with JSON marshaling and unmarshaling implementations.
type JSONTime time.Time

const (
	// "Mon, 02 Jan 2006 15:04:05 -0700"
	timeFormat = time.RFC1123Z
)

// NewTime returns a new JSONTime containing the current time.
func NewTime() JSONTime {
	return JSONTime(time.Now())
}

// MarshalJSON marshals JSONTime to JSON.
func (t *JSONTime) MarshalJSON() ([]byte, error) {
	stamp := fmt.Sprintf(`"%s"`, time.Time(*t).Format(timeFormat))

	return []byte(stamp), nil
}

// UnmarshalJSON unmarshals JSON to JSONTime.
func (t *JSONTime) UnmarshalJSON(b []byte) error {
	time, err := time.Parse(timeFormat, string(b)[1:len(b)-1])
	if err != nil {
		return err
	}

	*t = JSONTime(time)

	return nil
}

// String returns the string form of JSONTime.
func (t JSONTime) String() string {
	return fmt.Sprintf("%s", time.Time(t).Format(timeFormat))
}
<|endoftext|>


<commit_before>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2017 Red Hat, Inc.
 *
 */

package virt_api

import (
	"io/ioutil"
	"net/http"
	"os"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/ghttp"

	k8sv1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/util/cert"
	"k8s.io/client-go/util/cert/triple"
	apiregistrationv1beta1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1"

	v1 "kubevirt.io/kubevirt/pkg/api/v1"
	"kubevirt.io/kubevirt/pkg/kubecli"
	"kubevirt.io/kubevirt/pkg/log"
)

var _ = Describe("Virt-api", func() {
	var tmpDir string
	var server *ghttp.Server
	subresourceAggregatedApiName := v1.SubresourceGroupVersion.Version + "." + v1.SubresourceGroupName

	log.Log.SetIOWriter(GinkgoWriter)

	app := virtAPIApp{}
	BeforeEach(func() {
		tmpDir, err := ioutil.TempDir("", "api_tmp_dir")
		Expect(err).ToNot(HaveOccurred())
		server = ghttp.NewServer()
		app.virtCli, _ = kubecli.GetKubevirtClientFromFlags(server.URL(), "")
		app.certsDirectory = tmpDir
	})

	Context("Virt api server", func() {
		It("should generate certs the first time it is run", func(done Done) {
			server.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", "/api/v1/namespaces/kube-system/secrets/"+virtApiCertSecretName),
					ghttp.RespondWithJSONEncoded(http.StatusNotFound, nil),
				),
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("POST", "/api/v1/namespaces/kube-system/secrets"),
					ghttp.RespondWithJSONEncoded(http.StatusOK, nil),
				),
			)

			err := app.getSelfSignedCert()
			Expect(err).ToNot(HaveOccurred())

			Expect(len(app.signingCertBytes)).ToNot(Equal(0))
			Expect(len(app.certBytes)).ToNot(Equal(0))
			Expect(len(app.keyBytes)).ToNot(Equal(0))
			close(done)
		})

		It("should not generate certs if secret already exists", func(done Done) {
			caKeyPair, _ := triple.NewCA("kubevirt.io")
			keyPair, _ := triple.NewServerKeyPair(
				caKeyPair,
				"virt-api.kube-system.pod.cluster.local",
				"virt-api",
				"kube-system",
				"cluster.local",
				nil,
				nil,
			)
			keyBytes := cert.EncodePrivateKeyPEM(keyPair.Key)
			certBytes := cert.EncodeCertPEM(keyPair.Cert)
			signingCertBytes := cert.EncodeCertPEM(caKeyPair.Cert)
			secret := k8sv1.Secret{
				ObjectMeta: metav1.ObjectMeta{
					Name:      virtApiCertSecretName,
					Namespace: metav1.NamespaceSystem,
					Labels: map[string]string{
						v1.AppLabel: "virt-api-aggregator",
					},
				},
				Type: "Opaque",
				Data: map[string][]byte{
					certBytesValue:        certBytes,
					keyBytesValue:         keyBytes,
					signingCertBytesValue: signingCertBytes,
				},
			}

			server.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", "/api/v1/namespaces/kube-system/secrets/"+virtApiCertSecretName),
					ghttp.RespondWithJSONEncoded(http.StatusOK, secret),
				),
			)

			err := app.getSelfSignedCert()
			Expect(err).ToNot(HaveOccurred())
			Expect(app.signingCertBytes).To(Equal(signingCertBytes))
			Expect(app.certBytes).To(Equal(certBytes))
			Expect(app.keyBytes).To(Equal(keyBytes))
			close(done)
		})

		It("should return error if client CA doesn't exist", func(done Done) {
			server.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", "/api/v1/namespaces/kube-system/configmaps/extension-apiserver-authentication"),
					ghttp.RespondWithJSONEncoded(http.StatusNotFound, nil),
				),
			)

			err := app.getClientCert()
			Expect(err).To(HaveOccurred())

			close(done)
		})
		It("should retrieve client CA", func(done Done) {

			configMap := &k8sv1.ConfigMap{}
			configMap.Data = make(map[string]string)
			configMap.Data["client-ca-file"] = "fakedata"
			server.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", "/api/v1/namespaces/kube-system/configmaps/extension-apiserver-authentication"),
					ghttp.RespondWithJSONEncoded(http.StatusOK, configMap),
				),
			)

			err := app.getClientCert()
			Expect(err).ToNot(HaveOccurred())
			Expect(app.clientCABytes).To(Equal([]byte("fakedata")))
			close(done)
		})

		It("should create apiservice endpoint if one doesn't exist", func(done Done) {
			app.signingCertBytes = []byte("fake")
			apiService := &apiregistrationv1beta1.APIService{
				ObjectMeta: metav1.ObjectMeta{
					Name: subresourceAggregatedApiName,
					Labels: map[string]string{
						v1.AppLabel: "virt-api-aggregator",
					},
				},
				Spec: apiregistrationv1beta1.APIServiceSpec{
					Service: &apiregistrationv1beta1.ServiceReference{
						Namespace: "kube-system",
						Name:      "virt-api",
					},
					Group:                v1.SubresourceGroupName,
					Version:              v1.SubresourceGroupVersion.Version,
					CABundle:             app.signingCertBytes,
					GroupPriorityMinimum: 1000,
					VersionPriority:      15,
				},
			}
			server.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", "/api/v1/namespaces/kube-system/apiservices/"+subresourceAggregatedApiName),
					ghttp.RespondWithJSONEncoded(http.StatusNotFound, nil),
				),
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("POST", "/api/v1/namespaces/kube-system/apiservices"),
					ghttp.RespondWithJSONEncoded(http.StatusOK, apiService),
				),
			)
			app.createSubresourceApiservice()
			close(done)
		})

		It("should not create apiservice endpoint if one does exist", func(done Done) {
			app.signingCertBytes = []byte("fake")
			apiService := &apiregistrationv1beta1.APIService{
				ObjectMeta: metav1.ObjectMeta{
					Name: subresourceAggregatedApiName,
					Labels: map[string]string{
						v1.AppLabel: "virt-api-aggregator",
					},
				},
				Spec: apiregistrationv1beta1.APIServiceSpec{
					Service: &apiregistrationv1beta1.ServiceReference{
						Namespace: "kube-system",
						Name:      "virt-api",
					},
					Group:                v1.SubresourceGroupName,
					Version:              v1.SubresourceGroupVersion.Version,
					CABundle:             app.signingCertBytes,
					GroupPriorityMinimum: 1000,
					VersionPriority:      15,
				},
			}
			server.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", "/api/v1/namespaces/kube-system/apiservices/"+subresourceAggregatedApiName),
					ghttp.RespondWithJSONEncoded(http.StatusNotFound, apiService),
				),
			)
			app.createSubresourceApiservice()

			close(done)
		})

		It("should update apiservice endpoint if CA bundle changes", func(done Done) {
			app.signingCertBytes = []byte("fake")
			apiServiceDifferent := &apiregistrationv1beta1.APIService{
				ObjectMeta: metav1.ObjectMeta{
					Name: subresourceAggregatedApiName,
					Labels: map[string]string{
						v1.AppLabel: "virt-api-aggregator",
					},
				},
				Spec: apiregistrationv1beta1.APIServiceSpec{
					Service: &apiregistrationv1beta1.ServiceReference{
						Namespace: "kube-system",
						Name:      "virt-api",
					},
					Group:                v1.SubresourceGroupName,
					Version:              v1.SubresourceGroupVersion.Version,
					CABundle:             []byte("different"),
					GroupPriorityMinimum: 1000,
					VersionPriority:      15,
				},
			}
			apiServiceFixed := &apiregistrationv1beta1.APIService{
				ObjectMeta: metav1.ObjectMeta{
					Name: subresourceAggregatedApiName,
					Labels: map[string]string{
						v1.AppLabel: "virt-api-aggregator",
					},
				},
				Spec: apiregistrationv1beta1.APIServiceSpec{
					Service: &apiregistrationv1beta1.ServiceReference{
						Namespace: "kube-system",
						Name:      "virt-api",
					},
					Group:                v1.SubresourceGroupName,
					Version:              v1.SubresourceGroupVersion.Version,
					CABundle:             []byte("fake"),
					GroupPriorityMinimum: 1000,
					VersionPriority:      15,
				},
			}
			server.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", "/api/v1/namespaces/kube-system/apiservices/"+subresourceAggregatedApiName),
					ghttp.RespondWithJSONEncoded(http.StatusNotFound, apiServiceDifferent),
				),
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("PUT", "/api/v1/namespaces/kube-system/apiservices"),
					ghttp.RespondWithJSONEncoded(http.StatusOK, apiServiceFixed),
				),
			)
			app.createSubresourceApiservice()
			close(done)
		})
	})

	AfterEach(func() {
		server.Close()
		os.RemoveAll(tmpDir)
	})
})
<commit_msg>add timeouts for mock http server subresource tests<commit_after>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2017 Red Hat, Inc.
 *
 */

package virt_api

import (
	"io/ioutil"
	"net/http"
	"os"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/ghttp"

	k8sv1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/util/cert"
	"k8s.io/client-go/util/cert/triple"
	apiregistrationv1beta1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1"

	v1 "kubevirt.io/kubevirt/pkg/api/v1"
	"kubevirt.io/kubevirt/pkg/kubecli"
	"kubevirt.io/kubevirt/pkg/log"
)

var _ = Describe("Virt-api", func() {
	var tmpDir string
	var server *ghttp.Server
	subresourceAggregatedApiName := v1.SubresourceGroupVersion.Version + "." + v1.SubresourceGroupName

	log.Log.SetIOWriter(GinkgoWriter)

	app := virtAPIApp{}
	BeforeEach(func() {
		server = ghttp.NewServer()
		tmpDir, err := ioutil.TempDir("", "api_tmp_dir")
		Expect(err).ToNot(HaveOccurred())
		app.virtCli, _ = kubecli.GetKubevirtClientFromFlags(server.URL(), "")
		app.certsDirectory = tmpDir
	})

	Context("Virt api server", func() {
		It("should generate certs the first time it is run", func(done Done) {
			server.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", "/api/v1/namespaces/kube-system/secrets/"+virtApiCertSecretName),
					ghttp.RespondWithJSONEncoded(http.StatusNotFound, nil),
				),
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("POST", "/api/v1/namespaces/kube-system/secrets"),
					ghttp.RespondWithJSONEncoded(http.StatusOK, nil),
				),
			)

			err := app.getSelfSignedCert()
			Expect(err).ToNot(HaveOccurred())

			Expect(len(app.signingCertBytes)).ToNot(Equal(0))
			Expect(len(app.certBytes)).ToNot(Equal(0))
			Expect(len(app.keyBytes)).ToNot(Equal(0))
			close(done)
		}, 5)

		It("should not generate certs if secret already exists", func(done Done) {
			caKeyPair, _ := triple.NewCA("kubevirt.io")
			keyPair, _ := triple.NewServerKeyPair(
				caKeyPair,
				"virt-api.kube-system.pod.cluster.local",
				"virt-api",
				"kube-system",
				"cluster.local",
				nil,
				nil,
			)
			keyBytes := cert.EncodePrivateKeyPEM(keyPair.Key)
			certBytes := cert.EncodeCertPEM(keyPair.Cert)
			signingCertBytes := cert.EncodeCertPEM(caKeyPair.Cert)
			secret := k8sv1.Secret{
				ObjectMeta: metav1.ObjectMeta{
					Name:      virtApiCertSecretName,
					Namespace: metav1.NamespaceSystem,
					Labels: map[string]string{
						v1.AppLabel: "virt-api-aggregator",
					},
				},
				Type: "Opaque",
				Data: map[string][]byte{
					certBytesValue:        certBytes,
					keyBytesValue:         keyBytes,
					signingCertBytesValue: signingCertBytes,
				},
			}

			server.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", "/api/v1/namespaces/kube-system/secrets/"+virtApiCertSecretName),
					ghttp.RespondWithJSONEncoded(http.StatusOK, secret),
				),
			)

			err := app.getSelfSignedCert()
			Expect(err).ToNot(HaveOccurred())
			Expect(app.signingCertBytes).To(Equal(signingCertBytes))
			Expect(app.certBytes).To(Equal(certBytes))
			Expect(app.keyBytes).To(Equal(keyBytes))
			close(done)
		}, 5)

		It("should return error if client CA doesn't exist", func(done Done) {
			server.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", "/api/v1/namespaces/kube-system/configmaps/extension-apiserver-authentication"),
					ghttp.RespondWithJSONEncoded(http.StatusNotFound, nil),
				),
			)

			err := app.getClientCert()
			Expect(err).To(HaveOccurred())

			close(done)
		}, 5)

		It("should retrieve client CA", func(done Done) {

			configMap := &k8sv1.ConfigMap{}
			configMap.Data = make(map[string]string)
			configMap.Data["client-ca-file"] = "fakedata"
			server.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", "/api/v1/namespaces/kube-system/configmaps/extension-apiserver-authentication"),
					ghttp.RespondWithJSONEncoded(http.StatusOK, configMap),
				),
			)

			err := app.getClientCert()
			Expect(err).ToNot(HaveOccurred())
			Expect(app.clientCABytes).To(Equal([]byte("fakedata")))
			close(done)
		}, 5)

		It("should create apiservice endpoint if one doesn't exist", func(done Done) {
			app.signingCertBytes = []byte("fake")
			apiService := &apiregistrationv1beta1.APIService{
				ObjectMeta: metav1.ObjectMeta{
					Name: subresourceAggregatedApiName,
					Labels: map[string]string{
						v1.AppLabel: "virt-api-aggregator",
					},
				},
				Spec: apiregistrationv1beta1.APIServiceSpec{
					Service: &apiregistrationv1beta1.ServiceReference{
						Namespace: "kube-system",
						Name:      "virt-api",
					},
					Group:                v1.SubresourceGroupName,
					Version:              v1.SubresourceGroupVersion.Version,
					CABundle:             app.signingCertBytes,
					GroupPriorityMinimum: 1000,
					VersionPriority:      15,
				},
			}
			server.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", "/api/v1/namespaces/kube-system/apiservices/"+subresourceAggregatedApiName),
					ghttp.RespondWithJSONEncoded(http.StatusNotFound, nil),
				),
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("POST", "/api/v1/namespaces/kube-system/apiservices"),
					ghttp.RespondWithJSONEncoded(http.StatusOK, apiService),
				),
			)
			app.createSubresourceApiservice()
			close(done)
		}, 5)

		It("should not create apiservice endpoint if one does exist", func(done Done) {
			app.signingCertBytes = []byte("fake")
			apiService := &apiregistrationv1beta1.APIService{
				ObjectMeta: metav1.ObjectMeta{
					Name: subresourceAggregatedApiName,
					Labels: map[string]string{
						v1.AppLabel: "virt-api-aggregator",
					},
				},
				Spec: apiregistrationv1beta1.APIServiceSpec{
					Service: &apiregistrationv1beta1.ServiceReference{
						Namespace: "kube-system",
						Name:      "virt-api",
					},
					Group:                v1.SubresourceGroupName,
					Version:              v1.SubresourceGroupVersion.Version,
					CABundle:             app.signingCertBytes,
					GroupPriorityMinimum: 1000,
					VersionPriority:      15,
				},
			}
			server.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", "/api/v1/namespaces/kube-system/apiservices/"+subresourceAggregatedApiName),
					ghttp.RespondWithJSONEncoded(http.StatusNotFound, apiService),
				),
			)
			app.createSubresourceApiservice()

			close(done)
		}, 5)

		It("should update apiservice endpoint if CA bundle changes", func(done Done) {
			app.signingCertBytes = []byte("fake")
			apiServiceDifferent := &apiregistrationv1beta1.APIService{
				ObjectMeta: metav1.ObjectMeta{
					Name: subresourceAggregatedApiName,
					Labels: map[string]string{
						v1.AppLabel: "virt-api-aggregator",
					},
				},
				Spec: apiregistrationv1beta1.APIServiceSpec{
					Service: &apiregistrationv1beta1.ServiceReference{
						Namespace: "kube-system",
						Name:      "virt-api",
					},
					Group:                v1.SubresourceGroupName,
					Version:              v1.SubresourceGroupVersion.Version,
					CABundle:             []byte("different"),
					GroupPriorityMinimum: 1000,
					VersionPriority:      15,
				},
			}
			apiServiceFixed := &apiregistrationv1beta1.APIService{
				ObjectMeta: metav1.ObjectMeta{
					Name: subresourceAggregatedApiName,
					Labels: map[string]string{
						v1.AppLabel: "virt-api-aggregator",
					},
				},
				Spec: apiregistrationv1beta1.APIServiceSpec{
					Service: &apiregistrationv1beta1.ServiceReference{
						Namespace: "kube-system",
						Name:      "virt-api",
					},
					Group:                v1.SubresourceGroupName,
					Version:              v1.SubresourceGroupVersion.Version,
					CABundle:             []byte("fake"),
					GroupPriorityMinimum: 1000,
					VersionPriority:      15,
				},
			}
			server.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", "/api/v1/namespaces/kube-system/apiservices/"+subresourceAggregatedApiName),
					ghttp.RespondWithJSONEncoded(http.StatusNotFound, apiServiceDifferent),
				),
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("PUT", "/api/v1/namespaces/kube-system/apiservices"),
					ghttp.RespondWithJSONEncoded(http.StatusOK, apiServiceFixed),
				),
			)
			app.createSubresourceApiservice()
			close(done)
		}, 5)
	})

	AfterEach(func() {
		server.Close()
		os.RemoveAll(tmpDir)
	})
})
<|endoftext|>


<commit_before>package aws

import (
	"encoding/base64"
	"log"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/kms"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsKmsCiphertext() *schema.Resource {

	return &schema.Resource{
		Create: resourceAwsKmsCiphertextCreate,
		Read:   resourceAwsKmsCiphertextRead,
		Delete: resourceAwsKmsCiphertextDelete,

		Schema: map[string]*schema.Schema{
			"plaintext": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"key_id": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"context": {
				Type:     schema.TypeMap,
				Optional: true,
				Elem:     &schema.Schema{Type: schema.TypeString},
				ForceNew: true,
			},

			"ciphertext_blob": {
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func resourceAwsKmsCiphertextCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).kmsconn

	d.SetId(time.Now().UTC().String())

	req := &kms.EncryptInput{
		KeyId:     aws.String(d.Get("key_id").(string)),
		Plaintext: []byte(d.Get("plaintext").(string)),
	}

	if ec := d.Get("context"); ec != nil {
		req.EncryptionContext = stringMapToPointers(ec.(map[string]interface{}))
	}

	log.Printf("[DEBUG] KMS encrypt for key: %s", d.Get("key_id").(string))
	resp, err := conn.Encrypt(req)
	if err != nil {
		return err
	}

	d.Set("ciphertext_blob", base64.StdEncoding.EncodeToString(resp.CiphertextBlob))

	return nil
}

func resourceAwsKmsCiphertextDelete(d *schema.ResourceData, meta interface{}) error {
	d.SetId("")
	return nil
}

func resourceAwsKmsCiphertextRead(d *schema.ResourceData, meta interface{}) error {
	// If the input has changed, generate a new ciphertext_blob
	// This should never be the case since ForceNew set on all input.
	if d.HasChange("plaintext") || d.HasChange("key_id") || d.HasChange("context") {
		return resourceAwsKmsCiphertextCreate(d, meta)
	}
	return nil
}
<commit_msg>Mark the plaintext as sensitive<commit_after>package aws

import (
	"encoding/base64"
	"log"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/kms"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsKmsCiphertext() *schema.Resource {

	return &schema.Resource{
		Create: resourceAwsKmsCiphertextCreate,
		Read:   resourceAwsKmsCiphertextRead,
		Delete: resourceAwsKmsCiphertextDelete,

		Schema: map[string]*schema.Schema{
			"plaintext": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
				Sensitive: true,
			},

			"key_id": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"context": {
				Type:     schema.TypeMap,
				Optional: true,
				Elem:     &schema.Schema{Type: schema.TypeString},
				ForceNew: true,
			},

			"ciphertext_blob": {
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func resourceAwsKmsCiphertextCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).kmsconn

	d.SetId(time.Now().UTC().String())

	req := &kms.EncryptInput{
		KeyId:     aws.String(d.Get("key_id").(string)),
		Plaintext: []byte(d.Get("plaintext").(string)),
	}

	if ec := d.Get("context"); ec != nil {
		req.EncryptionContext = stringMapToPointers(ec.(map[string]interface{}))
	}

	log.Printf("[DEBUG] KMS encrypt for key: %s", d.Get("key_id").(string))
	resp, err := conn.Encrypt(req)
	if err != nil {
		return err
	}

	d.Set("ciphertext_blob", base64.StdEncoding.EncodeToString(resp.CiphertextBlob))

	return nil
}

func resourceAwsKmsCiphertextDelete(d *schema.ResourceData, meta interface{}) error {
	d.SetId("")
	return nil
}

func resourceAwsKmsCiphertextRead(d *schema.ResourceData, meta interface{}) error {
	// If the input has changed, generate a new ciphertext_blob
	// This should never be the case since ForceNew set on all input.
	if d.HasChange("plaintext") || d.HasChange("key_id") || d.HasChange("context") {
		return resourceAwsKmsCiphertextCreate(d, meta)
	}
	return nil
}
<|endoftext|>