<commit_before>package instructions

import (
    //"fmt"
    "jvmgo/rtda"
    rtc "jvmgo/rtda/class"
)

// Determine if object is of given type
type instanceof struct {Index16Instruction}
func (self *instanceof) Execute(thread *rtda.Thread) {
    frame := thread.CurrentFrame()
    stack := frame.OperandStack()
    ref := stack.PopRef()

    cp := frame.Method().Class().ConstantPool()
    cClass := cp.GetConstant(self.index).(*rtc.ConstantClass)
    class := cClass.Class()

    if class.NotInitialized() {
        // todo init class
        panic("class not initialized!" + class.Name())
    }

    // todo
    if _instanceof(ref, class) {
        stack.PushInt(1)
    } else {
        stack.PushInt(0)
    }
}

func _instanceof(ref *rtc.Obj, class *rtc.Class) (bool) {
    if ref == nil {
        return false
    } else {
        return ref.Class() == class
    }
}
<commit_msg>refactor instanceof<commit_after>package instructions

import (
    //"fmt"
    "jvmgo/rtda"
    rtc "jvmgo/rtda/class"
)

// Determine if object is of given type
type instanceof struct {Index16Instruction}
func (self *instanceof) Execute(frame *rtda.Frame) {
    stack := frame.OperandStack()
    ref := stack.PopRef()

    cp := frame.Method().Class().ConstantPool()
    cClass := cp.GetConstant(self.index).(*rtc.ConstantClass)
    class := cClass.Class()

    if class.NotInitialized() {
        // todo init class
        panic("class not initialized!" + class.Name())
    }

    // todo
    if _instanceof(ref, class) {
        stack.PushInt(1)
    } else {
        stack.PushInt(0)
    }
}

func _instanceof(ref *rtc.Obj, class *rtc.Class) (bool) {
    if ref == nil {
        return false
    } else {
        return ref.Class() == class
    }
}
<|endoftext|>


<commit_before><commit_msg>Fixed bug in e2e test<commit_after><|endoftext|>


<commit_before><commit_msg>Ensure NetIn test uses unique host port<commit_after><|endoftext|>


<commit_before><commit_msg>Update group_labels.go<commit_after><|endoftext|>


<commit_before>package requests

import (
	"io/ioutil"
	"net/http"
	"sync"

	"github.com/mschwager/riplink/src/parse"
	"github.com/mschwager/riplink/src/rpurl"
)

type Client interface {
	Do(*http.Request) (*http.Response, error)
}

type Result struct {
	Url  string
	Code int
	Err  error
}

type QueriedUrls struct {
	rwmutex sync.RWMutex
	urls    map[string]bool
}

func (qu *QueriedUrls) Add(url string) {
	qu.rwmutex.Lock()
	defer qu.rwmutex.Unlock()
	qu.urls[url] = true
}

func (qu *QueriedUrls) Exists(url string) bool {
	qu.rwmutex.RLock()
	defer qu.rwmutex.RUnlock()
	return qu.urls[url]
}

func SendRequest(client Client, request *http.Request) (responseBody []byte, responseCode int, err error) {
	response, err := client.Do(request)
	if err != nil {
		return nil, 0, err
	}
	defer response.Body.Close()

	bytes, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, 0, err
	}

	return bytes, response.StatusCode, nil
}

func RecursiveQueryToChanHelper(client Client, queryUrl string, depth uint, sameDomain bool, results chan *Result, wg *sync.WaitGroup, qu *QueriedUrls) {
	defer wg.Done()

	sendResult := func(url string, code int, err error) {
		result := &Result{
			Url:  url,
			Code: code,
			Err:  err,
		}
		results <- result
	}

	request, err := http.NewRequest("GET", queryUrl, nil)
	if err != nil {
		sendResult(queryUrl, 0, err)
		return
	}

	response, code, err := SendRequest(client, request)
	if err != nil {
		sendResult(queryUrl, code, err)
		return
	}

	sendResult(queryUrl, code, nil)

	if depth <= 0 {
		return
	}

	node, err := parse.BytesToHtmlNode(response)
	if err != nil {
		sendResult(queryUrl, code, err)
		return
	}

	anchors := parse.Anchors(node)

	hrefs, errs := parse.ValidHrefs(anchors)
	for _, err := range errs {
		sendResult(queryUrl, code, err)
	}

	urls, errs := rpurl.AbsoluteHttpUrls(queryUrl, hrefs)
	for _, err := range errs {
		sendResult(queryUrl, code, err)
	}

	for _, url := range urls {
		if sameDomain && !rpurl.IsSameDomain(queryUrl, url) {
			continue
		}

		if qu.Exists(url) {
			continue
		} else {
			qu.Add(url)
		}

		wg.Add(1)
		go RecursiveQueryToChanHelper(client, url, depth-1, sameDomain, results, wg, qu)
	}

}

func RecursiveQueryToChan(client Client, queryUrl string, depth uint, sameDomain bool, results chan *Result) {
	defer close(results)

	wg := &sync.WaitGroup{}
	qu := &QueriedUrls{
		urls: make(map[string]bool),
	}

	qu.Add(queryUrl)
	wg.Add(1)
	go RecursiveQueryToChanHelper(client, queryUrl, depth, sameDomain, results, wg, qu)

	wg.Wait()
}
<commit_msg>Created a generalized structure for waiting on unique strings<commit_after>package requests

import (
	"io/ioutil"
	"net/http"
	"sync"

	"github.com/mschwager/riplink/src/parse"
	"github.com/mschwager/riplink/src/rpurl"
)

type Client interface {
	Do(*http.Request) (*http.Response, error)
}

type Result struct {
	Url  string
	Code int
	Err  error
}

type UniqueWaitGroup struct {
	*sync.WaitGroup
	*sync.RWMutex
	existing map[string]bool
}

func (uwg *UniqueWaitGroup) AddString(str string) bool {
	uwg.RLock()
	if uwg.existing[str] {
		uwg.RUnlock()
		return false
	}
	uwg.RUnlock()

	uwg.Lock()
	uwg.existing[str] = true
	uwg.Unlock()

	uwg.Add(1)

	return true
}

func SendRequest(client Client, request *http.Request) (responseBody []byte, responseCode int, err error) {
	response, err := client.Do(request)
	if err != nil {
		return nil, 0, err
	}
	defer response.Body.Close()

	bytes, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, 0, err
	}

	return bytes, response.StatusCode, nil
}

func RecursiveQueryToChanHelper(client Client, queryUrl string, depth uint, sameDomain bool, results chan *Result, uwg *UniqueWaitGroup) {
	defer uwg.Done()

	sendResult := func(url string, code int, err error) {
		result := &Result{
			Url:  url,
			Code: code,
			Err:  err,
		}
		results <- result
	}

	request, err := http.NewRequest("GET", queryUrl, nil)
	if err != nil {
		sendResult(queryUrl, 0, err)
		return
	}

	response, code, err := SendRequest(client, request)
	if err != nil {
		sendResult(queryUrl, code, err)
		return
	}

	sendResult(queryUrl, code, nil)

	if depth <= 0 {
		return
	}

	node, err := parse.BytesToHtmlNode(response)
	if err != nil {
		sendResult(queryUrl, code, err)
		return
	}

	anchors := parse.Anchors(node)

	hrefs, errs := parse.ValidHrefs(anchors)
	for _, err := range errs {
		sendResult(queryUrl, code, err)
	}

	urls, errs := rpurl.AbsoluteHttpUrls(queryUrl, hrefs)
	for _, err := range errs {
		sendResult(queryUrl, code, err)
	}

	for _, url := range urls {
		if sameDomain && !rpurl.IsSameDomain(queryUrl, url) {
			continue
		}

		if added := uwg.AddString(url); !added {
			continue
		}

		go RecursiveQueryToChanHelper(client, url, depth-1, sameDomain, results, uwg)
	}

}

func RecursiveQueryToChan(client Client, queryUrl string, depth uint, sameDomain bool, results chan *Result) {
	defer close(results)

	uwg := &UniqueWaitGroup{
		&sync.WaitGroup{},
		&sync.RWMutex{},
		make(map[string]bool),
	}

	// Assume the UniqueWaitGroup is empty
	_ = uwg.AddString(queryUrl)

	go RecursiveQueryToChanHelper(client, queryUrl, depth, sameDomain, results, uwg)

	uwg.Wait()
}
<|endoftext|>


<commit_before>// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
// All rights reservefs.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package storage

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/conformal/goleveldb/leveldb/util"
)

var errFileOpen = errors.New("leveldb/storage: file still open")

type fileLock interface {
	release() error
}

type fileStorageLock struct {
	fs *fileStorage
}

func (lock *fileStorageLock) Release() {
	fs := lock.fs
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.slock == lock {
		fs.slock = nil
	}
	return
}

// fileStorage is a file-system backed storage.
type fileStorage struct {
	path string

	mu    sync.Mutex
	flock fileLock
	slock *fileStorageLock
	logw  *os.File
	buf   []byte
	// Opened file counter; if open < 0 means closed.
	open int
	day  int
}

// OpenFile returns a new filesytem-backed storage implementation with the given
// path. This also hold a file lock, so any subsequent attempt to open the same
// path will fail.
//
// The storage must be closed after use, by calling Close method.
func OpenFile(path string) (Storage, error) {
	if err := os.MkdirAll(path, 0755); err != nil {
		return nil, err
	}

	flock, err := newFileLock(filepath.Join(path, "LOCK"))
	if err != nil {
		return nil, err
	}

	defer func() {
		if err != nil {
			flock.release()
		}
	}()

	rename(filepath.Join(path, "LOG"), filepath.Join(path, "LOG.old"))
	logw, err := os.OpenFile(filepath.Join(path, "LOG"), os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		return nil, err
	}

	fs := &fileStorage{path: path, flock: flock, logw: logw}
	runtime.SetFinalizer(fs, (*fileStorage).Close)
	return fs, nil
}

func (fs *fileStorage) Lock() (util.Releaser, error) {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return nil, ErrClosed
	}
	if fs.slock != nil {
		return nil, ErrLocked
	}
	fs.slock = &fileStorageLock{fs: fs}
	return fs.slock, nil
}

func itoa(buf []byte, i int, wid int) []byte {
	var u uint = uint(i)
	if u == 0 && wid <= 1 {
		return append(buf, '0')
	}

	// Assemble decimal in reverse order.
	var b [32]byte
	bp := len(b)
	for ; u > 0 || wid > 0; u /= 10 {
		bp--
		wid--
		b[bp] = byte(u%10) + '0'
	}
	return append(buf, b[bp:]...)
}

func (fs *fileStorage) printDay(t time.Time) {
	if fs.day == t.Day() {
		return
	}
	fs.day = t.Day()
	fs.logw.Write([]byte("=============== " + t.Format("Jan 2, 2006 (MST)") + " ===============\n"))
}

func (fs *fileStorage) doLog(t time.Time, str string) {
	fs.printDay(t)
	hour, min, sec := t.Clock()
	msec := t.Nanosecond() / 1e3
	// time
	fs.buf = itoa(fs.buf[:0], hour, 2)
	fs.buf = append(fs.buf, ':')
	fs.buf = itoa(fs.buf, min, 2)
	fs.buf = append(fs.buf, ':')
	fs.buf = itoa(fs.buf, sec, 2)
	fs.buf = append(fs.buf, '.')
	fs.buf = itoa(fs.buf, msec, 6)
	fs.buf = append(fs.buf, ' ')
	// write
	fs.buf = append(fs.buf, []byte(str)...)
	fs.buf = append(fs.buf, '\n')
	fs.logw.Write(fs.buf)
}

func (fs *fileStorage) Log(str string) {
	t := time.Now()
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return
	}
	fs.doLog(t, str)
}

func (fs *fileStorage) log(str string) {
	fs.doLog(time.Now(), str)
}

func (fs *fileStorage) GetFile(num uint64, t FileType) File {
	return &file{fs: fs, num: num, t: t}
}

func (fs *fileStorage) GetFiles(t FileType) (ff []File, err error) {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return nil, ErrClosed
	}
	dir, err := os.Open(fs.path)
	if err != nil {
		return
	}
	fnn, err := dir.Readdirnames(0)
	// Close the dir first before checking for Readdirnames error.
	if err := dir.Close(); err != nil {
		fs.log(fmt.Sprintf("close dir: %v", err))
	}
	if err != nil {
		return
	}
	f := &file{fs: fs}
	for _, fn := range fnn {
		if f.parse(fn) && (f.t&t) != 0 {
			ff = append(ff, f)
			f = &file{fs: fs}
		}
	}
	return
}

func (fs *fileStorage) GetManifest() (f File, err error) {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return nil, ErrClosed
	}
	dir, err := os.Open(fs.path)
	if err != nil {
		return
	}
	fnn, err := dir.Readdirnames(0)
	// Close the dir first before checking for Readdirnames error.
	if err := dir.Close(); err != nil {
		fs.log(fmt.Sprintf("close dir: %v", err))
	}
	if err != nil {
		return
	}
	// Find latest CURRENT file.
	var rem []string
	var pend bool
	var cerr error
	for _, fn := range fnn {
		if strings.HasPrefix(fn, "CURRENT") {
			pend1 := len(fn) > 7
			// Make sure it is valid name for a CURRENT file, otherwise skip it.
			if pend1 {
				if fn[7] != '.' || len(fn) < 9 {
					fs.log(fmt.Sprintf("skipping %s: invalid file name", fn))
					continue
				}
				if _, e1 := strconv.ParseUint(fn[7:], 10, 0); e1 != nil {
					fs.log(fmt.Sprintf("skipping %s: invalid file num: %v", fn, e1))
					continue
				}
			}
			path := filepath.Join(fs.path, fn)
			r, e1 := os.OpenFile(path, os.O_RDONLY, 0)
			if e1 != nil {
				return nil, e1
			}
			b, e1 := ioutil.ReadAll(r)
			if e1 != nil {
				r.Close()
				return nil, e1
			}
			f1 := &file{fs: fs}
			if len(b) < 1 || b[len(b)-1] != '\n' || !f1.parse(string(b[:len(b)-1])) {
				fs.log(fmt.Sprintf("skipping %s: corrupted or incomplete", fn))
				if pend1 {
					rem = append(rem, fn)
				}
				if !pend1 || cerr == nil {
					cerr = fmt.Errorf("leveldb/storage: corrupted or incomplete %s file", fn)
				}
			} else if f != nil && f1.Num() < f.Num() {
				fs.log(fmt.Sprintf("skipping %s: obsolete", fn))
				if pend1 {
					rem = append(rem, fn)
				}
			} else {
				f = f1
				pend = pend1
			}
			if err := r.Close(); err != nil {
				fs.log(fmt.Sprintf("close %s: %v", fn, err))
			}
		}
	}
	// Don't remove any files if there is no valid CURRENT file.
	if f == nil {
		if cerr != nil {
			err = cerr
		} else {
			err = os.ErrNotExist
		}
		return
	}
	// Rename pending CURRENT file to an effective CURRENT.
	if pend {
		path := fmt.Sprintf("%s.%d", filepath.Join(fs.path, "CURRENT"), f.Num())
		if err := rename(path, filepath.Join(fs.path, "CURRENT")); err != nil {
			fs.log(fmt.Sprintf("CURRENT.%d -> CURRENT: %d", f.Num(), err))
		}
	}
	// Remove obsolete or incomplete pending CURRENT files.
	for _, fn := range rem {
		path := filepath.Join(fs.path, fn)
		if err := os.Remove(path); err != nil {
			fs.log(fmt.Sprintf("remove %s: %v", fn, err))
		}
	}
	return
}

func (fs *fileStorage) SetManifest(f File) (err error) {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return ErrClosed
	}
	f2, ok := f.(*file)
	if !ok || f2.t != TypeManifest {
		return ErrInvalidFile
	}
	defer func() {
		if err != nil {
			fs.log(fmt.Sprintf("CURRENT: %v", err))
		}
	}()
	path := fmt.Sprintf("%s.%d", filepath.Join(fs.path, "CURRENT"), f2.Num())
	w, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return err
	}
	_, err = fmt.Fprintln(w, f2.name())
	// Close the file first.
	if err := w.Close(); err != nil {
		fs.log(fmt.Sprintf("close CURRENT.%d: %v", f2.num, err))
	}
	if err != nil {
		return err
	}
	err = rename(path, filepath.Join(fs.path, "CURRENT"))
	return
}

func (fs *fileStorage) Close() error {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return ErrClosed
	}
	// Clear the finalizer.
	runtime.SetFinalizer(fs, nil)

	if fs.open > 0 {
		fs.log(fmt.Sprintf("refuse to close, %d files still open", fs.open))
		return fmt.Errorf("leveldb/storage: cannot close, %d files still open", fs.open)
	}
	fs.open = -1
	e1 := fs.logw.Close()
	err := fs.flock.release()
	if err == nil {
		err = e1
	}
	return err
}

type fileWrap struct {
	*os.File
	f *file
}

func (fw fileWrap) Close() error {
	f := fw.f
	f.fs.mu.Lock()
	defer f.fs.mu.Unlock()
	if !f.open {
		return ErrClosed
	}
	f.open = false
	f.fs.open--
	err := fw.File.Close()
	if err != nil {
		f.fs.log(fmt.Sprintf("close %s.%d: %v", f.Type(), f.Num(), err))
	}
	return err
}

type file struct {
	fs   *fileStorage
	num  uint64
	t    FileType
	open bool
}

func (f *file) Open() (Reader, error) {
	f.fs.mu.Lock()
	defer f.fs.mu.Unlock()
	if f.fs.open < 0 {
		return nil, ErrClosed
	}
	if f.open {
		return nil, errFileOpen
	}
	of, err := os.OpenFile(f.path(), os.O_RDONLY, 0)
	if err != nil {
		return nil, err
	}
	f.open = true
	f.fs.open++
	return fileWrap{of, f}, nil
}

func (f *file) Create() (Writer, error) {
	f.fs.mu.Lock()
	defer f.fs.mu.Unlock()
	if f.fs.open < 0 {
		return nil, ErrClosed
	}
	if f.open {
		return nil, errFileOpen
	}
	of, err := os.OpenFile(f.path(), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return nil, err
	}
	f.open = true
	f.fs.open++
	return fileWrap{of, f}, nil
}

func (f *file) Type() FileType {
	return f.t
}

func (f *file) Num() uint64 {
	return f.num
}

func (f *file) Remove() error {
	f.fs.mu.Lock()
	defer f.fs.mu.Unlock()
	if f.fs.open < 0 {
		return ErrClosed
	}
	if f.open {
		return errFileOpen
	}
	err := os.Remove(f.path())
	if err != nil {
		f.fs.log(fmt.Sprint("remove %s.%d: %v", f.Type(), f.Num(), err))
	}
	return err
}

func (f *file) name() string {
	switch f.t {
	case TypeManifest:
		return fmt.Sprintf("MANIFEST-%06d", f.num)
	case TypeJournal:
		return fmt.Sprintf("%06d.log", f.num)
	case TypeTable:
		return fmt.Sprintf("%06d.sst", f.num)
	default:
		panic("invalid file type")
	}
	return ""
}

func (f *file) path() string {
	return filepath.Join(f.fs.path, f.name())
}

func (f *file) parse(name string) bool {
	var num uint64
	var tail string
	_, err := fmt.Sscanf(name, "%d.%s", &num, &tail)
	if err == nil {
		switch tail {
		case "log":
			f.t = TypeJournal
		case "sst":
			f.t = TypeTable
		default:
			return false
		}
		f.num = num
		return true
	}
	n, _ := fmt.Sscanf(name, "MANIFEST-%d%s", &num, &tail)
	if n == 1 {
		f.t = TypeManifest
		f.num = num
		return true
	}

	return false
}
<commit_msg>use correct verbs. found by go vet.<commit_after>// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
// All rights reservefs.
//
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package storage

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/conformal/goleveldb/leveldb/util"
)

var errFileOpen = errors.New("leveldb/storage: file still open")

type fileLock interface {
	release() error
}

type fileStorageLock struct {
	fs *fileStorage
}

func (lock *fileStorageLock) Release() {
	fs := lock.fs
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.slock == lock {
		fs.slock = nil
	}
	return
}

// fileStorage is a file-system backed storage.
type fileStorage struct {
	path string

	mu    sync.Mutex
	flock fileLock
	slock *fileStorageLock
	logw  *os.File
	buf   []byte
	// Opened file counter; if open < 0 means closed.
	open int
	day  int
}

// OpenFile returns a new filesytem-backed storage implementation with the given
// path. This also hold a file lock, so any subsequent attempt to open the same
// path will fail.
//
// The storage must be closed after use, by calling Close method.
func OpenFile(path string) (Storage, error) {
	if err := os.MkdirAll(path, 0755); err != nil {
		return nil, err
	}

	flock, err := newFileLock(filepath.Join(path, "LOCK"))
	if err != nil {
		return nil, err
	}

	defer func() {
		if err != nil {
			flock.release()
		}
	}()

	rename(filepath.Join(path, "LOG"), filepath.Join(path, "LOG.old"))
	logw, err := os.OpenFile(filepath.Join(path, "LOG"), os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		return nil, err
	}

	fs := &fileStorage{path: path, flock: flock, logw: logw}
	runtime.SetFinalizer(fs, (*fileStorage).Close)
	return fs, nil
}

func (fs *fileStorage) Lock() (util.Releaser, error) {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return nil, ErrClosed
	}
	if fs.slock != nil {
		return nil, ErrLocked
	}
	fs.slock = &fileStorageLock{fs: fs}
	return fs.slock, nil
}

func itoa(buf []byte, i int, wid int) []byte {
	var u uint = uint(i)
	if u == 0 && wid <= 1 {
		return append(buf, '0')
	}

	// Assemble decimal in reverse order.
	var b [32]byte
	bp := len(b)
	for ; u > 0 || wid > 0; u /= 10 {
		bp--
		wid--
		b[bp] = byte(u%10) + '0'
	}
	return append(buf, b[bp:]...)
}

func (fs *fileStorage) printDay(t time.Time) {
	if fs.day == t.Day() {
		return
	}
	fs.day = t.Day()
	fs.logw.Write([]byte("=============== " + t.Format("Jan 2, 2006 (MST)") + " ===============\n"))
}

func (fs *fileStorage) doLog(t time.Time, str string) {
	fs.printDay(t)
	hour, min, sec := t.Clock()
	msec := t.Nanosecond() / 1e3
	// time
	fs.buf = itoa(fs.buf[:0], hour, 2)
	fs.buf = append(fs.buf, ':')
	fs.buf = itoa(fs.buf, min, 2)
	fs.buf = append(fs.buf, ':')
	fs.buf = itoa(fs.buf, sec, 2)
	fs.buf = append(fs.buf, '.')
	fs.buf = itoa(fs.buf, msec, 6)
	fs.buf = append(fs.buf, ' ')
	// write
	fs.buf = append(fs.buf, []byte(str)...)
	fs.buf = append(fs.buf, '\n')
	fs.logw.Write(fs.buf)
}

func (fs *fileStorage) Log(str string) {
	t := time.Now()
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return
	}
	fs.doLog(t, str)
}

func (fs *fileStorage) log(str string) {
	fs.doLog(time.Now(), str)
}

func (fs *fileStorage) GetFile(num uint64, t FileType) File {
	return &file{fs: fs, num: num, t: t}
}

func (fs *fileStorage) GetFiles(t FileType) (ff []File, err error) {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return nil, ErrClosed
	}
	dir, err := os.Open(fs.path)
	if err != nil {
		return
	}
	fnn, err := dir.Readdirnames(0)
	// Close the dir first before checking for Readdirnames error.
	if err := dir.Close(); err != nil {
		fs.log(fmt.Sprintf("close dir: %v", err))
	}
	if err != nil {
		return
	}
	f := &file{fs: fs}
	for _, fn := range fnn {
		if f.parse(fn) && (f.t&t) != 0 {
			ff = append(ff, f)
			f = &file{fs: fs}
		}
	}
	return
}

func (fs *fileStorage) GetManifest() (f File, err error) {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return nil, ErrClosed
	}
	dir, err := os.Open(fs.path)
	if err != nil {
		return
	}
	fnn, err := dir.Readdirnames(0)
	// Close the dir first before checking for Readdirnames error.
	if err := dir.Close(); err != nil {
		fs.log(fmt.Sprintf("close dir: %v", err))
	}
	if err != nil {
		return
	}
	// Find latest CURRENT file.
	var rem []string
	var pend bool
	var cerr error
	for _, fn := range fnn {
		if strings.HasPrefix(fn, "CURRENT") {
			pend1 := len(fn) > 7
			// Make sure it is valid name for a CURRENT file, otherwise skip it.
			if pend1 {
				if fn[7] != '.' || len(fn) < 9 {
					fs.log(fmt.Sprintf("skipping %s: invalid file name", fn))
					continue
				}
				if _, e1 := strconv.ParseUint(fn[7:], 10, 0); e1 != nil {
					fs.log(fmt.Sprintf("skipping %s: invalid file num: %v", fn, e1))
					continue
				}
			}
			path := filepath.Join(fs.path, fn)
			r, e1 := os.OpenFile(path, os.O_RDONLY, 0)
			if e1 != nil {
				return nil, e1
			}
			b, e1 := ioutil.ReadAll(r)
			if e1 != nil {
				r.Close()
				return nil, e1
			}
			f1 := &file{fs: fs}
			if len(b) < 1 || b[len(b)-1] != '\n' || !f1.parse(string(b[:len(b)-1])) {
				fs.log(fmt.Sprintf("skipping %s: corrupted or incomplete", fn))
				if pend1 {
					rem = append(rem, fn)
				}
				if !pend1 || cerr == nil {
					cerr = fmt.Errorf("leveldb/storage: corrupted or incomplete %s file", fn)
				}
			} else if f != nil && f1.Num() < f.Num() {
				fs.log(fmt.Sprintf("skipping %s: obsolete", fn))
				if pend1 {
					rem = append(rem, fn)
				}
			} else {
				f = f1
				pend = pend1
			}
			if err := r.Close(); err != nil {
				fs.log(fmt.Sprintf("close %s: %v", fn, err))
			}
		}
	}
	// Don't remove any files if there is no valid CURRENT file.
	if f == nil {
		if cerr != nil {
			err = cerr
		} else {
			err = os.ErrNotExist
		}
		return
	}
	// Rename pending CURRENT file to an effective CURRENT.
	if pend {
		path := fmt.Sprintf("%s.%d", filepath.Join(fs.path, "CURRENT"), f.Num())
		if err := rename(path, filepath.Join(fs.path, "CURRENT")); err != nil {
			fs.log(fmt.Sprintf("CURRENT.%d -> CURRENT: %v", f.Num(), err))
		}
	}
	// Remove obsolete or incomplete pending CURRENT files.
	for _, fn := range rem {
		path := filepath.Join(fs.path, fn)
		if err := os.Remove(path); err != nil {
			fs.log(fmt.Sprintf("remove %s: %v", fn, err))
		}
	}
	return
}

func (fs *fileStorage) SetManifest(f File) (err error) {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return ErrClosed
	}
	f2, ok := f.(*file)
	if !ok || f2.t != TypeManifest {
		return ErrInvalidFile
	}
	defer func() {
		if err != nil {
			fs.log(fmt.Sprintf("CURRENT: %v", err))
		}
	}()
	path := fmt.Sprintf("%s.%d", filepath.Join(fs.path, "CURRENT"), f2.Num())
	w, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return err
	}
	_, err = fmt.Fprintln(w, f2.name())
	// Close the file first.
	if err := w.Close(); err != nil {
		fs.log(fmt.Sprintf("close CURRENT.%d: %v", f2.num, err))
	}
	if err != nil {
		return err
	}
	err = rename(path, filepath.Join(fs.path, "CURRENT"))
	return
}

func (fs *fileStorage) Close() error {
	fs.mu.Lock()
	defer fs.mu.Unlock()
	if fs.open < 0 {
		return ErrClosed
	}
	// Clear the finalizer.
	runtime.SetFinalizer(fs, nil)

	if fs.open > 0 {
		fs.log(fmt.Sprintf("refuse to close, %d files still open", fs.open))
		return fmt.Errorf("leveldb/storage: cannot close, %d files still open", fs.open)
	}
	fs.open = -1
	e1 := fs.logw.Close()
	err := fs.flock.release()
	if err == nil {
		err = e1
	}
	return err
}

type fileWrap struct {
	*os.File
	f *file
}

func (fw fileWrap) Close() error {
	f := fw.f
	f.fs.mu.Lock()
	defer f.fs.mu.Unlock()
	if !f.open {
		return ErrClosed
	}
	f.open = false
	f.fs.open--
	err := fw.File.Close()
	if err != nil {
		f.fs.log(fmt.Sprintf("close %s.%d: %v", f.Type(), f.Num(), err))
	}
	return err
}

type file struct {
	fs   *fileStorage
	num  uint64
	t    FileType
	open bool
}

func (f *file) Open() (Reader, error) {
	f.fs.mu.Lock()
	defer f.fs.mu.Unlock()
	if f.fs.open < 0 {
		return nil, ErrClosed
	}
	if f.open {
		return nil, errFileOpen
	}
	of, err := os.OpenFile(f.path(), os.O_RDONLY, 0)
	if err != nil {
		return nil, err
	}
	f.open = true
	f.fs.open++
	return fileWrap{of, f}, nil
}

func (f *file) Create() (Writer, error) {
	f.fs.mu.Lock()
	defer f.fs.mu.Unlock()
	if f.fs.open < 0 {
		return nil, ErrClosed
	}
	if f.open {
		return nil, errFileOpen
	}
	of, err := os.OpenFile(f.path(), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return nil, err
	}
	f.open = true
	f.fs.open++
	return fileWrap{of, f}, nil
}

func (f *file) Type() FileType {
	return f.t
}

func (f *file) Num() uint64 {
	return f.num
}

func (f *file) Remove() error {
	f.fs.mu.Lock()
	defer f.fs.mu.Unlock()
	if f.fs.open < 0 {
		return ErrClosed
	}
	if f.open {
		return errFileOpen
	}
	err := os.Remove(f.path())
	if err != nil {
		f.fs.log(fmt.Sprintf("remove %s.%d: %v", f.Type(), f.Num(), err))
	}
	return err
}

func (f *file) name() string {
	switch f.t {
	case TypeManifest:
		return fmt.Sprintf("MANIFEST-%06d", f.num)
	case TypeJournal:
		return fmt.Sprintf("%06d.log", f.num)
	case TypeTable:
		return fmt.Sprintf("%06d.sst", f.num)
	default:
		panic("invalid file type")
	}
	return ""
}

func (f *file) path() string {
	return filepath.Join(f.fs.path, f.name())
}

func (f *file) parse(name string) bool {
	var num uint64
	var tail string
	_, err := fmt.Sscanf(name, "%d.%s", &num, &tail)
	if err == nil {
		switch tail {
		case "log":
			f.t = TypeJournal
		case "sst":
			f.t = TypeTable
		default:
			return false
		}
		f.num = num
		return true
	}
	n, _ := fmt.Sscanf(name, "MANIFEST-%d%s", &num, &tail)
	if n == 1 {
		f.t = TypeManifest
		f.num = num
		return true
	}

	return false
}
<|endoftext|>


<commit_before>// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Checking of compiler and linker flags.
// We must avoid flags like -fplugin=, which can allow
// arbitrary code execution during the build.
// Do not make changes here without carefully
// considering the implications.
// (That's why the code is isolated in a file named security.go.)
//
// Note that -Wl,foo means split foo on commas and pass to
// the linker, so that -Wl,-foo,bar means pass -foo bar to
// the linker. Similarly -Wa,foo for the assembler and so on.
// If any of these are permitted, the wildcard portion must
// disallow commas.
//
// Note also that GNU binutils accept any argument @foo
// as meaning "read more flags from the file foo", so we must
// guard against any command-line argument beginning with @,
// even things like "-I @foo".
// We use load.SafeArg (which is even more conservative)
// to reject these.
//
// Even worse, gcc -I@foo (one arg) turns into cc1 -I @foo (two args),
// so although gcc doesn't expand the @foo, cc1 will.
// So out of paranoia, we reject @ at the beginning of every
// flag argument that might be split into its own argument.

package work

import (
	"fmt"
	"internal/lazyregexp"
	"regexp"
	"strings"

	"cmd/go/internal/cfg"
	"cmd/go/internal/load"
)

var re = lazyregexp.New

var validCompilerFlags = []*lazyregexp.Regexp{
	re(`-D([A-Za-z_].*)`),
	re(`-U([A-Za-z_]*)`),
	re(`-F([^@\-].*)`),
	re(`-I([^@\-].*)`),
	re(`-O`),
	re(`-O([^@\-].*)`),
	re(`-W`),
	re(`-W([^@,]+)`), // -Wall but not -Wa,-foo.
	re(`-Wa,-mbig-obj`),
	re(`-Wp,-D([A-Za-z_].*)`),
	re(`-Wp,-U([A-Za-z_]*)`),
	re(`-ansi`),
	re(`-f(no-)?asynchronous-unwind-tables`),
	re(`-f(no-)?blocks`),
	re(`-f(no-)builtin-[a-zA-Z0-9_]*`),
	re(`-f(no-)?common`),
	re(`-f(no-)?constant-cfstrings`),
	re(`-fdiagnostics-show-note-include-stack`),
	re(`-f(no-)?eliminate-unused-debug-types`),
	re(`-f(no-)?exceptions`),
	re(`-f(no-)?fast-math`),
	re(`-f(no-)?inline-functions`),
	re(`-finput-charset=([^@\-].*)`),
	re(`-f(no-)?fat-lto-objects`),
	re(`-f(no-)?keep-inline-dllexport`),
	re(`-f(no-)?lto`),
	re(`-fmacro-backtrace-limit=(.+)`),
	re(`-fmessage-length=(.+)`),
	re(`-f(no-)?modules`),
	re(`-f(no-)?objc-arc`),
	re(`-f(no-)?objc-nonfragile-abi`),
	re(`-f(no-)?objc-legacy-dispatch`),
	re(`-f(no-)?omit-frame-pointer`),
	re(`-f(no-)?openmp(-simd)?`),
	re(`-f(no-)?permissive`),
	re(`-f(no-)?(pic|PIC|pie|PIE)`),
	re(`-f(no-)?plt`),
	re(`-f(no-)?rtti`),
	re(`-f(no-)?split-stack`),
	re(`-f(no-)?stack-(.+)`),
	re(`-f(no-)?strict-aliasing`),
	re(`-f(un)signed-char`),
	re(`-f(no-)?use-linker-plugin`), // safe if -B is not used; we don't permit -B
	re(`-f(no-)?visibility-inlines-hidden`),
	re(`-fsanitize=(.+)`),
	re(`-ftemplate-depth-(.+)`),
	re(`-fvisibility=(.+)`),
	re(`-g([^@\-].*)?`),
	re(`-m32`),
	re(`-m64`),
	re(`-m(abi|arch|cpu|fpu|tune)=([^@\-].*)`),
	re(`-m(no-)?v?aes`),
	re(`-marm`),
	re(`-m(no-)?avx[0-9a-z]*`),
	re(`-mfloat-abi=([^@\-].*)`),
	re(`-mfpmath=[0-9a-z,+]*`),
	re(`-m(no-)?avx[0-9a-z.]*`),
	re(`-m(no-)?ms-bitfields`),
	re(`-m(no-)?stack-(.+)`),
	re(`-mmacosx-(.+)`),
	re(`-mios-simulator-version-min=(.+)`),
	re(`-miphoneos-version-min=(.+)`),
	re(`-mtvos-simulator-version-min=(.+)`),
	re(`-mtvos-version-min=(.+)`),
	re(`-mwatchos-simulator-version-min=(.+)`),
	re(`-mwatchos-version-min=(.+)`),
	re(`-mnop-fun-dllimport`),
	re(`-m(no-)?sse[0-9.]*`),
	re(`-m(no-)?ssse3`),
	re(`-mthumb(-interwork)?`),
	re(`-mthreads`),
	re(`-mwindows`),
	re(`--param=ssp-buffer-size=[0-9]*`),
	re(`-pedantic(-errors)?`),
	re(`-pipe`),
	re(`-pthread`),
	re(`-?-std=([^@\-].*)`),
	re(`-?-stdlib=([^@\-].*)`),
	re(`--sysroot=([^@\-].*)`),
	re(`-w`),
	re(`-x([^@\-].*)`),
	re(`-v`),
}

var validCompilerFlagsWithNextArg = []string{
	"-arch",
	"-D",
	"-U",
	"-I",
	"-framework",
	"-isysroot",
	"-isystem",
	"--sysroot",
	"-target",
	"-x",
}

var validLinkerFlags = []*lazyregexp.Regexp{
	re(`-F([^@\-].*)`),
	re(`-l([^@\-].*)`),
	re(`-L([^@\-].*)`),
	re(`-O`),
	re(`-O([^@\-].*)`),
	re(`-f(no-)?(pic|PIC|pie|PIE)`),
	re(`-f(no-)?openmp(-simd)?`),
	re(`-fsanitize=([^@\-].*)`),
	re(`-flat_namespace`),
	re(`-g([^@\-].*)?`),
	re(`-headerpad_max_install_names`),
	re(`-m(abi|arch|cpu|fpu|tune)=([^@\-].*)`),
	re(`-mfloat-abi=([^@\-].*)`),
	re(`-mmacosx-(.+)`),
	re(`-mios-simulator-version-min=(.+)`),
	re(`-miphoneos-version-min=(.+)`),
	re(`-mthreads`),
	re(`-mwindows`),
	re(`-(pic|PIC|pie|PIE)`),
	re(`-pthread`),
	re(`-rdynamic`),
	re(`-shared`),
	re(`-?-static([-a-z0-9+]*)`),
	re(`-?-stdlib=([^@\-].*)`),
	re(`-v`),

	// Note that any wildcards in -Wl need to exclude comma,
	// since -Wl splits its argument at commas and passes
	// them all to the linker uninterpreted. Allowing comma
	// in a wildcard would allow tunnelling arbitrary additional
	// linker arguments through one of these.
	re(`-Wl,--(no-)?allow-multiple-definition`),
	re(`-Wl,--(no-)?allow-shlib-undefined`),
	re(`-Wl,--(no-)?as-needed`),
	re(`-Wl,-Bdynamic`),
	re(`-Wl,-berok`),
	re(`-Wl,-Bstatic`),
	re(`-WL,-O([^@,\-][^,]*)?`),
	re(`-Wl,-d[ny]`),
	re(`-Wl,--disable-new-dtags`),
	re(`-Wl,-e[=,][a-zA-Z0-9]*`),
	re(`-Wl,--enable-new-dtags`),
	re(`-Wl,--end-group`),
	re(`-Wl,--(no-)?export-dynamic`),
	re(`-Wl,-framework,[^,@\-][^,]+`),
	re(`-Wl,-headerpad_max_install_names`),
	re(`-Wl,--no-undefined`),
	re(`-Wl,-R([^@\-][^,@]*$)`),
	re(`-Wl,--just-symbols[=,]([^,@\-][^,@]+)`),
	re(`-Wl,-rpath(-link)?[=,]([^,@\-][^,]+)`),
	re(`-Wl,-s`),
	re(`-Wl,-search_paths_first`),
	re(`-Wl,-sectcreate,([^,@\-][^,]+),([^,@\-][^,]+),([^,@\-][^,]+)`),
	re(`-Wl,--start-group`),
	re(`-Wl,-?-static`),
	re(`-Wl,-?-subsystem,(native|windows|console|posix|xbox)`),
	re(`-Wl,-syslibroot[=,]([^,@\-][^,]+)`),
	re(`-Wl,-undefined[=,]([^,@\-][^,]+)`),
	re(`-Wl,-?-unresolved-symbols=[^,]+`),
	re(`-Wl,--(no-)?warn-([^,]+)`),
	re(`-Wl,-z,(no)?execstack`),
	re(`-Wl,-z,relro`),

	re(`[a-zA-Z0-9_/].*\.(a|o|obj|dll|dylib|so)`), // direct linker inputs: x.o or libfoo.so (but not -foo.o or @foo.o)
	re(`\./.*\.(a|o|obj|dll|dylib|so)`),
}

var validLinkerFlagsWithNextArg = []string{
	"-arch",
	"-F",
	"-l",
	"-L",
	"-framework",
	"-isysroot",
	"--sysroot",
	"-target",
	"-Wl,-framework",
	"-Wl,-rpath",
	"-Wl,-R",
	"-Wl,--just-symbols",
	"-Wl,-undefined",
}

func checkCompilerFlags(name, source string, list []string) error {
	return checkFlags(name, source, list, validCompilerFlags, validCompilerFlagsWithNextArg)
}

func checkLinkerFlags(name, source string, list []string) error {
	return checkFlags(name, source, list, validLinkerFlags, validLinkerFlagsWithNextArg)
}

func checkFlags(name, source string, list []string, valid []*lazyregexp.Regexp, validNext []string) error {
	// Let users override rules with $CGO_CFLAGS_ALLOW, $CGO_CFLAGS_DISALLOW, etc.
	var (
		allow    *regexp.Regexp
		disallow *regexp.Regexp
	)
	if env := cfg.Getenv("CGO_" + name + "_ALLOW"); env != "" {
		r, err := regexp.Compile(env)
		if err != nil {
			return fmt.Errorf("parsing $CGO_%s_ALLOW: %v", name, err)
		}
		allow = r
	}
	if env := cfg.Getenv("CGO_" + name + "_DISALLOW"); env != "" {
		r, err := regexp.Compile(env)
		if err != nil {
			return fmt.Errorf("parsing $CGO_%s_DISALLOW: %v", name, err)
		}
		disallow = r
	}

Args:
	for i := 0; i < len(list); i++ {
		arg := list[i]
		if disallow != nil && disallow.FindString(arg) == arg {
			goto Bad
		}
		if allow != nil && allow.FindString(arg) == arg {
			continue Args
		}
		for _, re := range valid {
			if re.FindString(arg) == arg { // must be complete match
				continue Args
			}
		}
		for _, x := range validNext {
			if arg == x {
				if i+1 < len(list) && load.SafeArg(list[i+1]) {
					i++
					continue Args
				}

				// Permit -Wl,-framework -Wl,name.
				if i+1 < len(list) &&
					strings.HasPrefix(arg, "-Wl,") &&
					strings.HasPrefix(list[i+1], "-Wl,") &&
					load.SafeArg(list[i+1][4:]) &&
					!strings.Contains(list[i+1][4:], ",") {
					i++
					continue Args
				}

				// Permit -I= /path, -I $SYSROOT.
				if i+1 < len(list) && arg == "-I" {
					if (strings.HasPrefix(list[i+1], "=") || strings.HasPrefix(list[i+1], "$SYSROOT")) &&
						load.SafeArg(list[i+1][1:]) {
						i++
						continue Args
					}
				}

				if i+1 < len(list) {
					return fmt.Errorf("invalid flag in %s: %s %s (see https://golang.org/s/invalidflag)", source, arg, list[i+1])
				}
				return fmt.Errorf("invalid flag in %s: %s without argument (see https://golang.org/s/invalidflag)", source, arg)
			}
		}
	Bad:
		return fmt.Errorf("invalid flag in %s: %s", source, arg)
	}
	return nil
}
<commit_msg>cmd/go: add -Wl,-E to linker flag whitelist (shortcut of --export-dynamic)<commit_after>// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Checking of compiler and linker flags.
// We must avoid flags like -fplugin=, which can allow
// arbitrary code execution during the build.
// Do not make changes here without carefully
// considering the implications.
// (That's why the code is isolated in a file named security.go.)
//
// Note that -Wl,foo means split foo on commas and pass to
// the linker, so that -Wl,-foo,bar means pass -foo bar to
// the linker. Similarly -Wa,foo for the assembler and so on.
// If any of these are permitted, the wildcard portion must
// disallow commas.
//
// Note also that GNU binutils accept any argument @foo
// as meaning "read more flags from the file foo", so we must
// guard against any command-line argument beginning with @,
// even things like "-I @foo".
// We use load.SafeArg (which is even more conservative)
// to reject these.
//
// Even worse, gcc -I@foo (one arg) turns into cc1 -I @foo (two args),
// so although gcc doesn't expand the @foo, cc1 will.
// So out of paranoia, we reject @ at the beginning of every
// flag argument that might be split into its own argument.

package work

import (
	"fmt"
	"internal/lazyregexp"
	"regexp"
	"strings"

	"cmd/go/internal/cfg"
	"cmd/go/internal/load"
)

var re = lazyregexp.New

var validCompilerFlags = []*lazyregexp.Regexp{
	re(`-D([A-Za-z_].*)`),
	re(`-U([A-Za-z_]*)`),
	re(`-F([^@\-].*)`),
	re(`-I([^@\-].*)`),
	re(`-O`),
	re(`-O([^@\-].*)`),
	re(`-W`),
	re(`-W([^@,]+)`), // -Wall but not -Wa,-foo.
	re(`-Wa,-mbig-obj`),
	re(`-Wp,-D([A-Za-z_].*)`),
	re(`-Wp,-U([A-Za-z_]*)`),
	re(`-ansi`),
	re(`-f(no-)?asynchronous-unwind-tables`),
	re(`-f(no-)?blocks`),
	re(`-f(no-)builtin-[a-zA-Z0-9_]*`),
	re(`-f(no-)?common`),
	re(`-f(no-)?constant-cfstrings`),
	re(`-fdiagnostics-show-note-include-stack`),
	re(`-f(no-)?eliminate-unused-debug-types`),
	re(`-f(no-)?exceptions`),
	re(`-f(no-)?fast-math`),
	re(`-f(no-)?inline-functions`),
	re(`-finput-charset=([^@\-].*)`),
	re(`-f(no-)?fat-lto-objects`),
	re(`-f(no-)?keep-inline-dllexport`),
	re(`-f(no-)?lto`),
	re(`-fmacro-backtrace-limit=(.+)`),
	re(`-fmessage-length=(.+)`),
	re(`-f(no-)?modules`),
	re(`-f(no-)?objc-arc`),
	re(`-f(no-)?objc-nonfragile-abi`),
	re(`-f(no-)?objc-legacy-dispatch`),
	re(`-f(no-)?omit-frame-pointer`),
	re(`-f(no-)?openmp(-simd)?`),
	re(`-f(no-)?permissive`),
	re(`-f(no-)?(pic|PIC|pie|PIE)`),
	re(`-f(no-)?plt`),
	re(`-f(no-)?rtti`),
	re(`-f(no-)?split-stack`),
	re(`-f(no-)?stack-(.+)`),
	re(`-f(no-)?strict-aliasing`),
	re(`-f(un)signed-char`),
	re(`-f(no-)?use-linker-plugin`), // safe if -B is not used; we don't permit -B
	re(`-f(no-)?visibility-inlines-hidden`),
	re(`-fsanitize=(.+)`),
	re(`-ftemplate-depth-(.+)`),
	re(`-fvisibility=(.+)`),
	re(`-g([^@\-].*)?`),
	re(`-m32`),
	re(`-m64`),
	re(`-m(abi|arch|cpu|fpu|tune)=([^@\-].*)`),
	re(`-m(no-)?v?aes`),
	re(`-marm`),
	re(`-m(no-)?avx[0-9a-z]*`),
	re(`-mfloat-abi=([^@\-].*)`),
	re(`-mfpmath=[0-9a-z,+]*`),
	re(`-m(no-)?avx[0-9a-z.]*`),
	re(`-m(no-)?ms-bitfields`),
	re(`-m(no-)?stack-(.+)`),
	re(`-mmacosx-(.+)`),
	re(`-mios-simulator-version-min=(.+)`),
	re(`-miphoneos-version-min=(.+)`),
	re(`-mtvos-simulator-version-min=(.+)`),
	re(`-mtvos-version-min=(.+)`),
	re(`-mwatchos-simulator-version-min=(.+)`),
	re(`-mwatchos-version-min=(.+)`),
	re(`-mnop-fun-dllimport`),
	re(`-m(no-)?sse[0-9.]*`),
	re(`-m(no-)?ssse3`),
	re(`-mthumb(-interwork)?`),
	re(`-mthreads`),
	re(`-mwindows`),
	re(`--param=ssp-buffer-size=[0-9]*`),
	re(`-pedantic(-errors)?`),
	re(`-pipe`),
	re(`-pthread`),
	re(`-?-std=([^@\-].*)`),
	re(`-?-stdlib=([^@\-].*)`),
	re(`--sysroot=([^@\-].*)`),
	re(`-w`),
	re(`-x([^@\-].*)`),
	re(`-v`),
}

var validCompilerFlagsWithNextArg = []string{
	"-arch",
	"-D",
	"-U",
	"-I",
	"-framework",
	"-isysroot",
	"-isystem",
	"--sysroot",
	"-target",
	"-x",
}

var validLinkerFlags = []*lazyregexp.Regexp{
	re(`-F([^@\-].*)`),
	re(`-l([^@\-].*)`),
	re(`-L([^@\-].*)`),
	re(`-O`),
	re(`-O([^@\-].*)`),
	re(`-f(no-)?(pic|PIC|pie|PIE)`),
	re(`-f(no-)?openmp(-simd)?`),
	re(`-fsanitize=([^@\-].*)`),
	re(`-flat_namespace`),
	re(`-g([^@\-].*)?`),
	re(`-headerpad_max_install_names`),
	re(`-m(abi|arch|cpu|fpu|tune)=([^@\-].*)`),
	re(`-mfloat-abi=([^@\-].*)`),
	re(`-mmacosx-(.+)`),
	re(`-mios-simulator-version-min=(.+)`),
	re(`-miphoneos-version-min=(.+)`),
	re(`-mthreads`),
	re(`-mwindows`),
	re(`-(pic|PIC|pie|PIE)`),
	re(`-pthread`),
	re(`-rdynamic`),
	re(`-shared`),
	re(`-?-static([-a-z0-9+]*)`),
	re(`-?-stdlib=([^@\-].*)`),
	re(`-v`),

	// Note that any wildcards in -Wl need to exclude comma,
	// since -Wl splits its argument at commas and passes
	// them all to the linker uninterpreted. Allowing comma
	// in a wildcard would allow tunnelling arbitrary additional
	// linker arguments through one of these.
	re(`-Wl,--(no-)?allow-multiple-definition`),
	re(`-Wl,--(no-)?allow-shlib-undefined`),
	re(`-Wl,--(no-)?as-needed`),
	re(`-Wl,-Bdynamic`),
	re(`-Wl,-berok`),
	re(`-Wl,-Bstatic`),
	re(`-WL,-O([^@,\-][^,]*)?`),
	re(`-Wl,-d[ny]`),
	re(`-Wl,--disable-new-dtags`),
	re(`-Wl,-e[=,][a-zA-Z0-9]*`),
	re(`-Wl,--enable-new-dtags`),
	re(`-Wl,--end-group`),
	re(`-Wl,--(no-)?export-dynamic`),
	re(`-Wl,-E`),
	re(`-Wl,-framework,[^,@\-][^,]+`),
	re(`-Wl,-headerpad_max_install_names`),
	re(`-Wl,--no-undefined`),
	re(`-Wl,-R([^@\-][^,@]*$)`),
	re(`-Wl,--just-symbols[=,]([^,@\-][^,@]+)`),
	re(`-Wl,-rpath(-link)?[=,]([^,@\-][^,]+)`),
	re(`-Wl,-s`),
	re(`-Wl,-search_paths_first`),
	re(`-Wl,-sectcreate,([^,@\-][^,]+),([^,@\-][^,]+),([^,@\-][^,]+)`),
	re(`-Wl,--start-group`),
	re(`-Wl,-?-static`),
	re(`-Wl,-?-subsystem,(native|windows|console|posix|xbox)`),
	re(`-Wl,-syslibroot[=,]([^,@\-][^,]+)`),
	re(`-Wl,-undefined[=,]([^,@\-][^,]+)`),
	re(`-Wl,-?-unresolved-symbols=[^,]+`),
	re(`-Wl,--(no-)?warn-([^,]+)`),
	re(`-Wl,-z,(no)?execstack`),
	re(`-Wl,-z,relro`),

	re(`[a-zA-Z0-9_/].*\.(a|o|obj|dll|dylib|so)`), // direct linker inputs: x.o or libfoo.so (but not -foo.o or @foo.o)
	re(`\./.*\.(a|o|obj|dll|dylib|so)`),
}

var validLinkerFlagsWithNextArg = []string{
	"-arch",
	"-F",
	"-l",
	"-L",
	"-framework",
	"-isysroot",
	"--sysroot",
	"-target",
	"-Wl,-framework",
	"-Wl,-rpath",
	"-Wl,-R",
	"-Wl,--just-symbols",
	"-Wl,-undefined",
}

func checkCompilerFlags(name, source string, list []string) error {
	return checkFlags(name, source, list, validCompilerFlags, validCompilerFlagsWithNextArg)
}

func checkLinkerFlags(name, source string, list []string) error {
	return checkFlags(name, source, list, validLinkerFlags, validLinkerFlagsWithNextArg)
}

func checkFlags(name, source string, list []string, valid []*lazyregexp.Regexp, validNext []string) error {
	// Let users override rules with $CGO_CFLAGS_ALLOW, $CGO_CFLAGS_DISALLOW, etc.
	var (
		allow    *regexp.Regexp
		disallow *regexp.Regexp
	)
	if env := cfg.Getenv("CGO_" + name + "_ALLOW"); env != "" {
		r, err := regexp.Compile(env)
		if err != nil {
			return fmt.Errorf("parsing $CGO_%s_ALLOW: %v", name, err)
		}
		allow = r
	}
	if env := cfg.Getenv("CGO_" + name + "_DISALLOW"); env != "" {
		r, err := regexp.Compile(env)
		if err != nil {
			return fmt.Errorf("parsing $CGO_%s_DISALLOW: %v", name, err)
		}
		disallow = r
	}

Args:
	for i := 0; i < len(list); i++ {
		arg := list[i]
		if disallow != nil && disallow.FindString(arg) == arg {
			goto Bad
		}
		if allow != nil && allow.FindString(arg) == arg {
			continue Args
		}
		for _, re := range valid {
			if re.FindString(arg) == arg { // must be complete match
				continue Args
			}
		}
		for _, x := range validNext {
			if arg == x {
				if i+1 < len(list) && load.SafeArg(list[i+1]) {
					i++
					continue Args
				}

				// Permit -Wl,-framework -Wl,name.
				if i+1 < len(list) &&
					strings.HasPrefix(arg, "-Wl,") &&
					strings.HasPrefix(list[i+1], "-Wl,") &&
					load.SafeArg(list[i+1][4:]) &&
					!strings.Contains(list[i+1][4:], ",") {
					i++
					continue Args
				}

				// Permit -I= /path, -I $SYSROOT.
				if i+1 < len(list) && arg == "-I" {
					if (strings.HasPrefix(list[i+1], "=") || strings.HasPrefix(list[i+1], "$SYSROOT")) &&
						load.SafeArg(list[i+1][1:]) {
						i++
						continue Args
					}
				}

				if i+1 < len(list) {
					return fmt.Errorf("invalid flag in %s: %s %s (see https://golang.org/s/invalidflag)", source, arg, list[i+1])
				}
				return fmt.Errorf("invalid flag in %s: %s without argument (see https://golang.org/s/invalidflag)", source, arg)
			}
		}
	Bad:
		return fmt.Errorf("invalid flag in %s: %s", source, arg)
	}
	return nil
}
<|endoftext|>


<commit_before>package mastodon

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"mime/multipart"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/tomnomnom/linkheader"
)

// Config is a setting for access mastodon APIs.
type Config struct {
	Server       string
	ClientID     string
	ClientSecret string
	AccessToken  string
}

// Client is a API client for mastodon.
type Client struct {
	http.Client
	config *Config
}

func (c *Client) doAPI(ctx context.Context, method string, uri string, params interface{}, res interface{}, pg *Pagination) error {
	u, err := url.Parse(c.config.Server)
	if err != nil {
		return err
	}
	u.Path = path.Join(u.Path, uri)

	var req *http.Request
	ct := "application/x-www-form-urlencoded"
	if values, ok := params.(url.Values); ok {
		var body io.Reader
		if method == http.MethodGet {
			if pg != nil {
				values = pg.setValues(values)
			}
			u.RawQuery = values.Encode()
		} else {
			body = strings.NewReader(values.Encode())
		}
		req, err = http.NewRequest(method, u.String(), body)
		if err != nil {
			return err
		}
	} else if file, ok := params.(string); ok {
		f, err := os.Open(file)
		if err != nil {
			return err
		}
		defer f.Close()

		var buf bytes.Buffer
		mw := multipart.NewWriter(&buf)
		part, err := mw.CreateFormFile("file", filepath.Base(file))
		if err != nil {
			return err
		}
		_, err = io.Copy(part, f)
		if err != nil {
			return err
		}
		err = mw.Close()
		if err != nil {
			return err
		}
		req, err = http.NewRequest(method, u.String(), &buf)
		if err != nil {
			return err
		}
		ct = mw.FormDataContentType()
	} else {
		if method == http.MethodGet && pg != nil {
			u.RawQuery = pg.toValues().Encode()
		}
		req, err = http.NewRequest(method, u.String(), nil)
		if err != nil {
			return err
		}
	}
	req = req.WithContext(ctx)
	req.Header.Set("Authorization", "Bearer "+c.config.AccessToken)
	if params != nil {
		req.Header.Set("Content-Type", ct)
	}

	resp, err := c.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return parseAPIError("bad request", resp)
	} else if res == nil {
		return nil
	} else if pg != nil {
		if lh := resp.Header.Get("Link"); lh != "" {
			pg2, err := newPagination(lh)
			if err != nil {
				return err
			}
			*pg = *pg2
		}
	}
	return json.NewDecoder(resp.Body).Decode(&res)
}

// NewClient return new mastodon API client.
func NewClient(config *Config) *Client {
	return &Client{
		Client: *http.DefaultClient,
		config: config,
	}
}

// Authenticate get access-token to the API.
func (c *Client) Authenticate(ctx context.Context, username, password string) error {
	params := url.Values{}
	params.Set("client_id", c.config.ClientID)
	params.Set("client_secret", c.config.ClientSecret)
	params.Set("grant_type", "password")
	params.Set("username", username)
	params.Set("password", password)
	params.Set("scope", "read write follow")

	u, err := url.Parse(c.config.Server)
	if err != nil {
		return err
	}
	u.Path = path.Join(u.Path, "/oauth/token")

	req, err := http.NewRequest(http.MethodPost, u.String(), strings.NewReader(params.Encode()))
	if err != nil {
		return err
	}
	req = req.WithContext(ctx)
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	resp, err := c.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return parseAPIError("bad authorization", resp)
	}

	res := struct {
		AccessToken string `json:"access_token"`
	}{}
	err = json.NewDecoder(resp.Body).Decode(&res)
	if err != nil {
		return err
	}
	c.config.AccessToken = res.AccessToken
	return nil
}

// Toot is struct to post status.
type Toot struct {
	Status      string  `json:"status"`
	InReplyToID int64   `json:"in_reply_to_id"`
	MediaIDs    []int64 `json:"media_ids"`
	Sensitive   bool    `json:"sensitive"`
	SpoilerText string  `json:"spoiler_text"`
	Visibility  string  `json:"visibility"`
}

// Mention hold information for mention.
type Mention struct {
	URL      string `json:"url"`
	Username string `json:"username"`
	Acct     string `json:"acct"`
	ID       int64  `json:"id"`
}

// Tag hold information for tag.
type Tag struct {
	Name string `json:"name"`
	URL  string `json:"url"`
}

// Attachment hold information for attachment.
type Attachment struct {
	ID         int64  `json:"id"`
	Type       string `json:"type"`
	URL        string `json:"url"`
	RemoteURL  string `json:"remote_url"`
	PreviewURL string `json:"preview_url"`
	TextURL    string `json:"text_url"`
}

// Results hold information for search result.
type Results struct {
	Accounts []*Account `json:"accounts"`
	Statuses []*Status  `json:"statuses"`
	Hashtags []string   `json:"hashtags"`
}

// Pagination is a struct for specifying the get range.
type Pagination struct {
	MaxID   int64
	SinceID int64
	Limit   int64
}

func newPagination(rawlink string) (*Pagination, error) {
	if rawlink == "" {
		return nil, errors.New("empty link header")
	}

	p := &Pagination{}
	for _, link := range linkheader.Parse(rawlink) {
		switch link.Rel {
		case "next":
			maxID, err := getPaginationID(link.URL, "max_id")
			if err != nil {
				return nil, err
			}
			p.MaxID = maxID
		case "prev":
			sinceID, err := getPaginationID(link.URL, "since_id")
			if err != nil {
				return nil, err
			}
			p.SinceID = sinceID
		}
	}

	return p, nil
}

func getPaginationID(rawurl, key string) (int64, error) {
	u, err := url.Parse(rawurl)
	if err != nil {
		return 0, err
	}

	id, err := strconv.ParseInt(u.Query().Get(key), 10, 64)
	if err != nil {
		return 0, err
	}

	return id, nil
}

func (p *Pagination) toValues() url.Values {
	return p.setValues(url.Values{})
}

func (p *Pagination) setValues(params url.Values) url.Values {
	if p.MaxID > 0 {
		params.Set("max_id", fmt.Sprint(p.MaxID))
	} else if p.SinceID > 0 {
		params.Set("since_id", fmt.Sprint(p.SinceID))
	}
	if p.Limit > 0 {
		params.Set("limit", fmt.Sprint(p.Limit))
	}

	return params
}
<commit_msg>Add package documents<commit_after>// Package mastodon provides functions and structs for accessing the mastodon API.
package mastodon

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"mime/multipart"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/tomnomnom/linkheader"
)

// Config is a setting for access mastodon APIs.
type Config struct {
	Server       string
	ClientID     string
	ClientSecret string
	AccessToken  string
}

// Client is a API client for mastodon.
type Client struct {
	http.Client
	config *Config
}

func (c *Client) doAPI(ctx context.Context, method string, uri string, params interface{}, res interface{}, pg *Pagination) error {
	u, err := url.Parse(c.config.Server)
	if err != nil {
		return err
	}
	u.Path = path.Join(u.Path, uri)

	var req *http.Request
	ct := "application/x-www-form-urlencoded"
	if values, ok := params.(url.Values); ok {
		var body io.Reader
		if method == http.MethodGet {
			if pg != nil {
				values = pg.setValues(values)
			}
			u.RawQuery = values.Encode()
		} else {
			body = strings.NewReader(values.Encode())
		}
		req, err = http.NewRequest(method, u.String(), body)
		if err != nil {
			return err
		}
	} else if file, ok := params.(string); ok {
		f, err := os.Open(file)
		if err != nil {
			return err
		}
		defer f.Close()

		var buf bytes.Buffer
		mw := multipart.NewWriter(&buf)
		part, err := mw.CreateFormFile("file", filepath.Base(file))
		if err != nil {
			return err
		}
		_, err = io.Copy(part, f)
		if err != nil {
			return err
		}
		err = mw.Close()
		if err != nil {
			return err
		}
		req, err = http.NewRequest(method, u.String(), &buf)
		if err != nil {
			return err
		}
		ct = mw.FormDataContentType()
	} else {
		if method == http.MethodGet && pg != nil {
			u.RawQuery = pg.toValues().Encode()
		}
		req, err = http.NewRequest(method, u.String(), nil)
		if err != nil {
			return err
		}
	}
	req = req.WithContext(ctx)
	req.Header.Set("Authorization", "Bearer "+c.config.AccessToken)
	if params != nil {
		req.Header.Set("Content-Type", ct)
	}

	resp, err := c.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return parseAPIError("bad request", resp)
	} else if res == nil {
		return nil
	} else if pg != nil {
		if lh := resp.Header.Get("Link"); lh != "" {
			pg2, err := newPagination(lh)
			if err != nil {
				return err
			}
			*pg = *pg2
		}
	}
	return json.NewDecoder(resp.Body).Decode(&res)
}

// NewClient return new mastodon API client.
func NewClient(config *Config) *Client {
	return &Client{
		Client: *http.DefaultClient,
		config: config,
	}
}

// Authenticate get access-token to the API.
func (c *Client) Authenticate(ctx context.Context, username, password string) error {
	params := url.Values{}
	params.Set("client_id", c.config.ClientID)
	params.Set("client_secret", c.config.ClientSecret)
	params.Set("grant_type", "password")
	params.Set("username", username)
	params.Set("password", password)
	params.Set("scope", "read write follow")

	u, err := url.Parse(c.config.Server)
	if err != nil {
		return err
	}
	u.Path = path.Join(u.Path, "/oauth/token")

	req, err := http.NewRequest(http.MethodPost, u.String(), strings.NewReader(params.Encode()))
	if err != nil {
		return err
	}
	req = req.WithContext(ctx)
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	resp, err := c.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return parseAPIError("bad authorization", resp)
	}

	res := struct {
		AccessToken string `json:"access_token"`
	}{}
	err = json.NewDecoder(resp.Body).Decode(&res)
	if err != nil {
		return err
	}
	c.config.AccessToken = res.AccessToken
	return nil
}

// Toot is struct to post status.
type Toot struct {
	Status      string  `json:"status"`
	InReplyToID int64   `json:"in_reply_to_id"`
	MediaIDs    []int64 `json:"media_ids"`
	Sensitive   bool    `json:"sensitive"`
	SpoilerText string  `json:"spoiler_text"`
	Visibility  string  `json:"visibility"`
}

// Mention hold information for mention.
type Mention struct {
	URL      string `json:"url"`
	Username string `json:"username"`
	Acct     string `json:"acct"`
	ID       int64  `json:"id"`
}

// Tag hold information for tag.
type Tag struct {
	Name string `json:"name"`
	URL  string `json:"url"`
}

// Attachment hold information for attachment.
type Attachment struct {
	ID         int64  `json:"id"`
	Type       string `json:"type"`
	URL        string `json:"url"`
	RemoteURL  string `json:"remote_url"`
	PreviewURL string `json:"preview_url"`
	TextURL    string `json:"text_url"`
}

// Results hold information for search result.
type Results struct {
	Accounts []*Account `json:"accounts"`
	Statuses []*Status  `json:"statuses"`
	Hashtags []string   `json:"hashtags"`
}

// Pagination is a struct for specifying the get range.
type Pagination struct {
	MaxID   int64
	SinceID int64
	Limit   int64
}

func newPagination(rawlink string) (*Pagination, error) {
	if rawlink == "" {
		return nil, errors.New("empty link header")
	}

	p := &Pagination{}
	for _, link := range linkheader.Parse(rawlink) {
		switch link.Rel {
		case "next":
			maxID, err := getPaginationID(link.URL, "max_id")
			if err != nil {
				return nil, err
			}
			p.MaxID = maxID
		case "prev":
			sinceID, err := getPaginationID(link.URL, "since_id")
			if err != nil {
				return nil, err
			}
			p.SinceID = sinceID
		}
	}

	return p, nil
}

func getPaginationID(rawurl, key string) (int64, error) {
	u, err := url.Parse(rawurl)
	if err != nil {
		return 0, err
	}

	id, err := strconv.ParseInt(u.Query().Get(key), 10, 64)
	if err != nil {
		return 0, err
	}

	return id, nil
}

func (p *Pagination) toValues() url.Values {
	return p.setValues(url.Values{})
}

func (p *Pagination) setValues(params url.Values) url.Values {
	if p.MaxID > 0 {
		params.Set("max_id", fmt.Sprint(p.MaxID))
	} else if p.SinceID > 0 {
		params.Set("since_id", fmt.Sprint(p.SinceID))
	}
	if p.Limit > 0 {
		params.Set("limit", fmt.Sprint(p.Limit))
	}

	return params
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/jessevdk/go-flags"
)

const version = "v0.0.1"

type opts struct {
	Help    bool   `short:"h" long:"help" description:"Print this message and quit"`
	Version bool   `short:"v" long:"version" description:"Print version information and quit"`
	OutDir  string `short:"o" long:"output" description:"Specify an output directory for Ghost posts"`
}

func init() {
}

func main() {
	opts := &opts{}
	p := flags.NewParser(opts, flags.PrintErrors)
	args, err := p.Parse()
	if err != nil {
		fmt.Fprintf(os.Stderr, helpText)
		return
	}

	if opts.Help {
		fmt.Fprintf(os.Stderr, helpText)
		return
	}

	if opts.Version {
		fmt.Fprintf(os.Stderr, "md2ghost: %s\n", version)
		return
	}

	if len(args) > 0 {
		targetDir := "."
		if len(os.Args) == 2 {
			targetDir = os.Args[1]
		}
		pattern := targetDir + "/*.md"
		files, err := filepath.Glob(pattern)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			return
		}
		for _, file := range files {
			fmt.Println(file)
		}
	}
}

const helpText = `md2ghost - Convert a markdown files into Ghost posts.

Usage: md2ghost [option] <file|directory>

Options:

  -o, --output  Specify an output directory for Ghost posts
  -h, --help    Print this message and quit
  -v, --version Print version information and quit

Example:

  $ md2ghost .
  $ md2ghost -o path/to/output_directory path/to/your_directory
`
<commit_msg>Implement a check file exists<commit_after>package main

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/jessevdk/go-flags"
)

const version = "v0.0.1"

type opts struct {
	Help    bool   `short:"h" long:"help" description:"Print this message and quit"`
	Version bool   `short:"v" long:"version" description:"Print version information and quit"`
	OutDir  string `short:"o" long:"output" description:"Specify an output directory for Ghost posts"`
}

func init() {
}

func main() {
	opts := &opts{}
	p := flags.NewParser(opts, flags.PrintErrors)
	args, err := p.Parse()
	if err != nil {
		fmt.Fprintf(os.Stderr, helpText)
		return
	}

	if opts.Help {
		fmt.Fprintf(os.Stderr, helpText)
		return
	}

	if opts.Version {
		fmt.Fprintf(os.Stderr, "md2ghost: %s\n", version)
		return
	}

	if len(args) > 0 {
		targetDir := "."
		if len(os.Args) == 2 {
			targetDir = os.Args[1]
		}
		pattern := targetDir + "/*.md"
		files, err := filepath.Glob(pattern)
		if err != nil {
			fmt.Fprintln(os.Stderr, err)
			return
		}
		for _, file := range files {
			fp, err = os.Open(file)
			if err != nil {
				fmt.Fprintln(os.Stderr, err)
				return
			}
			defer fp.Close()
		}
	}
}

const helpText = `md2ghost - Convert a markdown files into Ghost posts.

Usage: md2ghost [option] <file|directory>

Options:

  -o, --output  Specify an output directory for Ghost posts
  -h, --help    Print this message and quit
  -v, --version Print version information and quit

Example:

  $ md2ghost .
  $ md2ghost -o path/to/output_directory path/to/your_directory
`
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"log"
	"strconv"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/hashicorp/terraform/helper/hashcode"
	"github.com/hashicorp/terraform/helper/schema"
)

func dataSourceAwsVpcEndpointService() *schema.Resource {
	return &schema.Resource{
		Read: dataSourceAwsVpcEndpointServiceRead,

		Schema: map[string]*schema.Schema{
			"service": {
				Type:          schema.TypeString,
				Optional:      true,
				ConflictsWith: []string{"service_name"},
			},
			"service_name": {
				Type:          schema.TypeString,
				Optional:      true,
				Computed:      true,
				ConflictsWith: []string{"service"},
			},
			"service_type": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"owner": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"vpc_endpoint_policy_supported": {
				Type:     schema.TypeBool,
				Computed: true,
			},
			"acceptance_required": {
				Type:     schema.TypeBool,
				Computed: true,
			},
			"availability_zones": {
				Type:     schema.TypeSet,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Computed: true,
				Set:      schema.HashString,
			},
			"private_dns_name": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"base_endpoint_dns_names": {
				Type:     schema.TypeSet,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Computed: true,
				Set:      schema.HashString,
			},
		},
	}
}

func dataSourceAwsVpcEndpointServiceRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	var serviceName string
	if v, ok := d.GetOk("service_name"); ok {
		serviceName = v.(string)
	} else if v, ok := d.GetOk("service"); ok {
		serviceName = fmt.Sprintf("com.amazonaws.%s.%s", meta.(*AWSClient).region, v.(string))
	} else {
		return fmt.Errorf(
			"One of ['service', 'service_name'] must be set to query VPC Endpoint Services")
	}

	req := &ec2.DescribeVpcEndpointServicesInput{
		ServiceNames: aws.StringSlice([]string{serviceName}),
	}

	log.Printf("[DEBUG] Reading VPC Endpoint Services: %s", req)
	resp, err := conn.DescribeVpcEndpointServices(req)
	if err != nil {
		return fmt.Errorf("Error fetching VPC Endpoint Services: %s", err)
	}
	if resp == nil || len(resp.ServiceDetails) == 0 {
		return fmt.Errorf("no matching VPC Endpoint Service found")
	}
	if len(resp.ServiceDetails) > 1 {
		return fmt.Errorf("multiple VPC Endpoint Services matched; use additional constraints to reduce matches to a single VPC Endpoint Service")
	}

	sd := resp.ServiceDetails[0]
	serviceName = aws.StringValue(sd.ServiceName)
	d.SetId(strconv.Itoa(hashcode.String(serviceName)))
	d.Set("service_name", serviceName)
	d.Set("service_type", sd.ServiceType[0].ServiceType)
	d.Set("owner", sd.Owner)
	d.Set("vpc_endpoint_policy_supported", sd.VpcEndpointPolicySupported)
	d.Set("acceptance_required", sd.AcceptanceRequired)
	d.Set("availability_zones", flattenStringList(sd.AvailabilityZones))
	d.Set("private_dns_name", sd.PrivateDnsName)
	d.Set("base_endpoint_dns_names", flattenStringList(sd.BaseEndpointDnsNames))

	return nil
}
<commit_msg>Service details not yet available in GovCloud.<commit_after>package aws

import (
	"fmt"
	"log"
	"strconv"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/hashicorp/terraform/helper/hashcode"
	"github.com/hashicorp/terraform/helper/schema"
)

func dataSourceAwsVpcEndpointService() *schema.Resource {
	return &schema.Resource{
		Read: dataSourceAwsVpcEndpointServiceRead,

		Schema: map[string]*schema.Schema{
			"service": {
				Type:          schema.TypeString,
				Optional:      true,
				ConflictsWith: []string{"service_name"},
			},
			"service_name": {
				Type:          schema.TypeString,
				Optional:      true,
				Computed:      true,
				ConflictsWith: []string{"service"},
			},
			"service_type": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"owner": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"vpc_endpoint_policy_supported": {
				Type:     schema.TypeBool,
				Computed: true,
			},
			"acceptance_required": {
				Type:     schema.TypeBool,
				Computed: true,
			},
			"availability_zones": {
				Type:     schema.TypeSet,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Computed: true,
				Set:      schema.HashString,
			},
			"private_dns_name": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"base_endpoint_dns_names": {
				Type:     schema.TypeSet,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Computed: true,
				Set:      schema.HashString,
			},
		},
	}
}

func dataSourceAwsVpcEndpointServiceRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn
	isGovCloud := meta.(*AWSClient).IsGovCloud()

	var serviceName string
	if v, ok := d.GetOk("service_name"); ok {
		serviceName = v.(string)
	} else if v, ok := d.GetOk("service"); ok {
		serviceName = fmt.Sprintf("com.amazonaws.%s.%s", meta.(*AWSClient).region, v.(string))
	} else {
		return fmt.Errorf(
			"One of ['service', 'service_name'] must be set to query VPC Endpoint Services")
	}

	req := &ec2.DescribeVpcEndpointServicesInput{
		ServiceNames: aws.StringSlice([]string{serviceName}),
	}

	log.Printf("[DEBUG] Reading VPC Endpoint Services: %s", req)
	resp, err := conn.DescribeVpcEndpointServices(req)
	if err != nil {
		return fmt.Errorf("Error fetching VPC Endpoint Services: %s", err)
	}

	// Note: AWS Commercial now returns a response with `ServiceNames` and
	// `ServiceDetails`, but GovCloud responses only include `ServiceNames`
	if isGovCloud && resp.ServiceDetails == nil {
		if resp == nil || len(resp.ServiceNames) == 0 {
			return fmt.Errorf("no matching VPC Endpoint Service found")
		}
		if len(resp.ServiceNames) > 1 {
			return fmt.Errorf("multiple VPC Endpoint Services matched; use additional constraints to reduce matches to a single VPC Endpoint Service")
		}
		d.SetId(strconv.Itoa(hashcode.String(*resp.ServiceNames[0])))
		d.Set("service_name", resp.ServiceNames[0])
		return nil
	}

	if resp == nil || len(resp.ServiceDetails) == 0 {
		return fmt.Errorf("no matching VPC Endpoint Service found")
	}
	if len(resp.ServiceDetails) > 1 {
		return fmt.Errorf("multiple VPC Endpoint Services matched; use additional constraints to reduce matches to a single VPC Endpoint Service")
	}

	sd := resp.ServiceDetails[0]
	serviceName = aws.StringValue(sd.ServiceName)
	d.SetId(strconv.Itoa(hashcode.String(serviceName)))
	d.Set("service_name", serviceName)
	d.Set("service_type", sd.ServiceType[0].ServiceType)
	d.Set("owner", sd.Owner)
	d.Set("vpc_endpoint_policy_supported", sd.VpcEndpointPolicySupported)
	d.Set("acceptance_required", sd.AcceptanceRequired)
	d.Set("availability_zones", flattenStringList(sd.AvailabilityZones))
	d.Set("private_dns_name", sd.PrivateDnsName)
	d.Set("base_endpoint_dns_names", flattenStringList(sd.BaseEndpointDnsNames))

	return nil
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"log"

	"github.com/hashicorp/terraform/helper/schema"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	"github.com/hashicorp/terraform/helper/validation"
)

func resourceAwsCloudWatchMetricAlarm() *schema.Resource {
	return &schema.Resource{
		Create:        resourceAwsCloudWatchMetricAlarmCreate,
		Read:          resourceAwsCloudWatchMetricAlarmRead,
		Update:        resourceAwsCloudWatchMetricAlarmUpdate,
		Delete:        resourceAwsCloudWatchMetricAlarmDelete,
		SchemaVersion: 1,
		MigrateState:  resourceAwsCloudWatchMetricAlarmMigrateState,

		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"alarm_name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"comparison_operator": {
				Type:     schema.TypeString,
				Required: true,
			},
			"evaluation_periods": {
				Type:         schema.TypeInt,
				Required:     true,
				ValidateFunc: validation.IntAtLeast(1),
			},
			"metric_name": {
				Type:     schema.TypeString,
				Required: true,
			},
			"namespace": {
				Type:     schema.TypeString,
				Required: true,
			},
			"period": {
				Type:     schema.TypeInt,
				Required: true,
			},
			"statistic": {
				Type:          schema.TypeString,
				Optional:      true,
				ConflictsWith: []string{"extended_statistic"},
			},
			"threshold": {
				Type:     schema.TypeFloat,
				Required: true,
			},
			"actions_enabled": {
				Type:     schema.TypeBool,
				Optional: true,
				Default:  true,
			},
			"alarm_actions": {
				Type:     schema.TypeSet,
				Optional: true,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Set:      schema.HashString,
			},
			"alarm_description": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"datapoints_to_alarm": {
				Type:         schema.TypeInt,
				Optional:     true,
				ValidateFunc: validation.IntAtLeast(1),
			},
			"dimensions": {
				Type:     schema.TypeMap,
				Optional: true,
			},
			"insufficient_data_actions": {
				Type:     schema.TypeSet,
				Optional: true,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Set:      schema.HashString,
			},
			"ok_actions": {
				Type:     schema.TypeSet,
				Optional: true,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Set:      schema.HashString,
			},
			"unit": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"extended_statistic": {
				Type:          schema.TypeString,
				Optional:      true,
				ConflictsWith: []string{"statistic"},
			},
			"treat_missing_data": {
				Type:         schema.TypeString,
				Optional:     true,
				Default:      "missing",
				ValidateFunc: validation.StringInSlice([]string{"breaching", "notBreaching", "ignore", "missing"}, true),
			},
			"evaluate_low_sample_count_percentiles": {
				Type:         schema.TypeString,
				Optional:     true,
				Computed:     true,
				ValidateFunc: validation.StringInSlice([]string{"evaluate", "ignore"}, true),
			},
		},
	}
}

func resourceAwsCloudWatchMetricAlarmCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).cloudwatchconn

	_, statisticOk := d.GetOk("statistic")
	_, extendedStatisticOk := d.GetOk("extended_statistic")

	if !statisticOk && !extendedStatisticOk {
		return fmt.Errorf("One of `statistic` or `extended_statistic` must be set for a cloudwatch metric alarm")
	}

	params := getAwsCloudWatchPutMetricAlarmInput(d)

	log.Printf("[DEBUG] Creating CloudWatch Metric Alarm: %#v", params)
	_, err := conn.PutMetricAlarm(&params)
	if err != nil {
		return fmt.Errorf("Creating metric alarm failed: %s", err)
	}
	d.SetId(d.Get("alarm_name").(string))
	log.Println("[INFO] CloudWatch Metric Alarm created")

	return resourceAwsCloudWatchMetricAlarmRead(d, meta)
}

func resourceAwsCloudWatchMetricAlarmRead(d *schema.ResourceData, meta interface{}) error {
	a, err := getAwsCloudWatchMetricAlarm(d, meta)
	if err != nil {
		return err
	}
	if a == nil {
		d.SetId("")
		return nil
	}

	log.Printf("[DEBUG] Reading CloudWatch Metric Alarm: %s", d.Get("alarm_name"))

	d.Set("actions_enabled", a.ActionsEnabled)

	if err := d.Set("alarm_actions", _strArrPtrToList(a.AlarmActions)); err != nil {
		log.Printf("[WARN] Error setting Alarm Actions: %s", err)
	}
	d.Set("alarm_description", a.AlarmDescription)
	d.Set("alarm_name", a.AlarmName)
	d.Set("comparison_operator", a.ComparisonOperator)
	d.Set("datapoints_to_alarm", a.DatapointsToAlarm)
	if err := d.Set("dimensions", flattenDimensions(a.Dimensions)); err != nil {
		return err
	}
	d.Set("evaluation_periods", a.EvaluationPeriods)

	if err := d.Set("insufficient_data_actions", _strArrPtrToList(a.InsufficientDataActions)); err != nil {
		log.Printf("[WARN] Error setting Insufficient Data Actions: %s", err)
	}
	d.Set("metric_name", a.MetricName)
	d.Set("namespace", a.Namespace)

	if err := d.Set("ok_actions", _strArrPtrToList(a.OKActions)); err != nil {
		log.Printf("[WARN] Error setting OK Actions: %s", err)
	}
	d.Set("period", a.Period)
	d.Set("statistic", a.Statistic)
	d.Set("threshold", a.Threshold)
	d.Set("unit", a.Unit)
	d.Set("extended_statistic", a.ExtendedStatistic)
	d.Set("treat_missing_data", a.TreatMissingData)
	d.Set("evaluate_low_sample_count_percentiles", a.EvaluateLowSampleCountPercentile)

	return nil
}

func resourceAwsCloudWatchMetricAlarmUpdate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).cloudwatchconn
	params := getAwsCloudWatchPutMetricAlarmInput(d)

	log.Printf("[DEBUG] Updating CloudWatch Metric Alarm: %#v", params)
	_, err := conn.PutMetricAlarm(&params)
	if err != nil {
		return fmt.Errorf("Updating metric alarm failed: %s", err)
	}
	log.Println("[INFO] CloudWatch Metric Alarm updated")

	return resourceAwsCloudWatchMetricAlarmRead(d, meta)
}

func resourceAwsCloudWatchMetricAlarmDelete(d *schema.ResourceData, meta interface{}) error {
	p, err := getAwsCloudWatchMetricAlarm(d, meta)
	if err != nil {
		return err
	}
	if p == nil {
		log.Printf("[DEBUG] CloudWatch Metric Alarm %s is already gone", d.Id())
		return nil
	}

	log.Printf("[INFO] Deleting CloudWatch Metric Alarm: %s", d.Id())

	conn := meta.(*AWSClient).cloudwatchconn
	params := cloudwatch.DeleteAlarmsInput{
		AlarmNames: []*string{aws.String(d.Id())},
	}

	if _, err := conn.DeleteAlarms(&params); err != nil {
		return fmt.Errorf("Error deleting CloudWatch Metric Alarm: %s", err)
	}
	log.Println("[INFO] CloudWatch Metric Alarm deleted")

	return nil
}

func getAwsCloudWatchPutMetricAlarmInput(d *schema.ResourceData) cloudwatch.PutMetricAlarmInput {
	params := cloudwatch.PutMetricAlarmInput{
		AlarmName:          aws.String(d.Get("alarm_name").(string)),
		ComparisonOperator: aws.String(d.Get("comparison_operator").(string)),
		EvaluationPeriods:  aws.Int64(int64(d.Get("evaluation_periods").(int))),
		MetricName:         aws.String(d.Get("metric_name").(string)),
		Namespace:          aws.String(d.Get("namespace").(string)),
		Period:             aws.Int64(int64(d.Get("period").(int))),
		Threshold:          aws.Float64(d.Get("threshold").(float64)),
		TreatMissingData:   aws.String(d.Get("treat_missing_data").(string)),
	}

	if v := d.Get("actions_enabled"); v != nil {
		params.ActionsEnabled = aws.Bool(v.(bool))
	}

	if v, ok := d.GetOk("alarm_description"); ok {
		params.AlarmDescription = aws.String(v.(string))
	}

	if v, ok := d.GetOk("datapoints_to_alarm"); ok {
		params.DatapointsToAlarm = aws.Int64(int64(v.(int)))
	}

	if v, ok := d.GetOk("unit"); ok {
		params.Unit = aws.String(v.(string))
	}

	if v, ok := d.GetOk("statistic"); ok {
		params.Statistic = aws.String(v.(string))
	}

	if v, ok := d.GetOk("extended_statistic"); ok {
		params.ExtendedStatistic = aws.String(v.(string))
	}

	if v, ok := d.GetOk("evaluate_low_sample_count_percentiles"); ok {
		params.EvaluateLowSampleCountPercentile = aws.String(v.(string))
	}

	var alarmActions []*string
	if v := d.Get("alarm_actions"); v != nil {
		for _, v := range v.(*schema.Set).List() {
			str := v.(string)
			alarmActions = append(alarmActions, aws.String(str))
		}
		params.AlarmActions = alarmActions
	}

	var insufficientDataActions []*string
	if v := d.Get("insufficient_data_actions"); v != nil {
		for _, v := range v.(*schema.Set).List() {
			str := v.(string)
			insufficientDataActions = append(insufficientDataActions, aws.String(str))
		}
		params.InsufficientDataActions = insufficientDataActions
	}

	var okActions []*string
	if v := d.Get("ok_actions"); v != nil {
		for _, v := range v.(*schema.Set).List() {
			str := v.(string)
			okActions = append(okActions, aws.String(str))
		}
		params.OKActions = okActions
	}

	a := d.Get("dimensions").(map[string]interface{})
	dimensions := make([]*cloudwatch.Dimension, 0, len(a))
	for k, v := range a {
		dimensions = append(dimensions, &cloudwatch.Dimension{
			Name:  aws.String(k),
			Value: aws.String(v.(string)),
		})
	}
	params.Dimensions = dimensions

	return params
}

func getAwsCloudWatchMetricAlarm(d *schema.ResourceData, meta interface{}) (*cloudwatch.MetricAlarm, error) {
	conn := meta.(*AWSClient).cloudwatchconn

	params := cloudwatch.DescribeAlarmsInput{
		AlarmNames: []*string{aws.String(d.Id())},
	}

	resp, err := conn.DescribeAlarms(&params)
	if err != nil {
		return nil, err
	}

	// Find it and return it
	for idx, ma := range resp.MetricAlarms {
		if *ma.AlarmName == d.Id() {
			return resp.MetricAlarms[idx], nil
		}
	}

	return nil, nil
}

func _strArrPtrToList(strArrPtr []*string) []string {
	var result []string
	for _, elem := range strArrPtr {
		result = append(result, *elem)
	}
	return result
}

func flattenDimensions(dims []*cloudwatch.Dimension) map[string]interface{} {
	flatDims := make(map[string]interface{})
	for _, d := range dims {
		flatDims[*d.Name] = *d.Value
	}
	return flatDims
}
<commit_msg>add arn attribute for cloudwatch alarm<commit_after>package aws

import (
	"fmt"
	"log"

	"github.com/hashicorp/terraform/helper/schema"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	"github.com/hashicorp/terraform/helper/validation"
)

func resourceAwsCloudWatchMetricAlarm() *schema.Resource {
	return &schema.Resource{
		Create:        resourceAwsCloudWatchMetricAlarmCreate,
		Read:          resourceAwsCloudWatchMetricAlarmRead,
		Update:        resourceAwsCloudWatchMetricAlarmUpdate,
		Delete:        resourceAwsCloudWatchMetricAlarmDelete,
		SchemaVersion: 1,
		MigrateState:  resourceAwsCloudWatchMetricAlarmMigrateState,

		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"alarm_name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"arn": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
			"comparison_operator": {
				Type:     schema.TypeString,
				Required: true,
			},
			"evaluation_periods": {
				Type:         schema.TypeInt,
				Required:     true,
				ValidateFunc: validation.IntAtLeast(1),
			},
			"metric_name": {
				Type:     schema.TypeString,
				Required: true,
			},
			"namespace": {
				Type:     schema.TypeString,
				Required: true,
			},
			"period": {
				Type:     schema.TypeInt,
				Required: true,
			},
			"statistic": {
				Type:          schema.TypeString,
				Optional:      true,
				ConflictsWith: []string{"extended_statistic"},
			},
			"threshold": {
				Type:     schema.TypeFloat,
				Required: true,
			},
			"actions_enabled": {
				Type:     schema.TypeBool,
				Optional: true,
				Default:  true,
			},
			"alarm_actions": {
				Type:     schema.TypeSet,
				Optional: true,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Set:      schema.HashString,
			},
			"alarm_description": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"datapoints_to_alarm": {
				Type:         schema.TypeInt,
				Optional:     true,
				ValidateFunc: validation.IntAtLeast(1),
			},
			"dimensions": {
				Type:     schema.TypeMap,
				Optional: true,
			},
			"insufficient_data_actions": {
				Type:     schema.TypeSet,
				Optional: true,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Set:      schema.HashString,
			},
			"ok_actions": {
				Type:     schema.TypeSet,
				Optional: true,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Set:      schema.HashString,
			},
			"unit": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"extended_statistic": {
				Type:          schema.TypeString,
				Optional:      true,
				ConflictsWith: []string{"statistic"},
			},
			"treat_missing_data": {
				Type:         schema.TypeString,
				Optional:     true,
				Default:      "missing",
				ValidateFunc: validation.StringInSlice([]string{"breaching", "notBreaching", "ignore", "missing"}, true),
			},
			"evaluate_low_sample_count_percentiles": {
				Type:         schema.TypeString,
				Optional:     true,
				Computed:     true,
				ValidateFunc: validation.StringInSlice([]string{"evaluate", "ignore"}, true),
			},
		},
	}
}

func resourceAwsCloudWatchMetricAlarmCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).cloudwatchconn

	_, statisticOk := d.GetOk("statistic")
	_, extendedStatisticOk := d.GetOk("extended_statistic")

	if !statisticOk && !extendedStatisticOk {
		return fmt.Errorf("One of `statistic` or `extended_statistic` must be set for a cloudwatch metric alarm")
	}

	params := getAwsCloudWatchPutMetricAlarmInput(d)

	log.Printf("[DEBUG] Creating CloudWatch Metric Alarm: %#v", params)
	_, err := conn.PutMetricAlarm(&params)
	if err != nil {
		return fmt.Errorf("Creating metric alarm failed: %s", err)
	}
	d.SetId(d.Get("alarm_name").(string))
	log.Println("[INFO] CloudWatch Metric Alarm created")

	return resourceAwsCloudWatchMetricAlarmRead(d, meta)
}

func resourceAwsCloudWatchMetricAlarmRead(d *schema.ResourceData, meta interface{}) error {
	a, err := getAwsCloudWatchMetricAlarm(d, meta)
	if err != nil {
		return err
	}
	if a == nil {
		d.SetId("")
		return nil
	}

	log.Printf("[DEBUG] Reading CloudWatch Metric Alarm: %s", d.Get("alarm_name"))

	d.Set("actions_enabled", a.ActionsEnabled)

	if err := d.Set("alarm_actions", _strArrPtrToList(a.AlarmActions)); err != nil {
		log.Printf("[WARN] Error setting Alarm Actions: %s", err)
	}
	d.Set("alarm_description", a.AlarmDescription)
	d.Set("alarm_name", a.AlarmName)
	d.Set("arn", a.AlarmArn)
	d.Set("comparison_operator", a.ComparisonOperator)
	d.Set("datapoints_to_alarm", a.DatapointsToAlarm)
	if err := d.Set("dimensions", flattenDimensions(a.Dimensions)); err != nil {
		return err
	}
	d.Set("evaluation_periods", a.EvaluationPeriods)

	if err := d.Set("insufficient_data_actions", _strArrPtrToList(a.InsufficientDataActions)); err != nil {
		log.Printf("[WARN] Error setting Insufficient Data Actions: %s", err)
	}
	d.Set("metric_name", a.MetricName)
	d.Set("namespace", a.Namespace)

	if err := d.Set("ok_actions", _strArrPtrToList(a.OKActions)); err != nil {
		log.Printf("[WARN] Error setting OK Actions: %s", err)
	}
	d.Set("period", a.Period)
	d.Set("statistic", a.Statistic)
	d.Set("threshold", a.Threshold)
	d.Set("unit", a.Unit)
	d.Set("extended_statistic", a.ExtendedStatistic)
	d.Set("treat_missing_data", a.TreatMissingData)
	d.Set("evaluate_low_sample_count_percentiles", a.EvaluateLowSampleCountPercentile)

	return nil
}

func resourceAwsCloudWatchMetricAlarmUpdate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).cloudwatchconn
	params := getAwsCloudWatchPutMetricAlarmInput(d)

	log.Printf("[DEBUG] Updating CloudWatch Metric Alarm: %#v", params)
	_, err := conn.PutMetricAlarm(&params)
	if err != nil {
		return fmt.Errorf("Updating metric alarm failed: %s", err)
	}
	log.Println("[INFO] CloudWatch Metric Alarm updated")

	return resourceAwsCloudWatchMetricAlarmRead(d, meta)
}

func resourceAwsCloudWatchMetricAlarmDelete(d *schema.ResourceData, meta interface{}) error {
	p, err := getAwsCloudWatchMetricAlarm(d, meta)
	if err != nil {
		return err
	}
	if p == nil {
		log.Printf("[DEBUG] CloudWatch Metric Alarm %s is already gone", d.Id())
		return nil
	}

	log.Printf("[INFO] Deleting CloudWatch Metric Alarm: %s", d.Id())

	conn := meta.(*AWSClient).cloudwatchconn
	params := cloudwatch.DeleteAlarmsInput{
		AlarmNames: []*string{aws.String(d.Id())},
	}

	if _, err := conn.DeleteAlarms(&params); err != nil {
		return fmt.Errorf("Error deleting CloudWatch Metric Alarm: %s", err)
	}
	log.Println("[INFO] CloudWatch Metric Alarm deleted")

	return nil
}

func getAwsCloudWatchPutMetricAlarmInput(d *schema.ResourceData) cloudwatch.PutMetricAlarmInput {
	params := cloudwatch.PutMetricAlarmInput{
		AlarmName:          aws.String(d.Get("alarm_name").(string)),
		ComparisonOperator: aws.String(d.Get("comparison_operator").(string)),
		EvaluationPeriods:  aws.Int64(int64(d.Get("evaluation_periods").(int))),
		MetricName:         aws.String(d.Get("metric_name").(string)),
		Namespace:          aws.String(d.Get("namespace").(string)),
		Period:             aws.Int64(int64(d.Get("period").(int))),
		Threshold:          aws.Float64(d.Get("threshold").(float64)),
		TreatMissingData:   aws.String(d.Get("treat_missing_data").(string)),
	}

	if v := d.Get("actions_enabled"); v != nil {
		params.ActionsEnabled = aws.Bool(v.(bool))
	}

	if v, ok := d.GetOk("alarm_description"); ok {
		params.AlarmDescription = aws.String(v.(string))
	}

	if v, ok := d.GetOk("datapoints_to_alarm"); ok {
		params.DatapointsToAlarm = aws.Int64(int64(v.(int)))
	}

	if v, ok := d.GetOk("unit"); ok {
		params.Unit = aws.String(v.(string))
	}

	if v, ok := d.GetOk("statistic"); ok {
		params.Statistic = aws.String(v.(string))
	}

	if v, ok := d.GetOk("extended_statistic"); ok {
		params.ExtendedStatistic = aws.String(v.(string))
	}

	if v, ok := d.GetOk("evaluate_low_sample_count_percentiles"); ok {
		params.EvaluateLowSampleCountPercentile = aws.String(v.(string))
	}

	var alarmActions []*string
	if v := d.Get("alarm_actions"); v != nil {
		for _, v := range v.(*schema.Set).List() {
			str := v.(string)
			alarmActions = append(alarmActions, aws.String(str))
		}
		params.AlarmActions = alarmActions
	}

	var insufficientDataActions []*string
	if v := d.Get("insufficient_data_actions"); v != nil {
		for _, v := range v.(*schema.Set).List() {
			str := v.(string)
			insufficientDataActions = append(insufficientDataActions, aws.String(str))
		}
		params.InsufficientDataActions = insufficientDataActions
	}

	var okActions []*string
	if v := d.Get("ok_actions"); v != nil {
		for _, v := range v.(*schema.Set).List() {
			str := v.(string)
			okActions = append(okActions, aws.String(str))
		}
		params.OKActions = okActions
	}

	a := d.Get("dimensions").(map[string]interface{})
	dimensions := make([]*cloudwatch.Dimension, 0, len(a))
	for k, v := range a {
		dimensions = append(dimensions, &cloudwatch.Dimension{
			Name:  aws.String(k),
			Value: aws.String(v.(string)),
		})
	}
	params.Dimensions = dimensions

	return params
}

func getAwsCloudWatchMetricAlarm(d *schema.ResourceData, meta interface{}) (*cloudwatch.MetricAlarm, error) {
	conn := meta.(*AWSClient).cloudwatchconn

	params := cloudwatch.DescribeAlarmsInput{
		AlarmNames: []*string{aws.String(d.Id())},
	}

	resp, err := conn.DescribeAlarms(&params)
	if err != nil {
		return nil, err
	}

	// Find it and return it
	for idx, ma := range resp.MetricAlarms {
		if *ma.AlarmName == d.Id() {
			return resp.MetricAlarms[idx], nil
		}
	}

	return nil, nil
}

func _strArrPtrToList(strArrPtr []*string) []string {
	var result []string
	for _, elem := range strArrPtr {
		result = append(result, *elem)
	}
	return result
}

func flattenDimensions(dims []*cloudwatch.Dimension) map[string]interface{} {
	flatDims := make(map[string]interface{})
	for _, d := range dims {
		flatDims[*d.Name] = *d.Value
	}
	return flatDims
}
<|endoftext|>


<commit_before>package generator

import (
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"os"
	"socialapi/config"
	"socialapi/workers/helper"
	"socialapi/workers/sitemap/common"
	"socialapi/workers/sitemap/models"

	"github.com/garyburd/redigo/redis"
	"github.com/koding/logging"
	"github.com/robfig/cron"
)

type Controller struct {
	log          logging.Logger
	fileSelector FileSelector
	fileName     string
}

const (
	SCHEDULE = "0 0-59/30 * * * *"
)

var (
	cronJob *cron.Cron
)

func New(log logging.Logger) (*Controller, error) {
	c := &Controller{
		log:          log,
		fileSelector: CachedFileSelector{},
	}

	return c, c.initCron()
}

func (c *Controller) initCron() error {
	cronJob := cron.New()
	if err := cronJob.AddFunc(SCHEDULE, c.generate); err != nil {
		return err
	}
	cronJob.Start()

	return nil
}

func (c *Controller) Shutdown() {
	cronJob.Stop()
}

func (c *Controller) generate() {
	c.log.Info("Sitemap update started")
	for {
		name, err := c.fileSelector.Select()
		if err == redis.ErrNil {
			return
		}

		if err != nil {
			c.log.Error("Could not fetch file name: %s", err)
			return
		}
		c.log.Info("Updating sitemap: %s", name)
		// there is not any waiting sitemap updates
		if name == "" {
			return
		}

		c.fileName = name

		els, err := c.fetchElements()
		if err != nil {
			c.log.Error("Could not fetch updated elements: %s", err)
			return
		}

		container := c.buildContainer(els)

		s, err := c.getCurrentSet()
		if err != nil {
			c.log.Error("Could not get current set: %s", err)
			return
		}

		if err := c.updateFile(container, s); err != nil {
			c.log.Error("Could not update file: %s", err)
		}
	}
}

func (c *Controller) fetchElements() ([]*models.SitemapItem, error) {
	key := common.PrepareFileCacheKey(c.fileName)
	redisConn := helper.MustGetRedisConn()
	els := make([]*models.SitemapItem, 0)

	for {
		item, err := redisConn.PopSetMember(key)
		if err != nil && err != redis.ErrNil {
			return els, err
		}

		if item == "" {
			return els, nil
		}

		i := &models.SitemapItem{}

		if err := i.Populate(item); err != nil {
			c.log.Error("Could not update item %s: %s", item, err)
			continue
		}

		els = append(els, i)
	}
}

func (c *Controller) getCurrentSet() (*models.ItemSet, error) {
	// check if this is a new sitemap file or not
	n := fmt.Sprintf("%s.xml", c.fileName)
	if _, err := os.Stat(n); os.IsNotExist(err) {
		return models.NewItemSet(), nil
	}
	input, err := ioutil.ReadFile(n)
	if err != nil {
		return nil, err
	}

	s := models.NewItemSet()
	if err := xml.Unmarshal(input, s); err != nil {
		return nil, err
	}

	return s, nil
}

func (c *Controller) buildContainer(items []*models.SitemapItem) *models.ItemContainer {
	container := models.NewItemContainer()
	for _, v := range items {
		item := v.Definition(config.Get().Uri)
		switch v.Status {
		case models.STATUS_ADD:
			container.Add = append(container.Add, item)
		case models.STATUS_DELETE:
			container.Delete = append(container.Delete, item)
		case models.STATUS_UPDATE:
			container.Update = append(container.Update, item)
		}
	}

	return container
}

func (c *Controller) updateFile(container *models.ItemContainer, set *models.ItemSet) error {
	set.Populate(container)

	return common.XML(set, c.fileName)
}
<commit_msg>Sitemap: Update Sitemap.File table after sitemap file is updated<commit_after>package generator

import (
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"os"
	"socialapi/config"
	"socialapi/workers/helper"
	"socialapi/workers/sitemap/common"
	"socialapi/workers/sitemap/models"

	"github.com/garyburd/redigo/redis"
	"github.com/koding/logging"
	"github.com/robfig/cron"
)

type Controller struct {
	log          logging.Logger
	fileSelector FileSelector
	fileName     string
}

const (
	SCHEDULE = "0 0-59/30 * * * *"
)

var (
	cronJob *cron.Cron
)

func New(log logging.Logger) (*Controller, error) {
	c := &Controller{
		log:          log,
		fileSelector: CachedFileSelector{},
	}

	return c, c.initCron()
}

func (c *Controller) initCron() error {
	cronJob := cron.New()
	if err := cronJob.AddFunc(SCHEDULE, c.generate); err != nil {
		return err
	}
	cronJob.Start()

	return nil
}

func (c *Controller) Shutdown() {
	cronJob.Stop()
}

func (c *Controller) generate() {
	c.log.Info("Sitemap update started")
	for {
		name, err := c.fileSelector.Select()
		if err == redis.ErrNil {
			return
		}

		if err != nil {
			c.log.Error("Could not fetch file name: %s", err)
			return
		}
		c.log.Info("Updating sitemap: %s", name)
		// there is not any waiting sitemap updates
		if name == "" {
			return
		}

		c.fileName = name

		els, err := c.fetchElements()
		if err != nil {
			c.log.Error("Could not fetch updated elements: %s", err)
			return
		}

		container := c.buildContainer(els)

		s, err := c.getCurrentSet()
		if err != nil {
			c.log.Error("Could not get current set: %s", err)
			return
		}

		if err := c.updateFile(container, s); err != nil {
			c.log.Error("Could not update file: %s", err)
		}

		if err := new(models.SitemapFile).Upsert(name); err != nil {
			c.log.Error("Could not update file meta: %s", err)
		}

	}
}

func (c *Controller) fetchElements() ([]*models.SitemapItem, error) {
	key := common.PrepareFileCacheKey(c.fileName)
	redisConn := helper.MustGetRedisConn()
	els := make([]*models.SitemapItem, 0)

	for {
		item, err := redisConn.PopSetMember(key)
		if err != nil && err != redis.ErrNil {
			return els, err
		}

		if item == "" {
			return els, nil
		}

		i := &models.SitemapItem{}

		if err := i.Populate(item); err != nil {
			c.log.Error("Could not update item %s: %s", item, err)
			continue
		}

		els = append(els, i)
	}
}

func (c *Controller) getCurrentSet() (*models.ItemSet, error) {
	// check if this is a new sitemap file or not
	n := fmt.Sprintf("%s.xml", c.fileName)
	if _, err := os.Stat(n); os.IsNotExist(err) {
		return models.NewItemSet(), nil
	}
	input, err := ioutil.ReadFile(n)
	if err != nil {
		return nil, err
	}

	s := models.NewItemSet()
	if err := xml.Unmarshal(input, s); err != nil {
		return nil, err
	}

	return s, nil
}

func (c *Controller) buildContainer(items []*models.SitemapItem) *models.ItemContainer {
	container := models.NewItemContainer()
	for _, v := range items {
		item := v.Definition(config.Get().Uri)
		switch v.Status {
		case models.STATUS_ADD:
			container.Add = append(container.Add, item)
		case models.STATUS_DELETE:
			container.Delete = append(container.Delete, item)
		case models.STATUS_UPDATE:
			container.Update = append(container.Update, item)
		}
	}

	return container
}

func (c *Controller) updateFile(container *models.ItemContainer, set *models.ItemSet) error {
	set.Populate(container)

	return common.XML(set, c.fileName)
}
<|endoftext|>