<commit_before>//  Copyright (c) 2015 ikawaha.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package dic

import (
	"archive/zip"
	"encoding/gob"
	"fmt"
	"io"

	"github.com/ikawaha/kagome/internal/fst"
)

// Dic represents a dictionary of a tokenizer.
type Dic struct {
	Morphs       []Morph
	Contents     [][]string
	Connection   ConnectionTable
	Index        Trie
	CharClass    []string
	CharCategory []byte
	InvokeList   []bool
	GroupList    []bool

	UnkMorphs   []Morph
	UnkIndex    map[int]int
	UnkIndexDup map[int]int
	UnkContents [][]string
}

func (d Dic) CharactorCategory(r rune) byte {
	if int(r) <= len(d.CharCategory) {
		return d.CharCategory[r]
	}
	return d.CharCategory[0] // default
}

func (d *Dic) loadMorphDicPart(r io.Reader) error {
	dec := gob.NewDecoder(r)
	if e := dec.Decode(&d.Morphs); e != nil {
		return fmt.Errorf("dic initializer, Morphs: %v", e)
	}
	if e := dec.Decode(&d.Contents); e != nil {
		return fmt.Errorf("dic initializer, Contents: %v", e)
	}
	return nil
}

func (d *Dic) loadIndexDicPart(r io.Reader) error {
	t, e := fst.Read(r)
	if e != nil {
		return fmt.Errorf("dic initializer, Index: %v", e)
	}
	d.Index = t
	return nil
}

func (d *Dic) loadConnectionDicPart(r io.Reader) error {
	dec := gob.NewDecoder(r)
	if e := dec.Decode(&d.Connection); e != nil {
		return fmt.Errorf("dic initializer, Connection: %v", e)
	}
	return nil
}

func (d *Dic) loadCharDefDicPart(r io.Reader) error {
	dec := gob.NewDecoder(r)
	if e := dec.Decode(&d.CharClass); e != nil {
		return fmt.Errorf("dic initializer, CharClass: %v", e)
	}
	if e := dec.Decode(&d.CharCategory); e != nil {
		return fmt.Errorf("dic initializer, CharCategory: %v", e)
	}
	if e := dec.Decode(&d.InvokeList); e != nil {
		return fmt.Errorf("dic initializer, InvokeList: %v", e)
	}
	if e := dec.Decode(&d.GroupList); e != nil {
		return fmt.Errorf("dic initializer, GroupList: %v", e)
	}
	return nil
}

func (d *Dic) loadUnkDicPart(r io.Reader) error {
	dec := gob.NewDecoder(r)
	if e := dec.Decode(&d.UnkMorphs); e != nil {
		return fmt.Errorf("dic initializer, UnkMorphs: %v", e)
	}
	if e := dec.Decode(&d.UnkIndex); e != nil {
		return fmt.Errorf("dic initializer, UnkIndex: %v", e)
	}
	if e := dec.Decode(&d.UnkIndexDup); e != nil {
		return fmt.Errorf("dic initializer, UnkIndexDup: %v", e)
	}
	if e := dec.Decode(&d.UnkContents); e != nil {
		return fmt.Errorf("dic initializer, UnkContents: %v", e)
	}
	return nil
}

func Load(path string) (d *Dic, err error) {
	d = new(Dic)
	r, err := zip.OpenReader(path)
	if err != nil {
		return
	}
	defer r.Close()

	for _, f := range r.File {
		if err = func() error {
			rc, e := f.Open()
			if e != nil {
				return e
			}
			rc.Close()
			switch f.Name {
			case "morph.dic":
				if e = d.loadMorphDicPart(rc); err != nil {
					return e
				}
			case "index.dic":
				if e = d.loadIndexDicPart(rc); err != nil {
					return e
				}
			case "connection.dic":
				if e = d.loadConnectionDicPart(rc); err != nil {
					return e
				}
			case "chardef.dic":
				if e = d.loadCharDefDicPart(rc); err != nil {
					return e
				}
			case "unk.dic":
				if e = d.loadUnkDicPart(rc); err != nil {
					return e
				}
			}
			return nil
		}(); err != nil {
			return
		}
	}
	return
}
<commit_msg>Replace the load methods of morphs and  the connection table<commit_after>//  Copyright (c) 2015 ikawaha.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package dic

import (
	"archive/zip"
	"encoding/gob"
	"fmt"
	"io"

	"github.com/ikawaha/kagome/internal/fst"
)

// Dic represents a dictionary of a tokenizer.
type Dic struct {
	Morphs       []Morph
	Contents     [][]string
	Connection   ConnectionTable
	Index        Trie
	CharClass    []string
	CharCategory []byte
	InvokeList   []bool
	GroupList    []bool

	UnkMorphs   []Morph
	UnkIndex    map[int]int
	UnkIndexDup map[int]int
	UnkContents [][]string
}

// CharactorCategory returns the category of a rune.
func (d Dic) CharactorCategory(r rune) byte {
	if int(r) <= len(d.CharCategory) {
		return d.CharCategory[r]
	}
	return d.CharCategory[0] // default
}

func (d *Dic) loadMorphDicPart(r io.Reader) error {
	m, e := LoadMorphSlice(r)
	if e != nil {
		return fmt.Errorf("dic initializer, Morphs: %v", e)
	}
	d.Morphs = m
	dec := gob.NewDecoder(r)
	if e := dec.Decode(&d.Contents); e != nil {
		return fmt.Errorf("dic initializer, Contents: %v", e)
	}
	return nil
}

func (d *Dic) loadIndexDicPart(r io.Reader) error {
	t, e := fst.Read(r)
	if e != nil {
		return fmt.Errorf("dic initializer, Index: %v", e)
	}
	d.Index = t
	return nil
}

func (d *Dic) loadConnectionDicPart(r io.Reader) error {
	t, e := LoadConnectionTable(r)
	if e != nil {
		return fmt.Errorf("dic initializer, Connection: %v", e)
	}
	d.Connection = t
	return nil
}

func (d *Dic) loadCharDefDicPart(r io.Reader) error {
	dec := gob.NewDecoder(r)
	if e := dec.Decode(&d.CharClass); e != nil {
		return fmt.Errorf("dic initializer, CharClass: %v", e)
	}
	if e := dec.Decode(&d.CharCategory); e != nil {
		return fmt.Errorf("dic initializer, CharCategory: %v", e)
	}
	if e := dec.Decode(&d.InvokeList); e != nil {
		return fmt.Errorf("dic initializer, InvokeList: %v", e)
	}
	if e := dec.Decode(&d.GroupList); e != nil {
		return fmt.Errorf("dic initializer, GroupList: %v", e)
	}
	return nil
}

func (d *Dic) loadUnkDicPart(r io.Reader) error {
	dec := gob.NewDecoder(r)
	if e := dec.Decode(&d.UnkMorphs); e != nil {
		return fmt.Errorf("dic initializer, UnkMorphs: %v", e)
	}
	if e := dec.Decode(&d.UnkIndex); e != nil {
		return fmt.Errorf("dic initializer, UnkIndex: %v", e)
	}
	if e := dec.Decode(&d.UnkIndexDup); e != nil {
		return fmt.Errorf("dic initializer, UnkIndexDup: %v", e)
	}
	if e := dec.Decode(&d.UnkContents); e != nil {
		return fmt.Errorf("dic initializer, UnkContents: %v", e)
	}
	return nil
}

// Load loads a dictionary from a file.
func Load(path string) (d *Dic, err error) {
	d = new(Dic)
	r, err := zip.OpenReader(path)
	if err != nil {
		return
	}
	defer r.Close()

	for _, f := range r.File {
		if err = func() error {
			rc, e := f.Open()
			if e != nil {
				return e
			}
			rc.Close()
			switch f.Name {
			case "morph.dic":
				if e = d.loadMorphDicPart(rc); err != nil {
					return e
				}
			case "index.dic":
				if e = d.loadIndexDicPart(rc); err != nil {
					return e
				}
			case "connection.dic":
				if e = d.loadConnectionDicPart(rc); err != nil {
					return e
				}
			case "chardef.dic":
				if e = d.loadCharDefDicPart(rc); err != nil {
					return e
				}
			case "unk.dic":
				if e = d.loadUnkDicPart(rc); err != nil {
					return e
				}
			}
			return nil
		}(); err != nil {
			return
		}
	}
	return
}
<|endoftext|>


<commit_before>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"database/sql"
	"flag"
	"fmt"
	"io"
	"os"
	"runtime"
	"strings"
	"time"

	"github.com/juju/errors"
	"github.com/ngaut/log"
	"github.com/peterh/liner"
	"github.com/pingcap/tidb"
	"github.com/pingcap/tidb/util/errors2"
	"github.com/pingcap/tidb/util/printer"
)

var (
	logLevel = flag.String("L", "error", "log level")
	store    = flag.String("store", "goleveldb", "the name for the registered storage, e.g. memory, goleveldb, boltdb")
	dbPath   = flag.String("dbpath", "test/test", "db path")

	line        *liner.State
	historyPath = "/tmp/tidb_interpreter"
)

func openHistory() {
	if f, err := os.Open(historyPath); err == nil {
		line.ReadHistory(f)
		f.Close()
	}
}

func saveHistory() {
	if f, err := os.Create(historyPath); err == nil {
		line.WriteHistory(f)
		f.Close()
	}
}

func executeLine(tx *sql.Tx, txnLine string) error {
	start := time.Now()
	if tidb.IsQuery(txnLine) {
		rows, err := tx.Query(txnLine)
		if err != nil {
			return errors.Trace(err)
		}
		defer rows.Close()
		cols, err := rows.Columns()
		if err != nil {
			return errors.Trace(err)
		}

		values := make([][]byte, len(cols))
		scanArgs := make([]interface{}, len(values))
		for i := range values {
			scanArgs[i] = &values[i]
		}

		var datas [][]string
		for rows.Next() {
			err := rows.Scan(scanArgs...)
			if err != nil {
				return errors.Trace(err)
			}

			data := make([]string, len(cols))
			for i, value := range values {
				if value == nil {
					data[i] = "NULL"
				} else {
					data[i] = string(value)
				}
			}

			datas = append(datas, data)
		}

		// For `cols` and `datas[i]` always has the same length,
		// no need to check return validity.
		result, _ := printer.GetPrintResult(cols, datas)
		fmt.Printf("%s", result)

		// report elapsed time and rows in set
		elapsed := time.Since(start).Seconds()
		switch len(datas) {
		case 0:
			fmt.Printf("Empty set (%.2f sec)\n", elapsed)
		case 1:
			fmt.Printf("1 row in set (%.2f sec)\n", elapsed)
		default:
			fmt.Printf("%v rows in set (%.2f sec)\n", len(datas), elapsed)
		}

		if err := rows.Err(); err != nil {
			return errors.Trace(err)
		}
	} else {
		// TODO: rows affected and last insert id
		_, err := tx.Exec(txnLine)
		if err != nil {
			return errors.Trace(err)
		}
	}
	return nil
}

func mayExit(err error, l string) bool {
	if errors2.ErrorEqual(err, liner.ErrPromptAborted) || errors2.ErrorEqual(err, io.EOF) {
		fmt.Println("\nBye")
		saveHistory()
		return true
	}
	if err != nil {
		log.Fatal(errors.ErrorStack(err))
	}
	return false
}

func readStatement(prompt string) (string, error) {
	var ret string
	for {
		l, err := line.Prompt(prompt)
		if err != nil {
			return "", err
		}
		if strings.HasSuffix(l, ";") == false {
			ret += l + "\n"
			prompt = "   -> "
			continue
		}
		return ret + l, nil
	}
}

func main() {
	printer.PrintTiDBInfo()

	flag.Parse()
	log.SetLevelByString(*logLevel)
	// support for signal notify
	runtime.GOMAXPROCS(runtime.NumCPU())

	line = liner.NewLiner()
	defer line.Close()

	line.SetCtrlCAborts(true)
	openHistory()

	mdb, err := sql.Open(tidb.DriverName, *store+"://"+*dbPath)
	if err != nil {
		log.Fatal(errors.ErrorStack(err))
	}

	for {
		l, err := readStatement("tidb> ")
		if mayExit(err, l) {
			return
		}
		line.AppendHistory(l)

		// if we're in transaction
		if strings.HasPrefix(l, "BEGIN") || strings.HasPrefix(l, "begin") {
			tx, err := mdb.Begin()
			if err != nil {
				log.Error(errors.ErrorStack(err))
				continue
			}
			for {
				txnLine, err := readStatement(">> ")
				if mayExit(err, txnLine) {
					return
				}
				line.AppendHistory(txnLine)

				if !strings.HasSuffix(txnLine, ";") {
					txnLine += ";"
				}

				if strings.HasPrefix(txnLine, "COMMIT") || strings.HasPrefix(txnLine, "commit") {
					err := tx.Commit()
					if err != nil {
						log.Error(errors.ErrorStack(err))
						tx.Rollback()
					}
					break
				}
				// normal sql statement
				err = executeLine(tx, txnLine)
				if err != nil {
					log.Error(errors.ErrorStack(err))
					tx.Rollback()
					break
				}
			}
		} else {
			tx, err := mdb.Begin()
			if err != nil {
				log.Error(errors.ErrorStack(err))
				continue
			}
			err = executeLine(tx, l)
			if err != nil {
				log.Error(errors.ErrorStack(err))
				tx.Rollback()
			} else {
				tx.Commit()
			}
		}
	}
}
<commit_msg>time and affected row report to insert/update<commit_after>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"database/sql"
	"flag"
	"fmt"
	"io"
	"os"
	"runtime"
	"strings"
	"time"

	"github.com/juju/errors"
	"github.com/ngaut/log"
	"github.com/peterh/liner"
	"github.com/pingcap/tidb"
	"github.com/pingcap/tidb/util/errors2"
	"github.com/pingcap/tidb/util/printer"
)

var (
	logLevel = flag.String("L", "error", "log level")
	store    = flag.String("store", "goleveldb", "the name for the registered storage, e.g. memory, goleveldb, boltdb")
	dbPath   = flag.String("dbpath", "test/test", "db path")

	line        *liner.State
	historyPath = "/tmp/tidb_interpreter"
)

func openHistory() {
	if f, err := os.Open(historyPath); err == nil {
		line.ReadHistory(f)
		f.Close()
	}
}

func saveHistory() {
	if f, err := os.Create(historyPath); err == nil {
		line.WriteHistory(f)
		f.Close()
	}
}

func executeLine(tx *sql.Tx, txnLine string) error {
	start := time.Now()
	if tidb.IsQuery(txnLine) {
		rows, err := tx.Query(txnLine)
		if err != nil {
			return errors.Trace(err)
		}
		defer rows.Close()
		cols, err := rows.Columns()
		if err != nil {
			return errors.Trace(err)
		}

		values := make([][]byte, len(cols))
		scanArgs := make([]interface{}, len(values))
		for i := range values {
			scanArgs[i] = &values[i]
		}

		var datas [][]string
		for rows.Next() {
			err := rows.Scan(scanArgs...)
			if err != nil {
				return errors.Trace(err)
			}

			data := make([]string, len(cols))
			for i, value := range values {
				if value == nil {
					data[i] = "NULL"
				} else {
					data[i] = string(value)
				}
			}

			datas = append(datas, data)
		}

		// For `cols` and `datas[i]` always has the same length,
		// no need to check return validity.
		result, _ := printer.GetPrintResult(cols, datas)
		fmt.Printf("%s", result)

		// report elapsed time and rows in set
		switch len(datas) {
		case 0:
			fmt.Printf("Empty set")
		case 1:
			fmt.Printf("1 row in set")
		default:
			fmt.Printf("%v rows in set", len(datas))
		}

		if err := rows.Err(); err != nil {
			return errors.Trace(err)
		}
	} else {
		// TODO: rows affected and last insert id
		res, err := tx.Exec(txnLine)
		if err != nil {
			return errors.Trace(err)
		}
		cnt, err := res.RowsAffected()
		if err != nil {
			return errors.Trace(err)
		}
		switch cnt {
		case 0:
			fmt.Printf("Query OK, 0 row affected")
		case 1:
			fmt.Printf("Query OK, 1 row affected")
		default:
			fmt.Printf("Query OK, %v rows affected", cnt)
		}
	}
	fmt.Printf(" (%.2f sec)\n", time.Since(start).Seconds())
	return nil
}

func mayExit(err error, l string) bool {
	if errors2.ErrorEqual(err, liner.ErrPromptAborted) || errors2.ErrorEqual(err, io.EOF) {
		fmt.Println("\nBye")
		saveHistory()
		return true
	}
	if err != nil {
		log.Fatal(errors.ErrorStack(err))
	}
	return false
}

func readStatement(prompt string) (string, error) {
	var ret string
	for {
		l, err := line.Prompt(prompt)
		if err != nil {
			return "", err
		}
		if strings.HasSuffix(l, ";") == false {
			ret += l + "\n"
			prompt = "   -> "
			continue
		}
		return ret + l, nil
	}
}

func main() {
	printer.PrintTiDBInfo()

	flag.Parse()
	log.SetLevelByString(*logLevel)
	// support for signal notify
	runtime.GOMAXPROCS(runtime.NumCPU())

	line = liner.NewLiner()
	defer line.Close()

	line.SetCtrlCAborts(true)
	openHistory()

	mdb, err := sql.Open(tidb.DriverName, *store+"://"+*dbPath)
	if err != nil {
		log.Fatal(errors.ErrorStack(err))
	}

	for {
		l, err := readStatement("tidb> ")
		if mayExit(err, l) {
			return
		}
		line.AppendHistory(l)

		// if we're in transaction
		if strings.HasPrefix(l, "BEGIN") || strings.HasPrefix(l, "begin") {
			tx, err := mdb.Begin()
			if err != nil {
				log.Error(errors.ErrorStack(err))
				continue
			}
			for {
				txnLine, err := readStatement(">> ")
				if mayExit(err, txnLine) {
					return
				}
				line.AppendHistory(txnLine)

				if !strings.HasSuffix(txnLine, ";") {
					txnLine += ";"
				}

				if strings.HasPrefix(txnLine, "COMMIT") || strings.HasPrefix(txnLine, "commit") {
					err := tx.Commit()
					if err != nil {
						log.Error(errors.ErrorStack(err))
						tx.Rollback()
					}
					break
				}
				// normal sql statement
				err = executeLine(tx, txnLine)
				if err != nil {
					log.Error(errors.ErrorStack(err))
					tx.Rollback()
					break
				}
			}
		} else {
			tx, err := mdb.Begin()
			if err != nil {
				log.Error(errors.ErrorStack(err))
				continue
			}
			err = executeLine(tx, l)
			if err != nil {
				log.Error(errors.ErrorStack(err))
				tx.Rollback()
			} else {
				tx.Commit()
			}
		}
	}
}
<|endoftext|>


<commit_before>package gorma_test

import (
	"fmt"
	"testing"

	"github.com/goadesign/gorma"
	"github.com/goadesign/gorma/dsl"
	"github.com/goadesign/goa/design"
)

func TestModelContext(t *testing.T) {
	sg := &gorma.RelationalModelDefinition{
		UserTypeDefinition: &design.UserTypeDefinition{
			AttributeDefinition: &design.AttributeDefinition{},
		},
	}

	sg.Type = design.String
	sg.ModelName = "SG"

	c := sg.Context()
	exp := fmt.Sprintf("RelationalModel %#v", sg.Name())
	if c != exp {
		t.Errorf("Expected %s, got %s", exp, c)
	}

	sg.ModelName = ""

	c = sg.Context()
	exp = "unnamed RelationalModel"
	if c != exp {
		t.Errorf("Expected %s, got %s", exp, c)
	}
}

func TestModelDSL(t *testing.T) {
	sg := &gorma.RelationalModelDefinition{}
	f := func() {
		return
	}
	sg.DefinitionDSL = f
	c := sg.DSL()
	if c == nil {
		t.Errorf("Expected %s, got nil", f)
	}

}

func TestPKAttributesSingle(t *testing.T) {
	sg := &gorma.RelationalModelDefinition{}
	sg.RelationalFields = make(map[string]*gorma.RelationalFieldDefinition)
	f := makePK("id")
	sg.PrimaryKeys = append(sg.PrimaryKeys, f)
	sg.RelationalFields[f.FieldName] = f

	pka := sg.PKAttributes()

	if pka != "id int" {
		t.Errorf("Expected %s, got %s", "id int", pka)
	}

}
func TestPKAttributesMultiple(t *testing.T) {
	sg := &gorma.RelationalModelDefinition{}
	sg.RelationalFields = make(map[string]*gorma.RelationalFieldDefinition)
	f := makePK("Field1")
	sg.PrimaryKeys = append(sg.PrimaryKeys, f)
	sg.RelationalFields[f.FieldName] = f

	f2 := makePK("Field2")
	sg.RelationalFields[f2.FieldName] = f2
	sg.PrimaryKeys = append(sg.PrimaryKeys, f2)

	pka := sg.PKAttributes()

	if pka != "field1 int,field2 int" {
		t.Errorf("Expected %s, got %s", "field1 int,field2 int", pka)
	}

}
func makePK(name string) *gorma.RelationalFieldDefinition {

	f := &gorma.RelationalFieldDefinition{}
	f.FieldName = dsl.SanitizeFieldName(name)
	f.DatabaseFieldName = dsl.SanitizeDBFieldName(f.FieldName)
	f.Datatype = gorma.Integer
	f.PrimaryKey = true
	return f

}
func TestPKWhereSingle(t *testing.T) {
	sg := &gorma.RelationalModelDefinition{}
	sg.RelationalFields = make(map[string]*gorma.RelationalFieldDefinition)
	f := &gorma.RelationalFieldDefinition{}
	f.FieldName = dsl.SanitizeFieldName("ID")
	f.DatabaseFieldName = dsl.SanitizeDBFieldName(f.FieldName)
	f.Datatype = gorma.Integer
	f.PrimaryKey = true

	sg.RelationalFields[f.FieldName] = f
	sg.PrimaryKeys = append(sg.PrimaryKeys, f)

	pkw := sg.PKWhere()

	if pkw != "id = ?" {
		t.Errorf("Expected %s, got %s", "id = ?", pkw)
	}

}

func TestPKWhereMultiple(t *testing.T) {
	sg := &gorma.RelationalModelDefinition{}
	sg.RelationalFields = make(map[string]*gorma.RelationalFieldDefinition)
	f := makePK("Field1")
	sg.RelationalFields[f.FieldName] = f
	sg.PrimaryKeys = append(sg.PrimaryKeys, f)

	f2 := makePK("Field2")
	sg.RelationalFields[f2.FieldName] = f2
	sg.PrimaryKeys = append(sg.PrimaryKeys, f2)

	pkw := sg.PKWhere()

	if pkw != "field1 = ? and field2 = ?" {
		t.Errorf("Expected %s, got %s", "field1 = ? and field2 = ?", pkw)
	}

}

func TestPKWhereFieldsSingle(t *testing.T) {
	sg := &gorma.RelationalModelDefinition{}
	sg.RelationalFields = make(map[string]*gorma.RelationalFieldDefinition)
	f := &gorma.RelationalFieldDefinition{}
	f.FieldName = dsl.SanitizeFieldName("ID")
	f.DatabaseFieldName = dsl.SanitizeDBFieldName(f.FieldName)
	f.Datatype = gorma.Integer
	f.PrimaryKey = true

	sg.RelationalFields[f.FieldName] = f
	sg.PrimaryKeys = append(sg.PrimaryKeys, f)

	pkw := sg.PKWhereFields()

	if pkw != "id" {
		t.Errorf("Expected %s, got %s", "id", pkw)
	}

}

func TestPKWhereFieldsMultiple(t *testing.T) {
	sg := &gorma.RelationalModelDefinition{}
	sg.RelationalFields = make(map[string]*gorma.RelationalFieldDefinition)
	f := makePK("Field1")
	sg.RelationalFields[f.FieldName] = f
	sg.PrimaryKeys = append(sg.PrimaryKeys, f)

	f2 := makePK("Field2")
	sg.RelationalFields[f2.FieldName] = f2
	sg.PrimaryKeys = append(sg.PrimaryKeys, f2)

	pkw := sg.PKWhereFields()

	if pkw != "field1,field2" {
		t.Errorf("Expected %s, got %s", "field1,field2", pkw)
	}

}

func TestPKUpdateFieldsSingle(t *testing.T) {
	sg := &gorma.RelationalModelDefinition{}
	sg.RelationalFields = make(map[string]*gorma.RelationalFieldDefinition)
	f := &gorma.RelationalFieldDefinition{}
	f.FieldName = dsl.SanitizeFieldName("ID")
	f.DatabaseFieldName = dsl.SanitizeDBFieldName(f.FieldName)
	f.Datatype = gorma.Integer
	f.PrimaryKey = true

	sg.RelationalFields[f.FieldName] = f
	sg.PrimaryKeys = append(sg.PrimaryKeys, f)

	pkw := sg.PKUpdateFields("model")

	if pkw != "model.ID" {
		t.Errorf("Expected %s, got %s", "model.ID", pkw)
	}

}

func TestPKUpdateFieldsMultiple(t *testing.T) {
	sg := &gorma.RelationalModelDefinition{}
	sg.RelationalFields = make(map[string]*gorma.RelationalFieldDefinition)
	f := makePK("Field1")
	sg.RelationalFields[f.FieldName] = f
	sg.PrimaryKeys = append(sg.PrimaryKeys, f)

	f2 := makePK("Field2")
	sg.RelationalFields[f2.FieldName] = f2
	sg.PrimaryKeys = append(sg.PrimaryKeys, f2)

	pkw := sg.PKUpdateFields("model")

	if pkw != "model.Field1,model.Field2" {
		t.Errorf("Expected %s, got %s", "model.Field1,model.Field2", pkw)
	}

}
<commit_msg>go fmt yourself<commit_after>package gorma_test

import (
	"fmt"
	"testing"

	"github.com/goadesign/goa/design"
	"github.com/goadesign/gorma"
	"github.com/goadesign/gorma/dsl"
)

func TestModelContext(t *testing.T) {
	sg := &gorma.RelationalModelDefinition{
		UserTypeDefinition: &design.UserTypeDefinition{
			AttributeDefinition: &design.AttributeDefinition{},
		},
	}

	sg.Type = design.String
	sg.ModelName = "SG"

	c := sg.Context()
	exp := fmt.Sprintf("RelationalModel %#v", sg.Name())
	if c != exp {
		t.Errorf("Expected %s, got %s", exp, c)
	}

	sg.ModelName = ""

	c = sg.Context()
	exp = "unnamed RelationalModel"
	if c != exp {
		t.Errorf("Expected %s, got %s", exp, c)
	}
}

func TestModelDSL(t *testing.T) {
	sg := &gorma.RelationalModelDefinition{}
	f := func() {
		return
	}
	sg.DefinitionDSL = f
	c := sg.DSL()
	if c == nil {
		t.Errorf("Expected %v, got nil", f)
	}

}

func TestPKAttributesSingle(t *testing.T) {
	sg := &gorma.RelationalModelDefinition{}
	sg.RelationalFields = make(map[string]*gorma.RelationalFieldDefinition)
	f := makePK("id")
	sg.PrimaryKeys = append(sg.PrimaryKeys, f)
	sg.RelationalFields[f.FieldName] = f

	pka := sg.PKAttributes()

	if pka != "id int" {
		t.Errorf("Expected %s, got %s", "id int", pka)
	}

}
func TestPKAttributesMultiple(t *testing.T) {
	sg := &gorma.RelationalModelDefinition{}
	sg.RelationalFields = make(map[string]*gorma.RelationalFieldDefinition)
	f := makePK("Field1")
	sg.PrimaryKeys = append(sg.PrimaryKeys, f)
	sg.RelationalFields[f.FieldName] = f

	f2 := makePK("Field2")
	sg.RelationalFields[f2.FieldName] = f2
	sg.PrimaryKeys = append(sg.PrimaryKeys, f2)

	pka := sg.PKAttributes()

	if pka != "field1 int,field2 int" {
		t.Errorf("Expected %s, got %s", "field1 int,field2 int", pka)
	}

}
func makePK(name string) *gorma.RelationalFieldDefinition {

	f := &gorma.RelationalFieldDefinition{}
	f.FieldName = dsl.SanitizeFieldName(name)
	f.DatabaseFieldName = dsl.SanitizeDBFieldName(f.FieldName)
	f.Datatype = gorma.Integer
	f.PrimaryKey = true
	return f

}
func TestPKWhereSingle(t *testing.T) {
	sg := &gorma.RelationalModelDefinition{}
	sg.RelationalFields = make(map[string]*gorma.RelationalFieldDefinition)
	f := &gorma.RelationalFieldDefinition{}
	f.FieldName = dsl.SanitizeFieldName("ID")
	f.DatabaseFieldName = dsl.SanitizeDBFieldName(f.FieldName)
	f.Datatype = gorma.Integer
	f.PrimaryKey = true

	sg.RelationalFields[f.FieldName] = f
	sg.PrimaryKeys = append(sg.PrimaryKeys, f)

	pkw := sg.PKWhere()

	if pkw != "id = ?" {
		t.Errorf("Expected %s, got %s", "id = ?", pkw)
	}

}

func TestPKWhereMultiple(t *testing.T) {
	sg := &gorma.RelationalModelDefinition{}
	sg.RelationalFields = make(map[string]*gorma.RelationalFieldDefinition)
	f := makePK("Field1")
	sg.RelationalFields[f.FieldName] = f
	sg.PrimaryKeys = append(sg.PrimaryKeys, f)

	f2 := makePK("Field2")
	sg.RelationalFields[f2.FieldName] = f2
	sg.PrimaryKeys = append(sg.PrimaryKeys, f2)

	pkw := sg.PKWhere()

	if pkw != "field1 = ? and field2 = ?" {
		t.Errorf("Expected %s, got %s", "field1 = ? and field2 = ?", pkw)
	}

}

func TestPKWhereFieldsSingle(t *testing.T) {
	sg := &gorma.RelationalModelDefinition{}
	sg.RelationalFields = make(map[string]*gorma.RelationalFieldDefinition)
	f := &gorma.RelationalFieldDefinition{}
	f.FieldName = dsl.SanitizeFieldName("ID")
	f.DatabaseFieldName = dsl.SanitizeDBFieldName(f.FieldName)
	f.Datatype = gorma.Integer
	f.PrimaryKey = true

	sg.RelationalFields[f.FieldName] = f
	sg.PrimaryKeys = append(sg.PrimaryKeys, f)

	pkw := sg.PKWhereFields()

	if pkw != "id" {
		t.Errorf("Expected %s, got %s", "id", pkw)
	}

}

func TestPKWhereFieldsMultiple(t *testing.T) {
	sg := &gorma.RelationalModelDefinition{}
	sg.RelationalFields = make(map[string]*gorma.RelationalFieldDefinition)
	f := makePK("Field1")
	sg.RelationalFields[f.FieldName] = f
	sg.PrimaryKeys = append(sg.PrimaryKeys, f)

	f2 := makePK("Field2")
	sg.RelationalFields[f2.FieldName] = f2
	sg.PrimaryKeys = append(sg.PrimaryKeys, f2)

	pkw := sg.PKWhereFields()

	if pkw != "field1,field2" {
		t.Errorf("Expected %s, got %s", "field1,field2", pkw)
	}

}

func TestPKUpdateFieldsSingle(t *testing.T) {
	sg := &gorma.RelationalModelDefinition{}
	sg.RelationalFields = make(map[string]*gorma.RelationalFieldDefinition)
	f := &gorma.RelationalFieldDefinition{}
	f.FieldName = dsl.SanitizeFieldName("ID")
	f.DatabaseFieldName = dsl.SanitizeDBFieldName(f.FieldName)
	f.Datatype = gorma.Integer
	f.PrimaryKey = true

	sg.RelationalFields[f.FieldName] = f
	sg.PrimaryKeys = append(sg.PrimaryKeys, f)

	pkw := sg.PKUpdateFields("model")

	if pkw != "model.ID" {
		t.Errorf("Expected %s, got %s", "model.ID", pkw)
	}

}

func TestPKUpdateFieldsMultiple(t *testing.T) {
	sg := &gorma.RelationalModelDefinition{}
	sg.RelationalFields = make(map[string]*gorma.RelationalFieldDefinition)
	f := makePK("Field1")
	sg.RelationalFields[f.FieldName] = f
	sg.PrimaryKeys = append(sg.PrimaryKeys, f)

	f2 := makePK("Field2")
	sg.RelationalFields[f2.FieldName] = f2
	sg.PrimaryKeys = append(sg.PrimaryKeys, f2)

	pkw := sg.PKUpdateFields("model")

	if pkw != "model.Field1,model.Field2" {
		t.Errorf("Expected %s, got %s", "model.Field1,model.Field2", pkw)
	}

}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math"
	"math/big"
	"net/http"
	"os"
	"path"
	"strings"

	"archive/tar"
	"crypto/rand"

	"github.com/fsouza/go-dockerclient"
	"golang.org/x/net/webdav"
)

type APIVersions struct {
	Versions []string `json:"versions"`
}

const (
	VERSION_TAG        = "v1"
	DOCKER_TAR_PREFIX  = "rootfs/"
	OWNER_PERM_RW      = 0600
	HEALTHZ_URL_PATH   = "/healthz"
	API_URL_PREFIX     = "/api"
	CONTENT_URL_PREFIX = API_URL_PREFIX + "/" + VERSION_TAG + "/content/"
	METADATA_URL_PATH  = API_URL_PREFIX + "/" + VERSION_TAG + "/metadata"
)

func handleTarStream(reader io.ReadCloser, destination string) {
	tr := tar.NewReader(reader)
	if tr != nil {
		err := processTarStream(tr, destination)
		if err != nil {
			log.Print(err)
		}
	} else {
		log.Printf("Unable to create image tar reader")
	}
	reader.Close()
}

func processTarStream(tr *tar.Reader, destination string) error {
	for {
		hdr, err := tr.Next()
		if err != nil {
			if err == io.EOF {
				return nil
			}
			return fmt.Errorf("Unable to extract container: %v\n", err)
		}

		hdrInfo := hdr.FileInfo()

		path := path.Join(destination, strings.TrimPrefix(hdr.Name, DOCKER_TAR_PREFIX))
		// Overriding permissions to allow writing content
		mode := hdrInfo.Mode() | OWNER_PERM_RW

		switch hdr.Typeflag {
		case tar.TypeDir:
			if err := os.Mkdir(path, mode); err != nil {
				if !os.IsExist(err) {
					return fmt.Errorf("Unable to create directory: %v", err)
				}
				err = os.Chmod(path, mode)
				if err != nil {
					return fmt.Errorf("Unable to update directory mode: %v", err)
				}
			}
		case tar.TypeReg, tar.TypeRegA:
			file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode)
			if err != nil {
				return fmt.Errorf("Unable to create file: %v", err)
			}
			if _, err := io.Copy(file, tr); err != nil {
				file.Close()
				return fmt.Errorf("Unable to write into file: %v", err)
			}
			file.Close()
		default:
			// For now we're skipping anything else. Special device files and
			// symlinks are not needed or anyway probably incorrect.
		}
	}
}

func generateRandomName() string {
	n, err := rand.Int(rand.Reader, big.NewInt(math.MaxInt64))
	if err != nil {
		log.Fatalf("Unable to generate random container name: %v\n", err)
	}
	return fmt.Sprintf("image-inspector-%016x", n)
}

func main() {
	uri := flag.String("docker", "unix:///var/run/docker.sock", "Daemon socket to connect to")
	image := flag.String("image", "", "Docker image to inspect")
	path := flag.String("path", "", "Destination path for the image files")
	serve := flag.String("serve", "", "Host and port where to serve the image with webdav")

	flag.Parse()

	if *uri == "" {
		log.Fatalf("Docker socket connection must be specified")
	}
	if *image == "" {
		log.Fatalf("Docker image to inspect must be specified")
	}

	client, err := docker.NewClient(*uri)
	if err != nil {
		log.Fatalf("Unable to connect to docker daemon: %v\n", err)
	}

	if _, err := client.InspectImage(*image); err != nil {
		log.Printf("Pulling image %s", *image)
		imagePullOption := docker.PullImageOptions{Repository: *image}
		imagePullAuth := docker.AuthConfiguration{} // TODO: support authentication
		if err := client.PullImage(imagePullOption, imagePullAuth); err != nil {
			log.Fatalf("Unable to pull docker image: %v\n", err)
		}
	} else {
		log.Printf("Image %s is available, skipping image pull", *image)
	}

	// For security purpose we don't define any entrypoint and command
	container, err := client.CreateContainer(docker.CreateContainerOptions{
		Name: generateRandomName(),
		Config: &docker.Config{
			Image:      *image,
			Entrypoint: []string{""},
			Cmd:        []string{""},
		},
	})
	if err != nil {
		log.Fatalf("Unable to create docker container: %v\n", err)
	}

	containerMetadata, err := client.InspectContainer(container.ID)
	if err != nil {
		log.Fatalf("Unable to get docker container information: %v\n", err)
	}

	imageMetadata, err := client.InspectImage(containerMetadata.Image)
	if err != nil {
		log.Fatalf("Unable to get docker image information: %v\n", err)
	}

	if path != nil && *path != "" {
		err = os.Mkdir(*path, 0755)
		if err != nil {
			if !os.IsExist(err) {
				log.Fatalf("Unable to create destination path: %v\n", err)
			}
		}
	} else {
		// forcing to use /var/tmp because often it's not an in-memory tmpfs
		*path, err = ioutil.TempDir("/var/tmp", "image-inspector-")
		if err != nil {
			log.Fatalf("Unable to create temporary path: %v\n", err)
		}
	}

	reader, writer := io.Pipe()
	go handleTarStream(reader, *path)

	log.Printf("Extracting image %s to %s", *image, *path)
	_ = client.CopyFromContainer(docker.CopyFromContainerOptions{
		Container:    container.ID,
		OutputStream: writer,
		Resource:     "/",
	})

	_ = client.RemoveContainer(docker.RemoveContainerOptions{
		ID: container.ID,
	})

	supportedVersions := APIVersions{Versions: []string{VERSION_TAG}}

	if serve != nil && *serve != "" {
		log.Printf("Serving image content %s on webdav://%s%s", *path, *serve, CONTENT_URL_PREFIX)

		http.HandleFunc(HEALTHZ_URL_PATH, func(w http.ResponseWriter, r *http.Request) {
			w.Write([]byte("ok\n"))
		})

		http.HandleFunc(API_URL_PREFIX, func(w http.ResponseWriter, r *http.Request) {
			body, err := json.MarshalIndent(supportedVersions, "", "  ")
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			w.Write(body)
		})

		http.HandleFunc(METADATA_URL_PATH, func(w http.ResponseWriter, r *http.Request) {
			body, err := json.MarshalIndent(imageMetadata, "", "  ")
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			w.Write(body)
		})

		http.Handle(CONTENT_URL_PREFIX, &webdav.Handler{
			Prefix:     CONTENT_URL_PREFIX,
			FileSystem: webdav.Dir(*path),
			LockSystem: webdav.NewMemLS(),
		})

		log.Fatal(http.ListenAndServe(*serve, nil))
	}
}
<commit_msg>add hard failure on container copy errors<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math"
	"math/big"
	"net/http"
	"os"
	"path"
	"strings"

	"archive/tar"
	"crypto/rand"

	"github.com/fsouza/go-dockerclient"
	"golang.org/x/net/webdav"
)

type APIVersions struct {
	Versions []string `json:"versions"`
}

const (
	VERSION_TAG        = "v1"
	DOCKER_TAR_PREFIX  = "rootfs/"
	OWNER_PERM_RW      = 0600
	HEALTHZ_URL_PATH   = "/healthz"
	API_URL_PREFIX     = "/api"
	CONTENT_URL_PREFIX = API_URL_PREFIX + "/" + VERSION_TAG + "/content/"
	METADATA_URL_PATH  = API_URL_PREFIX + "/" + VERSION_TAG + "/metadata"
)

func handleTarStream(reader io.ReadCloser, destination string) {
	tr := tar.NewReader(reader)
	if tr != nil {
		err := processTarStream(tr, destination)
		if err != nil {
			log.Print(err)
		}
	} else {
		log.Printf("Unable to create image tar reader")
	}
	reader.Close()
}

func processTarStream(tr *tar.Reader, destination string) error {
	for {
		hdr, err := tr.Next()
		if err != nil {
			if err == io.EOF {
				return nil
			}
			return fmt.Errorf("Unable to extract container: %v\n", err)
		}

		hdrInfo := hdr.FileInfo()

		path := path.Join(destination, strings.TrimPrefix(hdr.Name, DOCKER_TAR_PREFIX))
		// Overriding permissions to allow writing content
		mode := hdrInfo.Mode() | OWNER_PERM_RW

		switch hdr.Typeflag {
		case tar.TypeDir:
			if err := os.Mkdir(path, mode); err != nil {
				if !os.IsExist(err) {
					return fmt.Errorf("Unable to create directory: %v", err)
				}
				err = os.Chmod(path, mode)
				if err != nil {
					return fmt.Errorf("Unable to update directory mode: %v", err)
				}
			}
		case tar.TypeReg, tar.TypeRegA:
			file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode)
			if err != nil {
				return fmt.Errorf("Unable to create file: %v", err)
			}
			if _, err := io.Copy(file, tr); err != nil {
				file.Close()
				return fmt.Errorf("Unable to write into file: %v", err)
			}
			file.Close()
		default:
			// For now we're skipping anything else. Special device files and
			// symlinks are not needed or anyway probably incorrect.
		}
	}
}

func generateRandomName() string {
	n, err := rand.Int(rand.Reader, big.NewInt(math.MaxInt64))
	if err != nil {
		log.Fatalf("Unable to generate random container name: %v\n", err)
	}
	return fmt.Sprintf("image-inspector-%016x", n)
}

func main() {
	uri := flag.String("docker", "unix:///var/run/docker.sock", "Daemon socket to connect to")
	image := flag.String("image", "", "Docker image to inspect")
	path := flag.String("path", "", "Destination path for the image files")
	serve := flag.String("serve", "", "Host and port where to serve the image with webdav")

	flag.Parse()

	if *uri == "" {
		log.Fatalf("Docker socket connection must be specified")
	}
	if *image == "" {
		log.Fatalf("Docker image to inspect must be specified")
	}

	client, err := docker.NewClient(*uri)
	if err != nil {
		log.Fatalf("Unable to connect to docker daemon: %v\n", err)
	}

	if _, err := client.InspectImage(*image); err != nil {
		log.Printf("Pulling image %s", *image)
		imagePullOption := docker.PullImageOptions{Repository: *image}
		imagePullAuth := docker.AuthConfiguration{} // TODO: support authentication
		if err := client.PullImage(imagePullOption, imagePullAuth); err != nil {
			log.Fatalf("Unable to pull docker image: %v\n", err)
		}
	} else {
		log.Printf("Image %s is available, skipping image pull", *image)
	}

	// For security purpose we don't define any entrypoint and command
	container, err := client.CreateContainer(docker.CreateContainerOptions{
		Name: generateRandomName(),
		Config: &docker.Config{
			Image:      *image,
			Entrypoint: []string{""},
			Cmd:        []string{""},
		},
	})
	if err != nil {
		log.Fatalf("Unable to create docker container: %v\n", err)
	}

	containerMetadata, err := client.InspectContainer(container.ID)
	if err != nil {
		log.Fatalf("Unable to get docker container information: %v\n", err)
	}

	imageMetadata, err := client.InspectImage(containerMetadata.Image)
	if err != nil {
		log.Fatalf("Unable to get docker image information: %v\n", err)
	}

	if path != nil && *path != "" {
		err = os.Mkdir(*path, 0755)
		if err != nil {
			if !os.IsExist(err) {
				log.Fatalf("Unable to create destination path: %v\n", err)
			}
		}
	} else {
		// forcing to use /var/tmp because often it's not an in-memory tmpfs
		*path, err = ioutil.TempDir("/var/tmp", "image-inspector-")
		if err != nil {
			log.Fatalf("Unable to create temporary path: %v\n", err)
		}
	}

	reader, writer := io.Pipe()
	go handleTarStream(reader, *path)

	log.Printf("Extracting image %s to %s", *image, *path)
	err = client.CopyFromContainer(docker.CopyFromContainerOptions{
		Container:    container.ID,
		OutputStream: writer,
		Resource:     "/",
	})
	if err != nil {
		log.Fatalf("Unable to extract container: %v\n", err)
	}

	_ = client.RemoveContainer(docker.RemoveContainerOptions{
		ID: container.ID,
	})

	supportedVersions := APIVersions{Versions: []string{VERSION_TAG}}

	if serve != nil && *serve != "" {
		log.Printf("Serving image content %s on webdav://%s%s", *path, *serve, CONTENT_URL_PREFIX)

		http.HandleFunc(HEALTHZ_URL_PATH, func(w http.ResponseWriter, r *http.Request) {
			w.Write([]byte("ok\n"))
		})

		http.HandleFunc(API_URL_PREFIX, func(w http.ResponseWriter, r *http.Request) {
			body, err := json.MarshalIndent(supportedVersions, "", "  ")
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			w.Write(body)
		})

		http.HandleFunc(METADATA_URL_PATH, func(w http.ResponseWriter, r *http.Request) {
			body, err := json.MarshalIndent(imageMetadata, "", "  ")
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			w.Write(body)
		})

		http.Handle(CONTENT_URL_PREFIX, &webdav.Handler{
			Prefix:     CONTENT_URL_PREFIX,
			FileSystem: webdav.Dir(*path),
			LockSystem: webdav.NewMemLS(),
		})

		log.Fatal(http.ListenAndServe(*serve, nil))
	}
}
<|endoftext|>


<commit_before>/*
 * Copyright 2011 Nan Deng
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package push

import (
	"os/signal"
	"os"
	"syscall"
	"fmt"
)

func (s *UniqushSystem) signalSetup() {
	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGTERM, os.Kill)
	<-ch
	fmt.Printf("OK. Termitate!")
	s.Stopch <- true
}
<commit_msg>remove debug info<commit_after>/*
 * Copyright 2011 Nan Deng
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package push

import (
	"fmt"
	"os"
	"os/signal"
	"syscall"
)

func (s *UniqushSystem) signalSetup() {
	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGTERM, os.Kill)
	<-ch
	s.Stopch <- true
}
<|endoftext|>


<commit_before>/*
Copyright (c) 2014 Ashley Jeffs

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

package input

import (
	"fmt"
	"sync/atomic"
	"time"

	"github.com/jeffail/benthos/lib/types"
	"github.com/jeffail/util/log"
	"github.com/jeffail/util/metrics"
	"github.com/streadway/amqp"
)

//--------------------------------------------------------------------------------------------------

func init() {
	constructors["amqp"] = typeSpec{
		constructor: NewAMQP,
		description: `
AMQP is the underlying messaging protocol that is used my RabbitMQ. Support is
currently rather limited, but more configuration options are on the way.

Exchange type options are: direct|fanout|topic|x-custom`,
	}
}

//--------------------------------------------------------------------------------------------------

// AMQPConfig - Configuration for the AMQP input type.
type AMQPConfig struct {
	URI          string `json:"uri" yaml:"uri"`
	Exchange     string `json:"exchange" yaml:"exchange"`
	ExchangeType string `json:"exchange_type" yaml:"exchange_type"`
	Queue        string `json:"queue" yaml:"queue"`
	BindingKey   string `json:"key" yaml:"key"`
	ConsumerTag  string `json:"consumer_tag" yaml:"consumer_tag"`
}

// NewAMQPConfig - Creates a new AMQPConfig with default values.
func NewAMQPConfig() AMQPConfig {
	return AMQPConfig{
		URI:          "amqp://guest:guest@localhost:5672/",
		Exchange:     "benthos-exchange",
		ExchangeType: "direct",
		Queue:        "benthos-queue",
		BindingKey:   "benthos-key",
		ConsumerTag:  "benthos-consumer",
	}
}

//--------------------------------------------------------------------------------------------------

// AMQP - An input type that serves Scalability Protocols messages.
type AMQP struct {
	running int32

	conn         *amqp.Connection
	amqpChan     *amqp.Channel
	consumerChan <-chan amqp.Delivery

	conf  Config
	stats metrics.Type
	log   log.Modular

	messages  chan types.Message
	responses <-chan types.Response

	closeChan  chan struct{}
	closedChan chan struct{}
}

// NewAMQP - Create a new AMQP input type.
func NewAMQP(conf Config, log log.Modular, stats metrics.Type) (Type, error) {
	a := AMQP{
		running:    1,
		conf:       conf,
		stats:      stats,
		log:        log.NewModule(".input.amqp"),
		messages:   make(chan types.Message),
		responses:  nil,
		closeChan:  make(chan struct{}),
		closedChan: make(chan struct{}),
	}

	if err := a.connect(); err != nil {
		return nil, err
	}

	return &a, nil
}

//--------------------------------------------------------------------------------------------------

// connect - Establish a connection to an AMQP server.
func (a *AMQP) connect() (err error) {
	a.conn, err = amqp.Dial(a.conf.AMQP.URI)
	if err != nil {
		return fmt.Errorf("AMQP Connect: %s", err)
	}

	a.amqpChan, err = a.conn.Channel()
	if err != nil {
		return fmt.Errorf("AMQP Channel: %s", err)
	}

	if err = a.amqpChan.ExchangeDeclare(
		a.conf.AMQP.Exchange,     // name of the exchange
		a.conf.AMQP.ExchangeType, // type
		true,  // durable
		false, // delete when complete
		false, // internal
		false, // noWait
		nil,   // arguments
	); err != nil {
		return fmt.Errorf("Exchange Declare: %s", err)
	}

	if _, err = a.amqpChan.QueueDeclare(
		a.conf.AMQP.Queue, // name of the queue
		true,              // durable
		false,             // delete when usused
		false,             // exclusive
		false,             // noWait
		nil,               // arguments
	); err != nil {
		return fmt.Errorf("Queue Declare: %s", err)
	}

	if err = a.amqpChan.QueueBind(
		a.conf.AMQP.Queue,      // name of the queue
		a.conf.AMQP.BindingKey, // bindingKey
		a.conf.AMQP.Exchange,   // sourceExchange
		false,                  // noWait
		nil,                    // arguments
	); err != nil {
		return fmt.Errorf("Queue Bind: %s", err)
	}

	if a.consumerChan, err = a.amqpChan.Consume(
		a.conf.AMQP.Queue,       // name
		a.conf.AMQP.ConsumerTag, // consumerTag,
		false, // noAck
		false, // exclusive
		false, // noLocal
		false, // noWait
		nil,   // arguments
	); err != nil {
		return fmt.Errorf("Queue Consume: %s", err)
	}

	return
}

// disconnect - Safely close a connection to an AMQP server.
func (a *AMQP) disconnect() error {
	if a.amqpChan != nil {
		if err := a.amqpChan.Cancel(a.conf.AMQP.ConsumerTag, true); err != nil {
			return fmt.Errorf("Consumer cancel failed: %s", err)
		}
		a.amqpChan = nil
	}
	if a.conn != nil {
		if err := a.conn.Close(); err != nil {
			return fmt.Errorf("AMQP connection close error: %s", err)
		}
		a.conn = nil
	}
	return nil
}

//--------------------------------------------------------------------------------------------------

func (a *AMQP) loop() {
	defer func() {
		atomic.StoreInt32(&a.running, 0)
		a.disconnect()

		close(a.messages)
		close(a.closedChan)
	}()

	var data []byte

	for atomic.LoadInt32(&a.running) == 1 {
		// If no bytes then read a message
		if data == nil {
			select {
			case msg := <-a.consumerChan:
				data = msg.Body
			case <-a.closeChan:
				return
			}
		}

		// If bytes are read then try and propagate.
		if data != nil {
			select {
			case a.messages <- types.Message{Parts: [][]byte{data}}:
			case <-a.closeChan:
				return
			}
			res, open := <-a.responses
			if !open {
				return
			}
			if resErr := res.Error(); resErr == nil {
				a.stats.Incr("input.amqp.count", 1)
				data = nil
			} else if resErr == types.ErrMessageTooLarge {
				a.stats.Incr("input.amqp.send.rejected", 1)
				data = nil
			} else {
				a.stats.Incr("input.amqp.send.error", 1)
			}
		}
	}

}

// StartListening - Sets the channel used by the input to validate message receipt.
func (a *AMQP) StartListening(responses <-chan types.Response) error {
	if a.responses != nil {
		return types.ErrAlreadyStarted
	}
	a.responses = responses
	go a.loop()
	return nil
}

// MessageChan - Returns the messages channel.
func (a *AMQP) MessageChan() <-chan types.Message {
	return a.messages
}

// CloseAsync - Shuts down the AMQP input and stops processing requests.
func (a *AMQP) CloseAsync() {
	if atomic.CompareAndSwapInt32(&a.running, 1, 0) {
		close(a.closeChan)
	}
}

// WaitForClose - Blocks until the AMQP input has closed down.
func (a *AMQP) WaitForClose(timeout time.Duration) error {
	select {
	case <-a.closedChan:
	case <-time.After(timeout):
		return types.ErrTimeout
	}
	return nil
}

//--------------------------------------------------------------------------------------------------
<commit_msg>Add Ack and Qos to AMQP input<commit_after>/*
Copyright (c) 2014 Ashley Jeffs

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

package input

import (
	"fmt"
	"sync/atomic"
	"time"

	"github.com/jeffail/benthos/lib/types"
	"github.com/jeffail/util/log"
	"github.com/jeffail/util/metrics"
	"github.com/streadway/amqp"
)

//--------------------------------------------------------------------------------------------------

func init() {
	constructors["amqp"] = typeSpec{
		constructor: NewAMQP,
		description: `
AMQP is the underlying messaging protocol that is used my RabbitMQ. Support is
currently rather limited, but more configuration options are on the way.

Exchange type options are: direct|fanout|topic|x-custom`,
	}
}

//--------------------------------------------------------------------------------------------------

// AMQPConfig - Configuration for the AMQP input type.
type AMQPConfig struct {
	URI           string `json:"uri" yaml:"uri"`
	Exchange      string `json:"exchange" yaml:"exchange"`
	ExchangeType  string `json:"exchange_type" yaml:"exchange_type"`
	Queue         string `json:"queue" yaml:"queue"`
	BindingKey    string `json:"key" yaml:"key"`
	ConsumerTag   string `json:"consumer_tag" yaml:"consumer_tag"`
	PrefetchCount int    `json:"prefetch_count"`
	PrefetchSize  int    `json:"prefetch_size"`
}

// NewAMQPConfig - Creates a new AMQPConfig with default values.
func NewAMQPConfig() AMQPConfig {
	return AMQPConfig{
		URI:           "amqp://guest:guest@localhost:5672/",
		Exchange:      "benthos-exchange",
		ExchangeType:  "direct",
		Queue:         "benthos-queue",
		BindingKey:    "benthos-key",
		ConsumerTag:   "benthos-consumer",
		PrefetchCount: 1,
		PrefetchSize:  0,
	}
}

//--------------------------------------------------------------------------------------------------

// AMQP - An input type that serves Scalability Protocols messages.
type AMQP struct {
	running int32

	conn         *amqp.Connection
	amqpChan     *amqp.Channel
	consumerChan <-chan amqp.Delivery

	conf  Config
	stats metrics.Type
	log   log.Modular

	messages  chan types.Message
	responses <-chan types.Response

	closeChan  chan struct{}
	closedChan chan struct{}
}

// NewAMQP - Create a new AMQP input type.
func NewAMQP(conf Config, log log.Modular, stats metrics.Type) (Type, error) {
	a := AMQP{
		running:    1,
		conf:       conf,
		stats:      stats,
		log:        log.NewModule(".input.amqp"),
		messages:   make(chan types.Message),
		responses:  nil,
		closeChan:  make(chan struct{}),
		closedChan: make(chan struct{}),
	}

	if err := a.connect(); err != nil {
		return nil, err
	}

	return &a, nil
}

//--------------------------------------------------------------------------------------------------

// connect - Establish a connection to an AMQP server.
func (a *AMQP) connect() (err error) {
	a.conn, err = amqp.Dial(a.conf.AMQP.URI)
	if err != nil {
		return fmt.Errorf("AMQP Connect: %s", err)
	}

	a.amqpChan, err = a.conn.Channel()
	if err != nil {
		return fmt.Errorf("AMQP Channel: %s", err)
	}

	if err = a.amqpChan.ExchangeDeclare(
		a.conf.AMQP.Exchange,     // name of the exchange
		a.conf.AMQP.ExchangeType, // type
		true,  // durable
		false, // delete when complete
		false, // internal
		false, // noWait
		nil,   // arguments
	); err != nil {
		return fmt.Errorf("Exchange Declare: %s", err)
	}

	if _, err = a.amqpChan.QueueDeclare(
		a.conf.AMQP.Queue, // name of the queue
		true,              // durable
		false,             // delete when usused
		false,             // exclusive
		false,             // noWait
		nil,               // arguments
	); err != nil {
		return fmt.Errorf("Queue Declare: %s", err)
	}

	if err = a.amqpChan.QueueBind(
		a.conf.AMQP.Queue,      // name of the queue
		a.conf.AMQP.BindingKey, // bindingKey
		a.conf.AMQP.Exchange,   // sourceExchange
		false,                  // noWait
		nil,                    // arguments
	); err != nil {
		return fmt.Errorf("Queue Bind: %s", err)
	}

	if err = a.amqpChan.Qos(
		a.conf.AMQP.PrefetchCount, a.conf.AMQP.PrefetchSize, false,
	); err != nil {
		return fmt.Errorf("Qos: %s", err)
	}

	if a.consumerChan, err = a.amqpChan.Consume(
		a.conf.AMQP.Queue,       // name
		a.conf.AMQP.ConsumerTag, // consumerTag,
		false, // noAck
		false, // exclusive
		false, // noLocal
		false, // noWait
		nil,   // arguments
	); err != nil {
		return fmt.Errorf("Queue Consume: %s", err)
	}

	return
}

// disconnect - Safely close a connection to an AMQP server.
func (a *AMQP) disconnect() error {
	if a.amqpChan != nil {
		if err := a.amqpChan.Cancel(a.conf.AMQP.ConsumerTag, true); err != nil {
			return fmt.Errorf("Consumer cancel failed: %s", err)
		}
		a.amqpChan = nil
	}
	if a.conn != nil {
		if err := a.conn.Close(); err != nil {
			return fmt.Errorf("AMQP connection close error: %s", err)
		}
		a.conn = nil
	}
	return nil
}

//--------------------------------------------------------------------------------------------------

func (a *AMQP) loop() {
	defer func() {
		atomic.StoreInt32(&a.running, 0)
		a.disconnect()

		close(a.messages)
		close(a.closedChan)
	}()

	var data *amqp.Delivery

	for atomic.LoadInt32(&a.running) == 1 {
		// If no bytes then read a message
		if data == nil {
			select {
			case msg := <-a.consumerChan:
				data = &msg
			case <-a.closeChan:
				return
			}
		}

		// If bytes are read then try and propagate.
		if data != nil {
			select {
			case a.messages <- types.Message{Parts: [][]byte{data.Body}}:
			case <-a.closeChan:
				return
			}
			res, open := <-a.responses
			if !open {
				return
			}
			if resErr := res.Error(); resErr == nil {
				a.stats.Incr("input.amqp.count", 1)
				data.Ack(false)
				data = nil
			} else if resErr == types.ErrMessageTooLarge {
				a.stats.Incr("input.amqp.send.rejected", 1)
				data = nil
			} else {
				a.stats.Incr("input.amqp.send.error", 1)
			}
		}
	}

}

// StartListening - Sets the channel used by the input to validate message receipt.
func (a *AMQP) StartListening(responses <-chan types.Response) error {
	if a.responses != nil {
		return types.ErrAlreadyStarted
	}
	a.responses = responses
	go a.loop()
	return nil
}

// MessageChan - Returns the messages channel.
func (a *AMQP) MessageChan() <-chan types.Message {
	return a.messages
}

// CloseAsync - Shuts down the AMQP input and stops processing requests.
func (a *AMQP) CloseAsync() {
	if atomic.CompareAndSwapInt32(&a.running, 1, 0) {
		close(a.closeChan)
	}
}

// WaitForClose - Blocks until the AMQP input has closed down.
func (a *AMQP) WaitForClose(timeout time.Duration) error {
	select {
	case <-a.closedChan:
	case <-time.After(timeout):
		return types.ErrTimeout
	}
	return nil
}

//--------------------------------------------------------------------------------------------------
<|endoftext|>


<commit_before>package filesystem

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/naoty/todo/todo"
)

// FileSystem represents a repository backed by file system.
type FileSystem struct {
	root string
}

type index struct {
	Todos map[string][]int `json:"todos"`
}

// New returns a new FileSystem.
func New() (*FileSystem, error) {
	home := os.Getenv("TODOS_PATH")
	if home == "" {
		var err error
		home, err = os.UserHomeDir()
		if err != nil {
			return nil, err
		}
	}

	root := filepath.Join(home, ".todos")
	if _, err := os.Stat(root); os.IsNotExist(err) {
		err := os.Mkdir(root, 0755)
		if err != nil {
			return nil, err
		}
	}

	return &FileSystem{root: root}, nil
}

// Get implements Repository interface.
func (repo *FileSystem) Get(id int) (*todo.Todo, error) {
	filename := fmt.Sprintf("%d.md", id)
	path := filepath.Join(repo.root, filename)

	if _, err := os.Stat(path); os.IsNotExist(err) {
		return nil, fmt.Errorf("TODO not found: %d", id)
	}

	file, err := os.Open(path)
	if err != nil {
		return nil, fmt.Errorf("failed to get TODO: %w", err)
	}
	defer file.Close()

	content, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, fmt.Errorf("failed to get TODO: %w", err)
	}

	td, err := Parse(string(content))
	if err != nil {
		return nil, fmt.Errorf("failed to get TODO: %w", err)
	}

	td.ID = id
	return td, nil
}

// List implements Repository interface.
func (repo *FileSystem) List() ([]*todo.Todo, error) {
	todos := map[int]*todo.Todo{}

	err := filepath.Walk(repo.root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if !strings.HasSuffix(path, ".md") {
			return nil
		}

		id, err := parseID(path)
		if err != nil {
			return err
		}

		file, err := os.Open(path)
		if err != nil {
			return err
		}
		defer file.Close()

		content, err := ioutil.ReadAll(file)
		if err != nil {
			return err
		}

		td, err := Parse(string(content))
		if err != nil {
			return err
		}

		td.ID = id
		todos[id] = td

		return nil
	})

	if err != nil {
		return nil, fmt.Errorf("failed to get todos from %s: %w", repo.root, err)
	}

	st, err := repo.readIndex()
	if err != nil {
		return nil, fmt.Errorf("failed to get todos from %s: %w", repo.root, err)
	}

	var ordered []*todo.Todo
	for k, ids := range st.Todos {
		// NOTE: when TODOs have subTODOs, k is parent TODO's ID.
		if k != "" {
			continue
		}

		for _, id := range ids {
			td, ok := todos[id]
			if !ok {
				continue
			}

			ordered = append(ordered, td)
		}
	}

	return ordered, nil
}

// Add implements Repository interface.
func (repo *FileSystem) Add(title string, parent *int) error {
	todos, err := repo.List()
	if err != nil {
		return fmt.Errorf("failed to get next id: %w", err)
	}

	lastID := 0
	for _, td := range todos {
		id := td.ID
		if id > lastID {
			lastID = id
		}
	}

	nextID := lastID + 1
	filename := fmt.Sprintf("%d.md", nextID)
	path := filepath.Join(repo.root, filename)

	if _, err := os.Stat(path); err == nil {
		return fmt.Errorf("already exist: %s", path)
	}

	file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644)
	if err != nil {
		return fmt.Errorf("failed to add TODO: %w", err)
	}
	defer file.Close()

	content := newContent(title)
	_, err = file.WriteString(content)
	if err != nil {
		return fmt.Errorf("failed to add TODO: %w", err)
	}

	st, err := repo.readIndex()
	if err != nil {
		return fmt.Errorf("failed to add TODO: %w", err)
	}

	st.Todos[""] = append(st.Todos[""], nextID)
	err = repo.writeIndex(st)
	if err != nil {
		return fmt.Errorf("failed to add TODO: %w", err)
	}

	return nil
}

// Update implements Repository interface.
func (repo *FileSystem) Update(td *todo.Todo) error {
	filename := fmt.Sprintf("%d.md", td.ID)
	path := filepath.Join(repo.root, filename)

	if _, err := os.Stat(path); os.IsNotExist(err) {
		return fmt.Errorf("file not found: %s", path)
	}

	file, err := os.OpenFile(path, os.O_RDWR|os.O_TRUNC, 0644)
	if err != nil {
		return fmt.Errorf("failed to update TODO: %w", err)
	}
	defer file.Close()

	data := Marshal(td)
	_, err = file.Write(data)
	if err != nil {
		return fmt.Errorf("failed to update TODO: %w", err)
	}

	return nil
}

// Delete implements Repository interface.
func (repo *FileSystem) Delete(id int) error {
	filename := fmt.Sprintf("%d.md", id)
	path := filepath.Join(repo.root, filename)

	if _, err := os.Stat(path); os.IsNotExist(err) {
		return fmt.Errorf("file not found: %s", path)
	}

	err := os.Remove(path)
	if err != nil {
		return fmt.Errorf("failed to delete TODO: %w", err)
	}

	index, err := repo.readIndex()
	if err != nil {
		return fmt.Errorf("failed to delete TODO from index: %w", err)
	}

	for k, todos := range index.Todos {
		if k != "" {
			continue
		}

		var ids []int
		for _, _id := range todos {
			if _id == id {
				continue
			}
			ids = append(ids, _id)
		}
		index.Todos[k] = ids
	}

	err = repo.writeIndex(index)
	if err != nil {
		return fmt.Errorf("failed to delete TODO from index: %w", err)
	}

	return nil
}

// Open implements Repository interface.
func (repo *FileSystem) Open(id int) error {
	filename := fmt.Sprintf("%d.md", id)
	path := filepath.Join(repo.root, filename)

	cmd := exec.Command("open", path)
	err := cmd.Run()
	if err != nil {
		return fmt.Errorf("failed to open %s: %w", path, err)
	}

	return nil
}

// Move implements Repository interface.
func (repo *FileSystem) Move(id, position int) error {
	if position < 1 {
		return fmt.Errorf("position number must be larger than 0: %d", position)
	}

	to := position - 1

	st, err := repo.readIndex()
	if err != nil {
		return fmt.Errorf("failed to move TODO: %w", err)
	}

	ids, _ := st.Todos[""]
	if to >= len(ids) {
		return fmt.Errorf("position number is too large: %d", position)
	}

	var from int
	for j, _id := range ids {
		if _id == id {
			from = j
		}
	}

	st.Todos[""] = swapped(ids, from, to)

	err = repo.writeIndex(st)
	if err != nil {
		return fmt.Errorf("failed to move TODO: %w", err)
	}

	return nil
}

func parseID(path string) (int, error) {
	text := strings.TrimRight(filepath.Base(path), filepath.Ext(path))
	return strconv.Atoi(text)
}

func newContent(title string) string {
	return strings.Trim(fmt.Sprintf(`
---
title: %s
state: undone
---
`, title), "\n")
}

func (repo *FileSystem) readIndex() (*index, error) {
	path := filepath.Join(repo.root, "index.json")
	if _, err := os.Stat(path); os.IsNotExist(err) {
		return &index{
			Todos: map[string][]int{
				"": {},
			},
		}, nil
	}

	file, err := os.Open(path)
	if err != nil {
		return nil, fmt.Errorf("failed to read index from %s: %w", path, err)
	}
	defer file.Close()

	content, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, fmt.Errorf("failed to read index from %s: %w", path, err)
	}

	var i index
	err = json.Unmarshal(content, &i)
	if err != nil {
		return nil, fmt.Errorf("failed to read index from %s: %w", path, err)
	}

	return &i, nil
}

func (repo *FileSystem) writeIndex(i *index) error {
	path := filepath.Join(repo.root, "index.json")

	data, err := json.MarshalIndent(*i, "", "  ")
	if err != nil {
		return fmt.Errorf("failed to write index to %s: %w", path, err)
	}

	err = ioutil.WriteFile(path, data, 0644)
	if err != nil {
		return fmt.Errorf("failed to write index to %s: %w", path, err)
	}

	return nil
}

func swapped(s []int, from, to int) []int {
	if from < 0 || from >= len(s) {
		return s
	}

	if to < 0 || to >= len(s) {
		return s
	}

	copied := make([]int, len(s), cap(s))
	copy(copied, s)

	e := copied[from]
	copied = append(copied[:from], copied[(from+1):]...)
	copied = append(copied[:to], append([]int{e}, copied[to:]...)...)
	return copied
}
<commit_msg>Add the ID of sub-TODOs to index.json<commit_after>package filesystem

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/naoty/todo/todo"
)

// FileSystem represents a repository backed by file system.
type FileSystem struct {
	root string
}

type index struct {
	Todos map[string][]int `json:"todos"`
}

// New returns a new FileSystem.
func New() (*FileSystem, error) {
	home := os.Getenv("TODOS_PATH")
	if home == "" {
		var err error
		home, err = os.UserHomeDir()
		if err != nil {
			return nil, err
		}
	}

	root := filepath.Join(home, ".todos")
	if _, err := os.Stat(root); os.IsNotExist(err) {
		err := os.Mkdir(root, 0755)
		if err != nil {
			return nil, err
		}
	}

	return &FileSystem{root: root}, nil
}

// Get implements Repository interface.
func (repo *FileSystem) Get(id int) (*todo.Todo, error) {
	filename := fmt.Sprintf("%d.md", id)
	path := filepath.Join(repo.root, filename)

	if _, err := os.Stat(path); os.IsNotExist(err) {
		return nil, fmt.Errorf("TODO not found: %d", id)
	}

	file, err := os.Open(path)
	if err != nil {
		return nil, fmt.Errorf("failed to get TODO: %w", err)
	}
	defer file.Close()

	content, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, fmt.Errorf("failed to get TODO: %w", err)
	}

	td, err := Parse(string(content))
	if err != nil {
		return nil, fmt.Errorf("failed to get TODO: %w", err)
	}

	td.ID = id
	return td, nil
}

// List implements Repository interface.
func (repo *FileSystem) List() ([]*todo.Todo, error) {
	todos := map[int]*todo.Todo{}

	err := filepath.Walk(repo.root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if !strings.HasSuffix(path, ".md") {
			return nil
		}

		id, err := parseID(path)
		if err != nil {
			return err
		}

		file, err := os.Open(path)
		if err != nil {
			return err
		}
		defer file.Close()

		content, err := ioutil.ReadAll(file)
		if err != nil {
			return err
		}

		td, err := Parse(string(content))
		if err != nil {
			return err
		}

		td.ID = id
		todos[id] = td

		return nil
	})

	if err != nil {
		return nil, fmt.Errorf("failed to get todos from %s: %w", repo.root, err)
	}

	st, err := repo.readIndex()
	if err != nil {
		return nil, fmt.Errorf("failed to get todos from %s: %w", repo.root, err)
	}

	var ordered []*todo.Todo
	for k, ids := range st.Todos {
		// NOTE: when TODOs have subTODOs, k is parent TODO's ID.
		if k != "" {
			continue
		}

		for _, id := range ids {
			td, ok := todos[id]
			if !ok {
				continue
			}

			ordered = append(ordered, td)
		}
	}

	return ordered, nil
}

// Add implements Repository interface.
func (repo *FileSystem) Add(title string, parent *int) error {
	todos, err := repo.List()
	if err != nil {
		return fmt.Errorf("failed to get next id: %w", err)
	}

	lastID := 0
	for _, td := range todos {
		id := td.ID
		if id > lastID {
			lastID = id
		}
	}

	nextID := lastID + 1
	filename := fmt.Sprintf("%d.md", nextID)
	path := filepath.Join(repo.root, filename)

	if _, err := os.Stat(path); err == nil {
		return fmt.Errorf("already exist: %s", path)
	}

	file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644)
	if err != nil {
		return fmt.Errorf("failed to add TODO: %w", err)
	}
	defer file.Close()

	content := newContent(title)
	_, err = file.WriteString(content)
	if err != nil {
		return fmt.Errorf("failed to add TODO: %w", err)
	}

	st, err := repo.readIndex()
	if err != nil {
		return fmt.Errorf("failed to add TODO: %w", err)
	}

	parentID := ""
	if *parent > 0 {
		parentID = fmt.Sprintf("%d", *parent)
	}

	st.Todos[parentID] = append(st.Todos[parentID], nextID)
	err = repo.writeIndex(st)
	if err != nil {
		return fmt.Errorf("failed to add TODO: %w", err)
	}

	return nil
}

// Update implements Repository interface.
func (repo *FileSystem) Update(td *todo.Todo) error {
	filename := fmt.Sprintf("%d.md", td.ID)
	path := filepath.Join(repo.root, filename)

	if _, err := os.Stat(path); os.IsNotExist(err) {
		return fmt.Errorf("file not found: %s", path)
	}

	file, err := os.OpenFile(path, os.O_RDWR|os.O_TRUNC, 0644)
	if err != nil {
		return fmt.Errorf("failed to update TODO: %w", err)
	}
	defer file.Close()

	data := Marshal(td)
	_, err = file.Write(data)
	if err != nil {
		return fmt.Errorf("failed to update TODO: %w", err)
	}

	return nil
}

// Delete implements Repository interface.
func (repo *FileSystem) Delete(id int) error {
	filename := fmt.Sprintf("%d.md", id)
	path := filepath.Join(repo.root, filename)

	if _, err := os.Stat(path); os.IsNotExist(err) {
		return fmt.Errorf("file not found: %s", path)
	}

	err := os.Remove(path)
	if err != nil {
		return fmt.Errorf("failed to delete TODO: %w", err)
	}

	index, err := repo.readIndex()
	if err != nil {
		return fmt.Errorf("failed to delete TODO from index: %w", err)
	}

	for k, todos := range index.Todos {
		if k != "" {
			continue
		}

		var ids []int
		for _, _id := range todos {
			if _id == id {
				continue
			}
			ids = append(ids, _id)
		}
		index.Todos[k] = ids
	}

	err = repo.writeIndex(index)
	if err != nil {
		return fmt.Errorf("failed to delete TODO from index: %w", err)
	}

	return nil
}

// Open implements Repository interface.
func (repo *FileSystem) Open(id int) error {
	filename := fmt.Sprintf("%d.md", id)
	path := filepath.Join(repo.root, filename)

	cmd := exec.Command("open", path)
	err := cmd.Run()
	if err != nil {
		return fmt.Errorf("failed to open %s: %w", path, err)
	}

	return nil
}

// Move implements Repository interface.
func (repo *FileSystem) Move(id, position int) error {
	if position < 1 {
		return fmt.Errorf("position number must be larger than 0: %d", position)
	}

	to := position - 1

	st, err := repo.readIndex()
	if err != nil {
		return fmt.Errorf("failed to move TODO: %w", err)
	}

	ids, _ := st.Todos[""]
	if to >= len(ids) {
		return fmt.Errorf("position number is too large: %d", position)
	}

	var from int
	for j, _id := range ids {
		if _id == id {
			from = j
		}
	}

	st.Todos[""] = swapped(ids, from, to)

	err = repo.writeIndex(st)
	if err != nil {
		return fmt.Errorf("failed to move TODO: %w", err)
	}

	return nil
}

func parseID(path string) (int, error) {
	text := strings.TrimRight(filepath.Base(path), filepath.Ext(path))
	return strconv.Atoi(text)
}

func newContent(title string) string {
	return strings.Trim(fmt.Sprintf(`
---
title: %s
state: undone
---
`, title), "\n")
}

func (repo *FileSystem) readIndex() (*index, error) {
	path := filepath.Join(repo.root, "index.json")
	if _, err := os.Stat(path); os.IsNotExist(err) {
		return &index{
			Todos: map[string][]int{
				"": {},
			},
		}, nil
	}

	file, err := os.Open(path)
	if err != nil {
		return nil, fmt.Errorf("failed to read index from %s: %w", path, err)
	}
	defer file.Close()

	content, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, fmt.Errorf("failed to read index from %s: %w", path, err)
	}

	var i index
	err = json.Unmarshal(content, &i)
	if err != nil {
		return nil, fmt.Errorf("failed to read index from %s: %w", path, err)
	}

	return &i, nil
}

func (repo *FileSystem) writeIndex(i *index) error {
	path := filepath.Join(repo.root, "index.json")

	data, err := json.MarshalIndent(*i, "", "  ")
	if err != nil {
		return fmt.Errorf("failed to write index to %s: %w", path, err)
	}

	err = ioutil.WriteFile(path, data, 0644)
	if err != nil {
		return fmt.Errorf("failed to write index to %s: %w", path, err)
	}

	return nil
}

func swapped(s []int, from, to int) []int {
	if from < 0 || from >= len(s) {
		return s
	}

	if to < 0 || to >= len(s) {
		return s
	}

	copied := make([]int, len(s), cap(s))
	copy(copied, s)

	e := copied[from]
	copied = append(copied[:from], copied[(from+1):]...)
	copied = append(copied[:to], append([]int{e}, copied[to:]...)...)
	return copied
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/raintank/raintank-metric/schema"
	"io/ioutil"
	"log"
	"net/http"
)

type Hit struct {
	Index  string                  `json:"_index"`
	Type   string                  `json:"_type"`
	Id     string                  `json:"_id"`
	Score  float64                 `json:"_score"`
	Source schema.MetricDefinition `json:"_source"`
}

type EsResult struct {
	Took     int
	TimedOut bool
	_shards  struct {
		total      int
		successful int
		failed     int
	}
	Hits struct {
		Total    int
		MaxScore int
		Hits     []Hit
	}
}

func perror(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func main() {
	res, err := http.Get("http://localhost:9200/metric/_search?q=*:*&size=10000000")
	perror(err)
	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	perror(err)
	var data EsResult
	err = json.Unmarshal(body, &data)
	perror(err)
	for _, h := range data.Hits.Hits {
		fmt.Println(h.Source.Name)
	}
}
<commit_msg>new version doesn't like >10k requests anymore, use metricdef scrolling<commit_after>package main

import (
	"fmt"
	"github.com/raintank/raintank-metric/metricdef"
	"github.com/raintank/raintank-metric/schema"
	"log"
)

type Hit struct {
	Index  string                  `json:"_index"`
	Type   string                  `json:"_type"`
	Id     string                  `json:"_id"`
	Score  float64                 `json:"_score"`
	Source schema.MetricDefinition `json:"_source"`
}

type EsResult struct {
	Took     int
	TimedOut bool
	_shards  struct {
		total      int
		successful int
		failed     int
	}
	Hits struct {
		Total    int
		MaxScore int
		Hits     []Hit
	}
}

func perror(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func main() {
	defs, err := metricdef.NewDefsEs("localhost:9200", "", "", "metric")
	show := func(ds []*schema.MetricDefinition) {
		for _, d := range ds {
			fmt.Println(d.OrgId, d.Name)
		}
	}
	met, scroll_id, err := defs.GetMetrics("")
	perror(err)
	show(met)
	for scroll_id != "" {
		met, scroll_id, err = defs.GetMetrics(scroll_id)
		perror(err)
		show(met)
	}
}
<|endoftext|>


<commit_before>//
// media.go
// Copyright 2017 Konstantin Dovnar
//
// Licensed under the Apache License, Version 2.0 (the "License");
// http://www.apache.org/licenses/LICENSE-2.0
//

package instagram

import (
	"strconv"
	"encoding/json"
)

// TypeImage is a string that define image type for media.
const TypeImage = "image"

// TypeVideo is a string that define video type for media.
const TypeVideo = "video"

// TypeCarousel is a string that define carousel (collection of media) type for media.
const TypeCarousel = "carousel"

const (
	graphVideo   = "GraphVideo"
	graphSidebar = "GraphSidecar"

	video    = "video"
	carousel = "carousel"
)

// A Media describes an Instagram media info.
type Media struct {
	Caption       string
	Code          string
	CommentsCount uint32
	Date          uint64
	ID            string
	AD            bool
	LikesCount    uint32
	Type          string
	MediaURL      string
	Owner         Account
	MediaList     []mediaItem
}

type mediaItem struct {
	Type string
	URL  string
	Code string
}

// Update try to update media data
func (m *Media) Update() error {
	media, err := GetMediaByCode(m.Code)
	if err != nil {
		return err
	}
	*m = media
	return nil
}

func getFromMediaPage(data []byte) (Media, error) {
	var mediaJSON struct {
		Graphql struct {
			ShortcodeMedia struct {
				Typename   string `json:"__typename"`
				ID         string `json:"id"`
				Shortcode  string `json:"shortcode"`
				DisplayURL string `json:"display_url"`
				VideoURL   string `json:"video_url"`
				IsVideo    bool `json:"is_video"`
				EdgeMediaToCaption struct {
					Edges []struct {
						Node struct {
							Text string `json:"text"`
						} `json:"node"`
					} `json:"edges"`
				} `json:"edge_media_to_caption"`
				EdgeMediaToComment struct {
					Count int `json:"count"`
				} `json:"edge_media_to_comment"`
				TakenAtTimestamp int `json:"taken_at_timestamp"`
				EdgeMediaPreviewLike struct {
					Count int `json:"count"`
				} `json:"edge_media_preview_like"`
				Owner struct {
					ID            string `json:"id"`
					ProfilePicURL string `json:"profile_pic_url"`
					Username      string `json:"username"`
					FullName      string `json:"full_name"`
					IsPrivate     bool `json:"is_private"`
				} `json:"owner"`
				IsAd bool `json:"is_ad"`
				EdgeSidecarToChildren struct {
					Edges []struct {
						Node struct {
							Typename   string `json:"__typename"`
							ID         string `json:"id"`
							Shortcode  string `json:"shortcode"`
							DisplayURL string `json:"display_url"`
							VideoURL   string `json:"video_url"`
							IsVideo    bool `json:"is_video"`
						} `json:"node"`
					} `json:"edges"`
				} `json:"edge_sidecar_to_children"`
			} `json:"shortcode_media"`
		} `json:"graphql"`
	}

	err := json.Unmarshal(data, &mediaJSON)
	if err != nil {
		return Media{}, err
	}

	media := Media{}
	media.Code = mediaJSON.Graphql.ShortcodeMedia.Shortcode
	media.ID = mediaJSON.Graphql.ShortcodeMedia.ID
	media.AD = mediaJSON.Graphql.ShortcodeMedia.IsAd
	media.Date = uint64(mediaJSON.Graphql.ShortcodeMedia.TakenAtTimestamp)
	media.CommentsCount = uint32(mediaJSON.Graphql.ShortcodeMedia.EdgeMediaToComment.Count)
	media.LikesCount = uint32(mediaJSON.Graphql.ShortcodeMedia.EdgeMediaPreviewLike.Count)
	media.Caption = mediaJSON.Graphql.ShortcodeMedia.EdgeMediaToCaption.Edges[0].Node.Text

	var mediaType = mediaJSON.Graphql.ShortcodeMedia.Typename
	if mediaType == graphSidebar {
		for _, itemJSON := range mediaJSON.Graphql.ShortcodeMedia.EdgeSidecarToChildren.Edges {
			var item mediaItem
			item.Code = itemJSON.Node.Shortcode
			if itemJSON.Node.IsVideo {
				item.URL = itemJSON.Node.VideoURL
				item.Type = TypeVideo
			} else {
				item.URL = itemJSON.Node.DisplayURL
				item.Type = TypeImage
			}
			media.MediaList = append(media.MediaList, item)
		}
		media.Type = TypeCarousel
	} else {
		if mediaType == graphVideo {
			media.Type = TypeVideo
			media.MediaURL = mediaJSON.Graphql.ShortcodeMedia.VideoURL
		} else {
			media.Type = TypeImage
			media.MediaURL = mediaJSON.Graphql.ShortcodeMedia.DisplayURL
		}
		var item mediaItem
		item.Code = media.Code
		item.Type = media.Type
		item.URL = media.MediaURL
		media.MediaList = append(media.MediaList, item)
	}

	media.Owner.ID = mediaJSON.Graphql.ShortcodeMedia.Owner.ID
	media.Owner.ProfilePicURL = mediaJSON.Graphql.ShortcodeMedia.Owner.ProfilePicURL
	media.Owner.Username = mediaJSON.Graphql.ShortcodeMedia.Owner.Username
	media.Owner.FullName = mediaJSON.Graphql.ShortcodeMedia.Owner.FullName
	media.Owner.Private = mediaJSON.Graphql.ShortcodeMedia.Owner.IsPrivate

	return media, nil
}

func getFromAccountMediaList(data []byte) (Media, error) {
	var mediaJSON struct {
		ID   string `json:"id"`
		Code string `json:"code"`
		User struct {
			ID             string `json:"id"`
			FullName       string `json:"full_name"`
			ProfilePicture string `json:"profile_picture"`
			Username       string `json:"username"`
		} `json:"user"`
		Images struct {
			StandardResolution struct {
				Width  int `json:"width"`
				Height int `json:"height"`
				URL    string `json:"url"`
			} `json:"standard_resolution"`
		} `json:"images"`
		CreatedTime string `json:"created_time"`
		Caption struct {
			Text string `json:"text"`
		} `json:"caption"`
		Likes struct {
			Count float64 `json:"count"`
		} `json:"likes"`
		Comments struct {
			Count float64 `json:"count"`
		} `json:"comments"`
		Type string `json:"type"`
		Videos struct {
			StandardResolution struct {
				Width  int `json:"width"`
				Height int `json:"height"`
				URL    string `json:"url"`
			} `json:"standard_resolution"`
		} `json:"videos"`
		CarouselMedia []struct {
			Images struct {
				StandardResolution struct {
					URL string `json:"url"`
				} `json:"standard_resolution"`
			} `json:"images"`
			Videos struct {
				StandardResolution struct {
					URL string `json:"url"`
				} `json:"standard_resolution"`
			} `json:"videos"`
			UsersInPhoto []interface{} `json:"users_in_photo"`
			Type         string `json:"type"`
		} `json:"carousel_media"`
	}

	err := json.Unmarshal(data, &mediaJSON)
	if err != nil {
		return Media{}, err
	}

	media := Media{}
	media.Code = mediaJSON.Code
	media.ID = mediaJSON.ID
	media.Caption = mediaJSON.Caption.Text
	media.LikesCount = uint32(mediaJSON.Likes.Count)
	media.CommentsCount = uint32(mediaJSON.Comments.Count)

	date, err := strconv.ParseUint(mediaJSON.CreatedTime, 10, 64)
	if err == nil {
		media.Date = date
	}

	if mediaJSON.Type == carousel {
		media.Type = TypeCarousel
		for _, itemJSOM := range mediaJSON.CarouselMedia {
			var item mediaItem
			item.Type = itemJSOM.Type
			if item.Type == video {
				item.URL = itemJSOM.Videos.StandardResolution.URL
			} else {
				item.URL = itemJSOM.Images.StandardResolution.URL
			}
			media.MediaList = append(media.MediaList, item)
		}
	} else {
		if mediaJSON.Type == video {
			media.MediaURL = mediaJSON.Videos.StandardResolution.URL
			media.Type = TypeVideo
		} else {
			media.MediaURL = mediaJSON.Images.StandardResolution.URL
			media.Type = TypeImage
		}
		var item mediaItem
		item.Type = media.Type
		item.URL = media.MediaURL
		item.Code = media.Code
		media.MediaList = append(media.MediaList, item)
	}

	media.Owner.Username = mediaJSON.User.Username
	media.Owner.FullName = mediaJSON.User.FullName
	media.Owner.ID = mediaJSON.User.ID
	media.Owner.ProfilePicURL = mediaJSON.User.ProfilePicture

	return media, nil
}

func getFromSearchMediaList(data []byte) (Media, error) {
	var mediaJSON struct {
		CommentsDisabled bool `json:"comments_disabled"`
		ID               string `json:"id"`
		Owner struct {
			ID string `json:"id"`
		} `json:"owner"`
		ThumbnailSrc string `json:"thumbnail_src"`
		IsVideo      bool `json:"is_video"`
		Code         string `json:"code"`
		Date         float64 `json:"date"`
		DisplaySrc   string `json:"display_src"`
		Caption      string `json:"caption"`
		Comments struct {
			Count float64 `json:"count"`
		} `json:"comments"`
		Likes struct {
			Count float64 `json:"count"`
		} `json:"likes"`
	}

	err := json.Unmarshal(data, &mediaJSON)
	if err != nil {
		return Media{}, err
	}

	media := Media{}
	media.ID = mediaJSON.ID
	media.Code = mediaJSON.Code
	media.MediaURL = mediaJSON.DisplaySrc
	media.Caption = mediaJSON.Caption
	media.Date = uint64(mediaJSON.Date)
	media.LikesCount = uint32(mediaJSON.Likes.Count)
	media.CommentsCount = uint32(mediaJSON.Comments.Count)
	media.Owner.ID = mediaJSON.Owner.ID

	if mediaJSON.IsVideo {
		media.Type = TypeVideo
	} else {
		media.Type = TypeImage
	}

	return media, nil
}
<commit_msg>Fix variable name<commit_after>//
// media.go
// Copyright 2017 Konstantin Dovnar
//
// Licensed under the Apache License, Version 2.0 (the "License");
// http://www.apache.org/licenses/LICENSE-2.0
//

package instagram

import (
	"strconv"
	"encoding/json"
)

// TypeImage is a string that define image type for media.
const TypeImage = "image"

// TypeVideo is a string that define video type for media.
const TypeVideo = "video"

// TypeCarousel is a string that define carousel (collection of media) type for media.
const TypeCarousel = "carousel"

const (
	graphVideo   = "GraphVideo"
	graphSidecar = "GraphSidecar"

	video    = "video"
	carousel = "carousel"
)

// A Media describes an Instagram media info.
type Media struct {
	Caption       string
	Code          string
	CommentsCount uint32
	Date          uint64
	ID            string
	AD            bool
	LikesCount    uint32
	Type          string
	MediaURL      string
	Owner         Account
	MediaList     []mediaItem
}

type mediaItem struct {
	Type string
	URL  string
	Code string
}

// Update try to update media data
func (m *Media) Update() error {
	media, err := GetMediaByCode(m.Code)
	if err != nil {
		return err
	}
	*m = media
	return nil
}

func getFromMediaPage(data []byte) (Media, error) {
	var mediaJSON struct {
		Graphql struct {
			ShortcodeMedia struct {
				Typename   string `json:"__typename"`
				ID         string `json:"id"`
				Shortcode  string `json:"shortcode"`
				DisplayURL string `json:"display_url"`
				VideoURL   string `json:"video_url"`
				IsVideo    bool `json:"is_video"`
				EdgeMediaToCaption struct {
					Edges []struct {
						Node struct {
							Text string `json:"text"`
						} `json:"node"`
					} `json:"edges"`
				} `json:"edge_media_to_caption"`
				EdgeMediaToComment struct {
					Count int `json:"count"`
				} `json:"edge_media_to_comment"`
				TakenAtTimestamp int `json:"taken_at_timestamp"`
				EdgeMediaPreviewLike struct {
					Count int `json:"count"`
				} `json:"edge_media_preview_like"`
				Owner struct {
					ID            string `json:"id"`
					ProfilePicURL string `json:"profile_pic_url"`
					Username      string `json:"username"`
					FullName      string `json:"full_name"`
					IsPrivate     bool `json:"is_private"`
				} `json:"owner"`
				IsAd bool `json:"is_ad"`
				EdgeSidecarToChildren struct {
					Edges []struct {
						Node struct {
							Typename   string `json:"__typename"`
							ID         string `json:"id"`
							Shortcode  string `json:"shortcode"`
							DisplayURL string `json:"display_url"`
							VideoURL   string `json:"video_url"`
							IsVideo    bool `json:"is_video"`
						} `json:"node"`
					} `json:"edges"`
				} `json:"edge_sidecar_to_children"`
			} `json:"shortcode_media"`
		} `json:"graphql"`
	}

	err := json.Unmarshal(data, &mediaJSON)
	if err != nil {
		return Media{}, err
	}

	media := Media{}
	media.Code = mediaJSON.Graphql.ShortcodeMedia.Shortcode
	media.ID = mediaJSON.Graphql.ShortcodeMedia.ID
	media.AD = mediaJSON.Graphql.ShortcodeMedia.IsAd
	media.Date = uint64(mediaJSON.Graphql.ShortcodeMedia.TakenAtTimestamp)
	media.CommentsCount = uint32(mediaJSON.Graphql.ShortcodeMedia.EdgeMediaToComment.Count)
	media.LikesCount = uint32(mediaJSON.Graphql.ShortcodeMedia.EdgeMediaPreviewLike.Count)
	media.Caption = mediaJSON.Graphql.ShortcodeMedia.EdgeMediaToCaption.Edges[0].Node.Text

	var mediaType = mediaJSON.Graphql.ShortcodeMedia.Typename
	if mediaType == graphSidecar {
		for _, itemJSON := range mediaJSON.Graphql.ShortcodeMedia.EdgeSidecarToChildren.Edges {
			var item mediaItem
			item.Code = itemJSON.Node.Shortcode
			if itemJSON.Node.IsVideo {
				item.URL = itemJSON.Node.VideoURL
				item.Type = TypeVideo
			} else {
				item.URL = itemJSON.Node.DisplayURL
				item.Type = TypeImage
			}
			media.MediaList = append(media.MediaList, item)
		}
		media.Type = TypeCarousel
	} else {
		if mediaType == graphVideo {
			media.Type = TypeVideo
			media.MediaURL = mediaJSON.Graphql.ShortcodeMedia.VideoURL
		} else {
			media.Type = TypeImage
			media.MediaURL = mediaJSON.Graphql.ShortcodeMedia.DisplayURL
		}
		var item mediaItem
		item.Code = media.Code
		item.Type = media.Type
		item.URL = media.MediaURL
		media.MediaList = append(media.MediaList, item)
	}

	media.Owner.ID = mediaJSON.Graphql.ShortcodeMedia.Owner.ID
	media.Owner.ProfilePicURL = mediaJSON.Graphql.ShortcodeMedia.Owner.ProfilePicURL
	media.Owner.Username = mediaJSON.Graphql.ShortcodeMedia.Owner.Username
	media.Owner.FullName = mediaJSON.Graphql.ShortcodeMedia.Owner.FullName
	media.Owner.Private = mediaJSON.Graphql.ShortcodeMedia.Owner.IsPrivate

	return media, nil
}

func getFromAccountMediaList(data []byte) (Media, error) {
	var mediaJSON struct {
		ID   string `json:"id"`
		Code string `json:"code"`
		User struct {
			ID             string `json:"id"`
			FullName       string `json:"full_name"`
			ProfilePicture string `json:"profile_picture"`
			Username       string `json:"username"`
		} `json:"user"`
		Images struct {
			StandardResolution struct {
				Width  int `json:"width"`
				Height int `json:"height"`
				URL    string `json:"url"`
			} `json:"standard_resolution"`
		} `json:"images"`
		CreatedTime string `json:"created_time"`
		Caption struct {
			Text string `json:"text"`
		} `json:"caption"`
		Likes struct {
			Count float64 `json:"count"`
		} `json:"likes"`
		Comments struct {
			Count float64 `json:"count"`
		} `json:"comments"`
		Type string `json:"type"`
		Videos struct {
			StandardResolution struct {
				Width  int `json:"width"`
				Height int `json:"height"`
				URL    string `json:"url"`
			} `json:"standard_resolution"`
		} `json:"videos"`
		CarouselMedia []struct {
			Images struct {
				StandardResolution struct {
					URL string `json:"url"`
				} `json:"standard_resolution"`
			} `json:"images"`
			Videos struct {
				StandardResolution struct {
					URL string `json:"url"`
				} `json:"standard_resolution"`
			} `json:"videos"`
			UsersInPhoto []interface{} `json:"users_in_photo"`
			Type         string `json:"type"`
		} `json:"carousel_media"`
	}

	err := json.Unmarshal(data, &mediaJSON)
	if err != nil {
		return Media{}, err
	}

	media := Media{}
	media.Code = mediaJSON.Code
	media.ID = mediaJSON.ID
	media.Caption = mediaJSON.Caption.Text
	media.LikesCount = uint32(mediaJSON.Likes.Count)
	media.CommentsCount = uint32(mediaJSON.Comments.Count)

	date, err := strconv.ParseUint(mediaJSON.CreatedTime, 10, 64)
	if err == nil {
		media.Date = date
	}

	if mediaJSON.Type == carousel {
		media.Type = TypeCarousel
		for _, itemJSOM := range mediaJSON.CarouselMedia {
			var item mediaItem
			item.Type = itemJSOM.Type
			if item.Type == video {
				item.URL = itemJSOM.Videos.StandardResolution.URL
			} else {
				item.URL = itemJSOM.Images.StandardResolution.URL
			}
			media.MediaList = append(media.MediaList, item)
		}
	} else {
		if mediaJSON.Type == video {
			media.MediaURL = mediaJSON.Videos.StandardResolution.URL
			media.Type = TypeVideo
		} else {
			media.MediaURL = mediaJSON.Images.StandardResolution.URL
			media.Type = TypeImage
		}
		var item mediaItem
		item.Type = media.Type
		item.URL = media.MediaURL
		item.Code = media.Code
		media.MediaList = append(media.MediaList, item)
	}

	media.Owner.Username = mediaJSON.User.Username
	media.Owner.FullName = mediaJSON.User.FullName
	media.Owner.ID = mediaJSON.User.ID
	media.Owner.ProfilePicURL = mediaJSON.User.ProfilePicture

	return media, nil
}

func getFromSearchMediaList(data []byte) (Media, error) {
	var mediaJSON struct {
		CommentsDisabled bool `json:"comments_disabled"`
		ID               string `json:"id"`
		Owner struct {
			ID string `json:"id"`
		} `json:"owner"`
		ThumbnailSrc string `json:"thumbnail_src"`
		IsVideo      bool `json:"is_video"`
		Code         string `json:"code"`
		Date         float64 `json:"date"`
		DisplaySrc   string `json:"display_src"`
		Caption      string `json:"caption"`
		Comments struct {
			Count float64 `json:"count"`
		} `json:"comments"`
		Likes struct {
			Count float64 `json:"count"`
		} `json:"likes"`
	}

	err := json.Unmarshal(data, &mediaJSON)
	if err != nil {
		return Media{}, err
	}

	media := Media{}
	media.ID = mediaJSON.ID
	media.Code = mediaJSON.Code
	media.MediaURL = mediaJSON.DisplaySrc
	media.Caption = mediaJSON.Caption
	media.Date = uint64(mediaJSON.Date)
	media.LikesCount = uint32(mediaJSON.Likes.Count)
	media.CommentsCount = uint32(mediaJSON.Comments.Count)
	media.Owner.ID = mediaJSON.Owner.ID

	if mediaJSON.IsVideo {
		media.Type = TypeVideo
	} else {
		media.Type = TypeImage
	}

	return media, nil
}
<|endoftext|>