<commit_before>// Writing files in Go follows similar patterns to the
// ones we saw earlier for reading.

package main

import (
  "io/ioutil"
  "bufio"
  "os"
  "log"
  "fmt"
  "encoding/xml"
  "net/http"
	//"github.com/gorilla/mux"
  "encoding/json"
  "html/template"
  "strings"
  "net/url"
  "path/filepath"
  "time"
)

type Transaction struct {
  TRNTYPE   string
  DTPOSTED  string
  TRNAMT    float32
  FITID     string
  NAME      string
}

type BankAccount struct {
  BANKID   string
  ACCTID   string
  ACCTTYPE string
}

type OFX struct {
  XMLName     xml.Name      `xml: "OFX"`
  BankAccount BankAccount   `xml:"BANKMSGSRSV1>STMTRS>BANKACCTFROM"`
  Transaction []Transaction `xml:"BANKMSGSRSV1>STMTRS>BANKTRANLIST>STMTTRN"`
}

type settings struct {
  ClientId string
  ClientSecret string
}

type accessToken struct {
  Access_token string
  Client_id string
  Expires_in string
  Refresh_token string
  Token_type string
  User_id string
}

type account struct {
  Id string
  Created string
  Description string
}

type accounts struct {
  Accounts []account
}

type transaction struct {
	Account_balance int
	Amount         int
	Attachments    []interface{}
	Category       string
	Created        string
	Currency       string
	Description    string
	ID             string
	Is_load         bool
	Merchant       string
	Metadata       map[string]interface{}
	Notes          string
	Settled        string
}

type transactions struct {
  Transactions []transaction
}

var s settings

func check(e error) {
    if e != nil {
        log.Fatal(e)
    }
}

func getsettings() {
  // Client ID
  reader := bufio.NewReader(os.Stdin)
  fmt.Print("Enter clientId: ")
  s.ClientId, _ = reader.ReadString('\n')
  s.ClientId = strings.TrimSpace(s.ClientId)

  // Client Secret
  reader = bufio.NewReader(os.Stdin)
  fmt.Print("Enter clientSecret: ")
  s.ClientSecret, _ = reader.ReadString('\n')
  s.ClientSecret = strings.TrimSpace(s.ClientSecret)

}

func indexHandler(w http.ResponseWriter, r *http.Request) {
  t, _ := template.ParseFiles("index.html")
  t.Execute(w, &s)
}

func getAuthCode(code string) accessToken  {
  resp, err := http.PostForm("https://api.getmondo.co.uk/oauth2/token",
	   url.Values{
        "grant_type": {"authorization_code"},
        "client_id": {s.ClientId},
        "client_secret": {s.ClientSecret},
        "redirect_uri": {"http://localhost:8080/getTransactions/"},
        "code": {code},
    },
  )
  check(err)
  defer resp.Body.Close()
  body, err := ioutil.ReadAll(resp.Body)
  check(err)
  var result accessToken
  json.Unmarshal([]byte(body), &result)
  return result
}

func getAccounts(authStruct accessToken) accounts {
  // Prepare HTTP request
  client := &http.Client{}
  req, err := http.NewRequest("GET", "https://api.getmondo.co.uk/accounts", nil)
  check(err)
  req.Header.Add("authorization", `Bearer ` + authStruct.Access_token)
  q := req.URL.Query()
  q.Add("account_id", authStruct.User_id)
  req.URL.RawQuery = q.Encode()
  resp, err := client.Do(req)
  defer resp.Body.Close()
  body, err := ioutil.ReadAll(resp.Body)

  // JSON decode result
  var result accounts
  json.Unmarshal([]byte(body), &result)

  return result
}

func getTransactions(authStruct accessToken, acccountStruct account) transactions{
  // Fetch transactions
  client := &http.Client{}
  req, err := http.NewRequest("GET", "https://api.getmondo.co.uk/transactions", nil)
  check(err)
  req.Header.Add("authorization", `Bearer ` + authStruct.Access_token)
  q := req.URL.Query()
  q.Add("account_id", acccountStruct.Id)
  req.URL.RawQuery = q.Encode()
  resp, err := client.Do(req)
  defer resp.Body.Close()
  body, err := ioutil.ReadAll(resp.Body)

  // JSON decode result
  var result transactions
  json.Unmarshal([]byte(body), &result)

  return result
}

func getTransactionsHandler(w http.ResponseWriter, r *http.Request){

  // Authenticate
  code := r.FormValue("code")
  authStruct := getAuthCode(code)

  accounts := getAccounts(authStruct)
  account := accounts.Accounts[0]

  transactions := getTransactions(authStruct, account)

  // Create an OFX
  OFXStruct := &OFX{}
  OFXStruct.BankAccount = BankAccount{
    BANKID: "0",
    ACCTID: account.Id,
    ACCTTYPE: "CHECKING",
  }

  // Loop through the transactions adding them to the OFX struct
  for _, v := range transactions.Transactions {
    log.Print(string(v.Description))

    time, _ := time.Parse(time.RFC3339, v.Created)
    formattedTime := time.Format("20060102150405.000[-07]")
    var amount = float32(v.Amount)/float32(100)
    OFXStruct.Transaction = append(OFXStruct.Transaction,Transaction{
      TRNTYPE: "POS",
      DTPOSTED: formattedTime,
      TRNAMT: amount,
      FITID: v.ID,
      NAME: v.Description,
    })
  }

  // Get our CWD to write the ouput file to
  dir, err := os.Getwd()
  check(err)
  file := filepath.Join(dir, "output.ofx")

  // run WriteXML
  WriteXML(OFXStruct, file);

  t, _ := template.ParseFiles("getTransactions.html")
  t.Execute(w, &s)
}

func WriteXML(o *OFX, outputfile string) {

  output, err := xml.MarshalIndent(o, "  ", "    ")
  check(err)
  d1 := []byte(output)
  err = ioutil.WriteFile(outputfile, d1, 0644)
  check(err)
}

func main() {
  // getsettings outputs to global variable
  getsettings()
  //rtr := mux.NewRouter()
  http.HandleFunc("/", indexHandler)
  http.HandleFunc("/getTransactions/", getTransactionsHandler)
  http.ListenAndServe(":8080", nil)
  log.Print("Started webserver on 8080");

}
<commit_msg>Excluded 0 value transactions<commit_after>// Writing files in Go follows similar patterns to the
// ones we saw earlier for reading.

package main

import (
  "io/ioutil"
  "bufio"
  "os"
  "log"
  "fmt"
  "encoding/xml"
  "net/http"
	//"github.com/gorilla/mux"
  "encoding/json"
  "html/template"
  "strings"
  "net/url"
  "path/filepath"
  "time"
)

type Transaction struct {
  TRNTYPE   string
  DTPOSTED  string
  TRNAMT    float32
  FITID     string
  NAME      string
}

type BankAccount struct {
  BANKID   string
  ACCTID   string
  ACCTTYPE string
}

type OFX struct {
  XMLName     xml.Name      `xml: "OFX"`
  BankAccount BankAccount   `xml:"BANKMSGSRSV1>STMTRS>BANKACCTFROM"`
  Transaction []Transaction `xml:"BANKMSGSRSV1>STMTRS>BANKTRANLIST>STMTTRN"`
}

type settings struct {
  ClientId string
  ClientSecret string
}

type accessToken struct {
  Access_token string
  Client_id string
  Expires_in string
  Refresh_token string
  Token_type string
  User_id string
}

type account struct {
  Id string
  Created string
  Description string
}

type accounts struct {
  Accounts []account
}

type transaction struct {
	Account_balance int
	Amount         int
	Attachments    []interface{}
	Category       string
	Created        string
	Currency       string
	Description    string
	ID             string
	Is_load         bool
	Merchant       string
	Metadata       map[string]interface{}
	Notes          string
	Settled        string
}

type transactions struct {
  Transactions []transaction
}

var s settings

func check(e error) {
    if e != nil {
        log.Fatal(e)
    }
}

func getsettings() {
  // Client ID
  reader := bufio.NewReader(os.Stdin)
  fmt.Print("Enter clientId: ")
  s.ClientId, _ = reader.ReadString('\n')
  s.ClientId = strings.TrimSpace(s.ClientId)

  // Client Secret
  reader = bufio.NewReader(os.Stdin)
  fmt.Print("Enter clientSecret: ")
  s.ClientSecret, _ = reader.ReadString('\n')
  s.ClientSecret = strings.TrimSpace(s.ClientSecret)

}

func indexHandler(w http.ResponseWriter, r *http.Request) {
  t, _ := template.ParseFiles("index.html")
  t.Execute(w, &s)
}

func getAuthCode(code string) accessToken  {
  resp, err := http.PostForm("https://api.getmondo.co.uk/oauth2/token",
	   url.Values{
        "grant_type": {"authorization_code"},
        "client_id": {s.ClientId},
        "client_secret": {s.ClientSecret},
        "redirect_uri": {"http://localhost:8080/getTransactions/"},
        "code": {code},
    },
  )
  check(err)
  defer resp.Body.Close()
  body, err := ioutil.ReadAll(resp.Body)
  check(err)
  var result accessToken
  json.Unmarshal([]byte(body), &result)
  return result
}

func getAccounts(authStruct accessToken) accounts {
  // Prepare HTTP request
  client := &http.Client{}
  req, err := http.NewRequest("GET", "https://api.getmondo.co.uk/accounts", nil)
  check(err)
  req.Header.Add("authorization", `Bearer ` + authStruct.Access_token)
  q := req.URL.Query()
  q.Add("account_id", authStruct.User_id)
  req.URL.RawQuery = q.Encode()
  resp, err := client.Do(req)
  defer resp.Body.Close()
  body, err := ioutil.ReadAll(resp.Body)

  // JSON decode result
  var result accounts
  json.Unmarshal([]byte(body), &result)

  return result
}

func getTransactions(authStruct accessToken, acccountStruct account) transactions{
  // Fetch transactions
  client := &http.Client{}
  req, err := http.NewRequest("GET", "https://api.getmondo.co.uk/transactions", nil)
  check(err)
  req.Header.Add("authorization", `Bearer ` + authStruct.Access_token)
  q := req.URL.Query()
  q.Add("account_id", acccountStruct.Id)
  req.URL.RawQuery = q.Encode()
  resp, err := client.Do(req)
  defer resp.Body.Close()
  body, err := ioutil.ReadAll(resp.Body)

  // JSON decode result
  var result transactions
  json.Unmarshal([]byte(body), &result)

  return result
}

func getTransactionsHandler(w http.ResponseWriter, r *http.Request){

  // Authenticate
  code := r.FormValue("code")
  authStruct := getAuthCode(code)

  accounts := getAccounts(authStruct)
  account := accounts.Accounts[0]

  transactions := getTransactions(authStruct, account)

  // Create an OFX
  OFXStruct := &OFX{}
  OFXStruct.BankAccount = BankAccount{
    BANKID: "0",
    ACCTID: account.Id,
    ACCTTYPE: "CHECKING",
  }

  // Loop through the transactions adding them to the OFX struct
  for _, v := range transactions.Transactions {

    // Exclude 0 value transactions (e.g. pin resets)
    if v.Amount == 0 {
      continue
    }
    log.Print(string(v.Description))

    time, _ := time.Parse(time.RFC3339, v.Created)
    formattedTime := time.Format("20060102150405.000[-07]")
    var amount = float32(v.Amount)/float32(100)
    OFXStruct.Transaction = append(OFXStruct.Transaction,Transaction{
      TRNTYPE: "POS",
      DTPOSTED: formattedTime,
      TRNAMT: amount,
      FITID: v.ID,
      NAME: v.Description,
    })
  }

  // Get our CWD to write the ouput file to
  dir, err := os.Getwd()
  check(err)
  file := filepath.Join(dir, "output.ofx")

  // run WriteXML
  WriteXML(OFXStruct, file);

  t, _ := template.ParseFiles("getTransactions.html")
  t.Execute(w, &s)
}

func WriteXML(o *OFX, outputfile string) {

  output, err := xml.MarshalIndent(o, "  ", "    ")
  check(err)
  d1 := []byte(output)
  err = ioutil.WriteFile(outputfile, d1, 0644)
  check(err)
}

func main() {
  // getsettings outputs to global variable
  getsettings()
  //rtr := mux.NewRouter()
  http.HandleFunc("/", indexHandler)
  http.HandleFunc("/getTransactions/", getTransactionsHandler)
  http.ListenAndServe(":8080", nil)
  log.Print("Started webserver on 8080");

}
<|endoftext|>


<commit_before>// Copyright 2014 James McGuire. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"database/sql"
	"encoding/json"
	"fmt"
	"html"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"regexp"
	"runtime"
	"strings"
	"syscall"
	"time"
	"unicode/utf8"

	irc "github.com/fluffle/goirc/client"
	_ "github.com/go-sql-driver/mysql"
	"github.com/mvdan/xurls"
	"golang.org/x/net/html/charset"
)

var (
	config   Config
	urlRegex = regexp.MustCompile(`(?i)\b((?:https?://|www\d{0,3}[.]|[` +
		`a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+` +
		`\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s` + "`" + `!()\[` +
		`\]{};:'".,<>?«»“”‘’]))`)
	httpRegex      = regexp.MustCompile(`https?://.*`)
	findWhiteSpace = regexp.MustCompile(`\s+`)
	db             *sql.DB
	badWords       = make(map[string]*regexp.Regexp)
	titleStart     = []byte("<title>")
	titleEnd       = []byte("</title>")
)

const FREENODE = "irc.freenode.net"

type Config struct {
	Channels      []string
	DBConn        string
	Nick          string
	Ident         string
	FullName      string
	FlickrAPIKey  string
	WolframAPIKey string
	IRCPass       string
	RebuildWords  bool
	Commands      []struct {
		Name string
		Text string
	}
	BadWords []struct {
		Word  string
		Query string
	}
}

// Try and grab the title for any URL's posted in the channel
func sendUrl(channel, unparsedURL string, conn *irc.Conn, nick string) {
	if !httpRegex.MatchString(unparsedURL) {
		unparsedURL = `http://` + unparsedURL
	}
	postedUrl, err := url.Parse(unparsedURL)
	if err != nil {
		log.Println(err)
		return
	}
	log.Println("Fetching title for " + postedUrl.String() + " In channel " + channel)

	resp, err := http.Get(postedUrl.String())
	if err != nil {
		log.Println(err)
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode >= 400 {
		log.Println("http server return error.")
		return
	}
	respbody := []byte{}
	if resp.Header.Get("Content-Type") == "" {
		buf := make([]byte, 512)
		bufsize, err := resp.Body.Read(buf)
		if err != nil {
			log.Println("adding content type failed")
		}
		resp.Header.Set("Content-Type", http.DetectContentType(buf[:bufsize]))
		respbody = append(respbody, buf[:bufsize]...)
	}

	if !strings.Contains(resp.Header.Get("Content-Type"), "text/html") {
		log.Println("content-type is not text/html")
		return
	}

	utf8Body, err := charset.NewReader(resp.Body, resp.Header.Get("Content-Type"))
	if err != nil {
		log.Println("error converting page to utf8")
		return
	}
	restofbody, err := ioutil.ReadAll(io.LimitReader(utf8Body, 50000))
	if err != nil {
		log.Println("error reading posted link")
		return
	}
	respbody = append(respbody, restofbody...)
	titlestart := bytes.Index(respbody, titleStart)
	titleend := bytes.Index(respbody, titleEnd)
	if titlestart == -1 || titleend == -1 {
		return
	}
	title := respbody[titlestart+7 : titleend]
	title = bytes.TrimSpace(title)
	if len(title) == 0 || !utf8.Valid(title) {
		return
	}
	// Example:
	// Title: sadbox . org (at sadbox.org)
	formattedTitle := html.UnescapeString(string(title))
	formattedTitle = findWhiteSpace.ReplaceAllString(formattedTitle, " ")
	formattedTitle = fmt.Sprintf("(%s) %s (at %s)", nick, formattedTitle, postedUrl.Host)
	log.Println(formattedTitle)
	conn.Privmsg(channel, formattedTitle)
}

func logMessage(line *irc.Line, channel, message string) {
	_, err := db.Exec("insert into messages (Nick, Ident, Host, Src, Cmd, Channel,"+
		" Message, Time) values (?, ?, ?, ?, ?, ?, ?, ?)", line.Nick, line.Ident,
		line.Host, line.Src, line.Cmd, channel, message, line.Time)
	if err != nil {
		log.Println(err)
	}
	err = updateWords(line.Nick, message)
	if err != nil {
		log.Println(err)
	}
}

func checkForUrl(channel, message, nick string, conn *irc.Conn) {
	urllist := make(map[string]struct{})
	for _, item := range xurls.Relaxed.FindAllString(message, -1) {
		urllist[item] = struct{}{}
	}
	numlinks := 0
	for item, _ := range urllist {
		numlinks++
		if numlinks > 3 {
			break
		}
		go sendUrl(channel, item, conn, nick)
	}
}

// This function does all the dispatching for various commands
// as well as logging each message to the database
func handleMessage(conn *irc.Conn, line *irc.Line) {
	// This is so that the bot can properly respond to pm's
	channel := line.Target()
	message := line.Args[1]
	splitmessage := strings.Split(message, " ")
	cmd := strings.TrimSpace(splitmessage[0])

	// sadbox-only commands!
	if line.Nick == "sadbox" {
		switch cmd {
		case "!dance":
			go dance(channel, conn)
		case "!audio":
			go conn.Privmsg(channel, "https://sadbox.org/static/stuff/audiophile.html")
		case "!cst":
			go conn.Privmsg(channel, "\u00039,13#CSTMASTERRACE")
		case "!chatter":
			go markov(channel, conn)
		}
	}

	// Special commands
	switch cmd {
	case "!haata":
		go haata(channel, conn)
	case "!search":
		go googSearch(channel, message, conn)
	case "!ask":
		go wolfram(channel, message, line.Nick, conn)
	case "!meebcast":
		var command string
		if len(splitmessage) >= 2 {
			command = strings.TrimSpace(splitmessage[1])
		}
		go meeba(channel, line.Nick, command, conn)
	}

	// Commands that are read in from the config file
	for _, command := range config.Commands {
		if cmd == command.Name {
			var response string
			if len(splitmessage) >= 2 {
				response = fmt.Sprintf("%s: %s", splitmessage[1], command.Text)
			} else {
				response = command.Text
			}
			go conn.Privmsg(channel, response)
		}
	}

	// This is what looks at each word and tries to figure out if it's a URL
	go checkForUrl(channel, message, line.Nick, conn)

	// Shove that shit in the database!
	go logMessage(line, channel, message)
}

func init() {
	log.Println("Starting sadbot")

	runtime.GOMAXPROCS(runtime.NumCPU())

	rand.Seed(time.Now().UTC().UnixNano())

	configfile, err := os.Open("config.json")
	if err != nil {
		log.Fatal(err)
	}
	err = json.NewDecoder(configfile).Decode(&config)
	if err != nil {
		log.Fatal(err)
	}

	for _, word := range config.BadWords {
		badWords[word.Word] = regexp.MustCompile(word.Query)
	}

	log.Println("Loaded config file!")
	log.Printf("Joining: %s", config.Channels)
	log.Printf("Nick: %s", config.Nick)
	log.Printf("Ident: %s", config.Ident)
	log.Printf("FullName: %s", config.FullName)

	log.Printf("Found %d commands", len(config.Commands))
	for index, command := range config.Commands {
		log.Printf("%d %s: %s", index+1, command.Name, command.Text)
	}
}

func main() {
	var err error
	db, err = sql.Open("mysql", config.DBConn)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()
	db.SetMaxIdleConns(100)
	db.SetMaxOpenConns(200)

	go makeMarkov()

	buildchan := make(chan os.Signal, 1)
	signal.Notify(buildchan, syscall.SIGUSR1)
	go func() {
		for _ = range buildchan {
			genTables()
		}
	}()

	ircConfig := irc.NewConfig(config.Nick, config.Ident, config.FullName)
	ircConfig.SSL = true
	ircConfig.Server = FREENODE
	ircConfig.Pass = config.Nick + ":" + config.IRCPass

	c := irc.Client(ircConfig)

	c.HandleFunc(irc.CONNECTED,
		func(conn *irc.Conn, line *irc.Line) {
			for _, channel := range config.Channels {
				log.Printf("Joining %s", channel)
				conn.Join(channel)
			}
			log.Println("Connected!")
		})
	quit := make(chan bool)

	c.HandleFunc(irc.DISCONNECTED,
		func(conn *irc.Conn, line *irc.Line) { quit <- true })

	c.HandleFunc(irc.PRIVMSG, handleMessage)
	c.HandleFunc(irc.ACTION, handleMessage)

	if err := c.Connect(); err != nil {
		log.Fatalln("Connection error: %s\n", err)
	}

	<-quit
}
<commit_msg>remove unused regex<commit_after>// Copyright 2014 James McGuire. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"database/sql"
	"encoding/json"
	"fmt"
	"html"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"regexp"
	"runtime"
	"strings"
	"syscall"
	"time"
	"unicode/utf8"

	irc "github.com/fluffle/goirc/client"
	_ "github.com/go-sql-driver/mysql"
	"github.com/mvdan/xurls"
	"golang.org/x/net/html/charset"
)

var (
	config         Config
	httpRegex      = regexp.MustCompile(`https?://.*`)
	findWhiteSpace = regexp.MustCompile(`\s+`)
	db             *sql.DB
	badWords       = make(map[string]*regexp.Regexp)
	titleStart     = []byte("<title>")
	titleEnd       = []byte("</title>")
)

const FREENODE = "irc.freenode.net"

type Config struct {
	Channels      []string
	DBConn        string
	Nick          string
	Ident         string
	FullName      string
	FlickrAPIKey  string
	WolframAPIKey string
	IRCPass       string
	RebuildWords  bool
	Commands      []struct {
		Name string
		Text string
	}
	BadWords []struct {
		Word  string
		Query string
	}
}

// Try and grab the title for any URL's posted in the channel
func sendUrl(channel, unparsedURL string, conn *irc.Conn, nick string) {
	if !httpRegex.MatchString(unparsedURL) {
		unparsedURL = `http://` + unparsedURL
	}
	postedUrl, err := url.Parse(unparsedURL)
	if err != nil {
		log.Println(err)
		return
	}
	log.Println("Fetching title for " + postedUrl.String() + " In channel " + channel)

	resp, err := http.Get(postedUrl.String())
	if err != nil {
		log.Println(err)
		return
	}
	defer resp.Body.Close()
	if resp.StatusCode >= 400 {
		log.Println("http server return error.")
		return
	}
	respbody := []byte{}
	if resp.Header.Get("Content-Type") == "" {
		buf := make([]byte, 512)
		bufsize, err := resp.Body.Read(buf)
		if err != nil {
			log.Println("adding content type failed")
		}
		resp.Header.Set("Content-Type", http.DetectContentType(buf[:bufsize]))
		respbody = append(respbody, buf[:bufsize]...)
	}

	if !strings.Contains(resp.Header.Get("Content-Type"), "text/html") {
		log.Println("content-type is not text/html")
		return
	}

	utf8Body, err := charset.NewReader(resp.Body, resp.Header.Get("Content-Type"))
	if err != nil {
		log.Println("error converting page to utf8")
		return
	}
	restofbody, err := ioutil.ReadAll(io.LimitReader(utf8Body, 50000))
	if err != nil {
		log.Println("error reading posted link")
		return
	}
	respbody = append(respbody, restofbody...)
	titlestart := bytes.Index(respbody, titleStart)
	titleend := bytes.Index(respbody, titleEnd)
	if titlestart == -1 || titleend == -1 {
		return
	}
	title := respbody[titlestart+7 : titleend]
	title = bytes.TrimSpace(title)
	if len(title) == 0 || !utf8.Valid(title) {
		return
	}
	// Example:
	// Title: sadbox . org (at sadbox.org)
	formattedTitle := html.UnescapeString(string(title))
	formattedTitle = findWhiteSpace.ReplaceAllString(formattedTitle, " ")
	formattedTitle = fmt.Sprintf("(%s) %s (at %s)", nick, formattedTitle, postedUrl.Host)
	log.Println(formattedTitle)
	conn.Privmsg(channel, formattedTitle)
}

func logMessage(line *irc.Line, channel, message string) {
	_, err := db.Exec("insert into messages (Nick, Ident, Host, Src, Cmd, Channel,"+
		" Message, Time) values (?, ?, ?, ?, ?, ?, ?, ?)", line.Nick, line.Ident,
		line.Host, line.Src, line.Cmd, channel, message, line.Time)
	if err != nil {
		log.Println(err)
	}
	err = updateWords(line.Nick, message)
	if err != nil {
		log.Println(err)
	}
}

func checkForUrl(channel, message, nick string, conn *irc.Conn) {
	urllist := make(map[string]struct{})
	for _, item := range xurls.Relaxed.FindAllString(message, -1) {
		urllist[item] = struct{}{}
	}
	numlinks := 0
	for item, _ := range urllist {
		numlinks++
		if numlinks > 3 {
			break
		}
		go sendUrl(channel, item, conn, nick)
	}
}

// This function does all the dispatching for various commands
// as well as logging each message to the database
func handleMessage(conn *irc.Conn, line *irc.Line) {
	// This is so that the bot can properly respond to pm's
	channel := line.Target()
	message := line.Args[1]
	splitmessage := strings.Split(message, " ")
	cmd := strings.TrimSpace(splitmessage[0])

	// sadbox-only commands!
	if line.Nick == "sadbox" {
		switch cmd {
		case "!dance":
			go dance(channel, conn)
		case "!audio":
			go conn.Privmsg(channel, "https://sadbox.org/static/stuff/audiophile.html")
		case "!cst":
			go conn.Privmsg(channel, "\u00039,13#CSTMASTERRACE")
		case "!chatter":
			go markov(channel, conn)
		}
	}

	// Special commands
	switch cmd {
	case "!haata":
		go haata(channel, conn)
	case "!search":
		go googSearch(channel, message, conn)
	case "!ask":
		go wolfram(channel, message, line.Nick, conn)
	case "!meebcast":
		var command string
		if len(splitmessage) >= 2 {
			command = strings.TrimSpace(splitmessage[1])
		}
		go meeba(channel, line.Nick, command, conn)
	}

	// Commands that are read in from the config file
	for _, command := range config.Commands {
		if cmd == command.Name {
			var response string
			if len(splitmessage) >= 2 {
				response = fmt.Sprintf("%s: %s", splitmessage[1], command.Text)
			} else {
				response = command.Text
			}
			go conn.Privmsg(channel, response)
		}
	}

	// This is what looks at each word and tries to figure out if it's a URL
	go checkForUrl(channel, message, line.Nick, conn)

	// Shove that shit in the database!
	go logMessage(line, channel, message)
}

func init() {
	log.Println("Starting sadbot")

	runtime.GOMAXPROCS(runtime.NumCPU())

	rand.Seed(time.Now().UTC().UnixNano())

	configfile, err := os.Open("config.json")
	if err != nil {
		log.Fatal(err)
	}
	err = json.NewDecoder(configfile).Decode(&config)
	if err != nil {
		log.Fatal(err)
	}

	for _, word := range config.BadWords {
		badWords[word.Word] = regexp.MustCompile(word.Query)
	}

	log.Println("Loaded config file!")
	log.Printf("Joining: %s", config.Channels)
	log.Printf("Nick: %s", config.Nick)
	log.Printf("Ident: %s", config.Ident)
	log.Printf("FullName: %s", config.FullName)

	log.Printf("Found %d commands", len(config.Commands))
	for index, command := range config.Commands {
		log.Printf("%d %s: %s", index+1, command.Name, command.Text)
	}
}

func main() {
	var err error
	db, err = sql.Open("mysql", config.DBConn)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()
	db.SetMaxIdleConns(100)
	db.SetMaxOpenConns(200)

	go makeMarkov()

	buildchan := make(chan os.Signal, 1)
	signal.Notify(buildchan, syscall.SIGUSR1)
	go func() {
		for _ = range buildchan {
			genTables()
		}
	}()

	ircConfig := irc.NewConfig(config.Nick, config.Ident, config.FullName)
	ircConfig.SSL = true
	ircConfig.Server = FREENODE
	ircConfig.Pass = config.Nick + ":" + config.IRCPass

	c := irc.Client(ircConfig)

	c.HandleFunc(irc.CONNECTED,
		func(conn *irc.Conn, line *irc.Line) {
			for _, channel := range config.Channels {
				log.Printf("Joining %s", channel)
				conn.Join(channel)
			}
			log.Println("Connected!")
		})
	quit := make(chan bool)

	c.HandleFunc(irc.DISCONNECTED,
		func(conn *irc.Conn, line *irc.Line) { quit <- true })

	c.HandleFunc(irc.PRIVMSG, handleMessage)
	c.HandleFunc(irc.ACTION, handleMessage)

	if err := c.Connect(); err != nil {
		log.Fatalln("Connection error: %s\n", err)
	}

	<-quit
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/miekg/dns"
	"math/rand"
	"net"
	"os"
	"regexp"
	"strings"
	"time"
)

func rrsearch(ns string, q string, t uint16) *dns.Msg {
	c := new(dns.Client)
	m := new(dns.Msg)
	m.SetQuestion(dns.Fqdn(q), t)
	r, _, err := c.Exchange(m, net.JoinHostPort(ns, "53"))

	if err != nil {
		fmt.Println(err.Error())
		panic("rrsearch Error")
	}
	return r
}

func splitRR(rr dns.RR) []string {
	rrary := strings.SplitN(rr.String(), "\t", 5)
	return rrary
}

func recRsolve(dst, ns string, n int) string {
	var tab string
	for i := 0; i < n; i++ {
		tab = tab + "\t"
	}
	rand.Seed(time.Now().UnixNano())
	for {
		checkNS, _ := regexp.MatchString("^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}$", ns)
		if checkNS == false {
			ns = recRsolve(ns, "202.12.27.33", n+1)
		}
		r := rrsearch(ns, dst, dns.TypeA)
		if len(r.Answer) == 0 {
			rrs := splitRR(r.Ns[rand.Intn(len(r.Ns))])
			fmt.Println(tab + ns + "=>")
			fmt.Printf(tab+"\t%s -> %s\n",
				rrs[0],
				rrs[4],
			)
			if strings.Contains(rrs[4], rrs[0]) {
				for _, rr := range r.Extra {
					rrss := splitRR(rr)
					if strings.Compare(rrss[0], rrs[4]) == 0 {
						ns = rrss[4]
						break
					}
				}
			} else {
				ns = rrs[4]
			}
		} else {
			fmt.Println(tab + "Answer : " + ns + "=>")
			var ip string
			for _, ans := range r.Answer {
				anss := splitRR(ans)
				fmt.Printf(tab+"\t%s -> %s\n",
					dst,
					anss[4],
				)
				ip = anss[4]
			}
			return ip
		}
	}
}

func main() {
	dst := os.Args
	recRsolve(dst[1], "202.12.27.33", 0)
}
<commit_msg>AAAAレコードを無視<commit_after>package main

import (
	"fmt"
	"github.com/miekg/dns"
	"math/rand"
	"net"
	"os"
	"regexp"
	"strings"
	"time"
)

func rrsearch(ns string, q string, t uint16) *dns.Msg {
	c := new(dns.Client)
	m := new(dns.Msg)
	m.SetQuestion(dns.Fqdn(q), t)
	r, _, err := c.Exchange(m, net.JoinHostPort(ns, "53"))

	if err != nil {
		fmt.Println(err.Error())
		panic("rrsearch Error")
	}
	return r
}

func splitRR(rr dns.RR) []string {
	rrary := strings.SplitN(rr.String(), "\t", 5)
	return rrary
}

func setNS(rrs []string, r *dns.Msg) (string, string) {
	var ns string
	typ := "A"

	if strings.Contains(rrs[4], rrs[0]) {
		for _, rr := range r.Extra {
			rrss := splitRR(rr)
			if strings.Compare(rrss[0], rrs[4]) == 0 {
				ns = rrss[4]
				typ = rrss[3]
			}
		}
	} else {
		ns = rrs[4]
	}

	return ns, typ
}

func recRsolve(dst, ns string, n int) string {
	var tab string
	for i := 0; i < n; i++ {
		tab = tab + "\t"
	}
	rand.Seed(time.Now().UnixNano())
	for {
		checkNS, _ := regexp.MatchString("^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}$", ns)
		if checkNS == false {
			ns = recRsolve(ns, "202.12.27.33", n+1)
		}
		r := rrsearch(ns, dst, dns.TypeA)
		if len(r.Answer) == 0 {
			rrs := splitRR(r.Ns[rand.Intn(len(r.Ns))])
			nss, typ := setNS(rrs, r)
			for strings.Compare(typ, "AAAA") == 0 {
				rrs := splitRR(r.Ns[rand.Intn(len(r.Ns))])
				nss, typ = setNS(rrs, r)
			}
			ns = nss
			fmt.Println(tab + ns + "=>")
			fmt.Printf(tab+"\t%s -> %s\n",
				rrs[0],
				rrs[4],
			)

		} else {
			fmt.Println(tab + "Answer : " + ns + "=>")
			var ip string
			for _, ans := range r.Answer {
				anss := splitRR(ans)
				fmt.Printf(tab+"\t%s -> %s\n",
					dst,
					anss[4],
				)
				ip = anss[4]
			}
			return ip
		}
	}
}

func main() {
	dst := os.Args
	recRsolve(dst[1], "202.12.27.33", 0)
}
<|endoftext|>


<commit_before>package couchbase

import (
	"errors"
	"github.com/couchbase/goutils/logging"
	"time"

	"github.com/couchbase/gomemcached"
	"github.com/couchbase/gomemcached/client"
)

// GenericMcdAuthHandler is a kind of AuthHandler that performs
// special auth exchange (like non-standard auth, possibly followed by
// select-bucket).
type GenericMcdAuthHandler interface {
	AuthHandler
	AuthenticateMemcachedConn(host string, conn *memcached.Client) error
}

// Error raised when a connection can't be retrieved from a pool.
var TimeoutError = errors.New("timeout waiting to build connection")
var errClosedPool = errors.New("the connection pool is closed")
var errNoPool = errors.New("no connection pool")

// Default timeout for retrieving a connection from the pool.
var ConnPoolTimeout = time.Hour * 24 * 30

// ConnPoolAvailWaitTime is the amount of time to wait for an existing
// connection from the pool before considering the creation of a new
// one.
var ConnPoolAvailWaitTime = time.Millisecond

type connectionPool struct {
	host        string
	mkConn      func(host string, ah AuthHandler) (*memcached.Client, error)
	auth        AuthHandler
	connections chan *memcached.Client
	createsem   chan bool
	inUse       bool
}

func newConnectionPool(host string, ah AuthHandler, poolSize, poolOverflow int) *connectionPool {
	return &connectionPool{
		host:        host,
		connections: make(chan *memcached.Client, poolSize),
		createsem:   make(chan bool, poolSize+poolOverflow),
		mkConn:      defaultMkConn,
		auth:        ah,
	}
}

// ConnPoolTimeout is notified whenever connections are acquired from a pool.
var ConnPoolCallback func(host string, source string, start time.Time, err error)

func defaultMkConn(host string, ah AuthHandler) (*memcached.Client, error) {
	conn, err := memcached.Connect("tcp", host)
	if err != nil {
		return nil, err
	}

	if TCPKeepalive == true {
		conn.SetKeepAliveOptions(time.Duration(TCPKeepaliveInterval) * time.Second)
	}

	if EnableMutationToken == true {
		res, err := conn.EnableMutationToken()
		if err != nil || res.Status != gomemcached.SUCCESS {
			logging.Warnf("Unable to enable mutation token %v", err)
		}
	}

	if gah, ok := ah.(GenericMcdAuthHandler); ok {
		err = gah.AuthenticateMemcachedConn(host, conn)
		if err != nil {
			conn.Close()
			return nil, err
		}
		return conn, nil
	}
	name, pass, bucket := ah.GetCredentials()
	if name != "default" {
		_, err = conn.Auth(name, pass)
		if err != nil {
			conn.Close()
			return nil, err
		}
		// Select bucket (Required for cb_auth creds)
		// Required when doing auth with _admin credentials
		if bucket != "" && bucket != name {
			_, err = conn.SelectBucket(bucket)
			if err != nil {
				conn.Close()
				return nil, err
			}
		}
	}
	return conn, nil
}

func (cp *connectionPool) Close() (err error) {
	defer func() { err, _ = recover().(error) }()
	close(cp.connections)
	for c := range cp.connections {
		c.Close()
	}
	return
}

func (cp *connectionPool) GetWithTimeout(d time.Duration) (rv *memcached.Client, err error) {
	if cp == nil {
		return nil, errNoPool
	}

	path := ""

	if ConnPoolCallback != nil {
		defer func(path *string, start time.Time) {
			ConnPoolCallback(cp.host, *path, start, err)
		}(&path, time.Now())
	}

	path = "short-circuit"

	// short-circuit available connetions.
	select {
	case rv, isopen := <-cp.connections:
		if !isopen {
			return nil, errClosedPool
		}
		return rv, nil
	default:
	}

	t := time.NewTimer(ConnPoolAvailWaitTime)
	defer t.Stop()

	// Try to grab an available connection within 1ms
	select {
	case rv, isopen := <-cp.connections:
		path = "avail1"
		if !isopen {
			return nil, errClosedPool
		}
		return rv, nil
	case <-t.C:
		// No connection came around in time, let's see
		// whether we can get one or build a new one first.
		t.Reset(d) // Reuse the timer for the full timeout.
		select {
		case rv, isopen := <-cp.connections:
			path = "avail2"
			if !isopen {
				return nil, errClosedPool
			}
			return rv, nil
		case cp.createsem <- true:
			path = "create"
			// Build a connection if we can't get a real one.
			// This can potentially be an overflow connection, or
			// a pooled connection.
			rv, err := cp.mkConn(cp.host, cp.auth)
			if err != nil {
				// On error, release our create hold
				<-cp.createsem
			}
			return rv, err
		case <-t.C:
			return nil, ErrTimeout
		}
	}
}

func (cp *connectionPool) Get() (*memcached.Client, error) {
	return cp.GetWithTimeout(ConnPoolTimeout)
}

func (cp *connectionPool) Return(c *memcached.Client) {
	if c == nil {
		return
	}

	if cp == nil {
		c.Close()
	}

	if c.IsHealthy() {
		defer func() {
			if recover() != nil {
				// This happens when the pool has already been
				// closed and we're trying to return a
				// connection to it anyway.  Just close the
				// connection.
				c.Close()
			}
		}()

		select {
		case cp.connections <- c:
		default:
			// Overflow connection.
			<-cp.createsem
			c.Close()
		}
	} else {
		<-cp.createsem
		c.Close()
	}
}

func (cp *connectionPool) StartTapFeed(args *memcached.TapArguments) (*memcached.TapFeed, error) {
	if cp == nil {
		return nil, errNoPool
	}
	mc, err := cp.Get()
	if err != nil {
		return nil, err
	}

	// A connection can't be used after TAP; Dont' count it against the
	// connection pool capacity
	<-cp.createsem

	return mc.StartTapFeed(*args)
}

const DEFAULT_WINDOW_SIZE = 20 * 1024 * 1024 // 20 Mb

func (cp *connectionPool) StartUprFeed(name string, sequence uint32, dcp_buffer_size uint32, data_chan_size int) (*memcached.UprFeed, error) {
	if cp == nil {
		return nil, errNoPool
	}
	mc, err := cp.Get()
	if err != nil {
		return nil, err
	}

	// A connection can't be used after it has been allocated to UPR;
	// Dont' count it against the connection pool capacity
	<-cp.createsem

	uf, err := mc.NewUprFeed()
	if err != nil {
		return nil, err
	}

	if err := uf.UprOpen(name, sequence, dcp_buffer_size); err != nil {
		return nil, err
	}

	if err := uf.StartFeedWithConfig(data_chan_size); err != nil {
		return nil, err
	}

	return uf, nil
}
<commit_msg>fix conn pool Close() to pass unit tests<commit_after>package couchbase

import (
	"errors"
	"github.com/couchbase/goutils/logging"
	"time"

	"github.com/couchbase/gomemcached"
	"github.com/couchbase/gomemcached/client"
)

// GenericMcdAuthHandler is a kind of AuthHandler that performs
// special auth exchange (like non-standard auth, possibly followed by
// select-bucket).
type GenericMcdAuthHandler interface {
	AuthHandler
	AuthenticateMemcachedConn(host string, conn *memcached.Client) error
}

// Error raised when a connection can't be retrieved from a pool.
var TimeoutError = errors.New("timeout waiting to build connection")
var errClosedPool = errors.New("the connection pool is closed")
var errNoPool = errors.New("no connection pool")

// Default timeout for retrieving a connection from the pool.
var ConnPoolTimeout = time.Hour * 24 * 30

// ConnPoolAvailWaitTime is the amount of time to wait for an existing
// connection from the pool before considering the creation of a new
// one.
var ConnPoolAvailWaitTime = time.Millisecond

type connectionPool struct {
	host        string
	mkConn      func(host string, ah AuthHandler) (*memcached.Client, error)
	auth        AuthHandler
	connections chan *memcached.Client
	createsem   chan bool
	inUse       bool
}

func newConnectionPool(host string, ah AuthHandler, poolSize, poolOverflow int) *connectionPool {
	return &connectionPool{
		host:        host,
		connections: make(chan *memcached.Client, poolSize),
		createsem:   make(chan bool, poolSize+poolOverflow),
		mkConn:      defaultMkConn,
		auth:        ah,
	}
}

// ConnPoolTimeout is notified whenever connections are acquired from a pool.
var ConnPoolCallback func(host string, source string, start time.Time, err error)

func defaultMkConn(host string, ah AuthHandler) (*memcached.Client, error) {
	conn, err := memcached.Connect("tcp", host)
	if err != nil {
		return nil, err
	}

	if TCPKeepalive == true {
		conn.SetKeepAliveOptions(time.Duration(TCPKeepaliveInterval) * time.Second)
	}

	if EnableMutationToken == true {
		res, err := conn.EnableMutationToken()
		if err != nil || res.Status != gomemcached.SUCCESS {
			logging.Warnf("Unable to enable mutation token %v", err)
		}
	}

	if gah, ok := ah.(GenericMcdAuthHandler); ok {
		err = gah.AuthenticateMemcachedConn(host, conn)
		if err != nil {
			conn.Close()
			return nil, err
		}
		return conn, nil
	}
	name, pass, bucket := ah.GetCredentials()
	if name != "default" {
		_, err = conn.Auth(name, pass)
		if err != nil {
			conn.Close()
			return nil, err
		}
		// Select bucket (Required for cb_auth creds)
		// Required when doing auth with _admin credentials
		if bucket != "" && bucket != name {
			_, err = conn.SelectBucket(bucket)
			if err != nil {
				conn.Close()
				return nil, err
			}
		}
	}
	return conn, nil
}

func (cp *connectionPool) Close() (err error) {
	defer func() {
		if recover() != nil {
			err = errors.New("connectionPool.Close error")
		}
	}()
	close(cp.connections)
	for c := range cp.connections {
		c.Close()
	}
	return
}

func (cp *connectionPool) GetWithTimeout(d time.Duration) (rv *memcached.Client, err error) {
	if cp == nil {
		return nil, errNoPool
	}

	path := ""

	if ConnPoolCallback != nil {
		defer func(path *string, start time.Time) {
			ConnPoolCallback(cp.host, *path, start, err)
		}(&path, time.Now())
	}

	path = "short-circuit"

	// short-circuit available connetions.
	select {
	case rv, isopen := <-cp.connections:
		if !isopen {
			return nil, errClosedPool
		}
		return rv, nil
	default:
	}

	t := time.NewTimer(ConnPoolAvailWaitTime)
	defer t.Stop()

	// Try to grab an available connection within 1ms
	select {
	case rv, isopen := <-cp.connections:
		path = "avail1"
		if !isopen {
			return nil, errClosedPool
		}
		return rv, nil
	case <-t.C:
		// No connection came around in time, let's see
		// whether we can get one or build a new one first.
		t.Reset(d) // Reuse the timer for the full timeout.
		select {
		case rv, isopen := <-cp.connections:
			path = "avail2"
			if !isopen {
				return nil, errClosedPool
			}
			return rv, nil
		case cp.createsem <- true:
			path = "create"
			// Build a connection if we can't get a real one.
			// This can potentially be an overflow connection, or
			// a pooled connection.
			rv, err := cp.mkConn(cp.host, cp.auth)
			if err != nil {
				// On error, release our create hold
				<-cp.createsem
			}
			return rv, err
		case <-t.C:
			return nil, ErrTimeout
		}
	}
}

func (cp *connectionPool) Get() (*memcached.Client, error) {
	return cp.GetWithTimeout(ConnPoolTimeout)
}

func (cp *connectionPool) Return(c *memcached.Client) {
	if c == nil {
		return
	}

	if cp == nil {
		c.Close()
	}

	if c.IsHealthy() {
		defer func() {
			if recover() != nil {
				// This happens when the pool has already been
				// closed and we're trying to return a
				// connection to it anyway.  Just close the
				// connection.
				c.Close()
			}
		}()

		select {
		case cp.connections <- c:
		default:
			// Overflow connection.
			<-cp.createsem
			c.Close()
		}
	} else {
		<-cp.createsem
		c.Close()
	}
}

func (cp *connectionPool) StartTapFeed(args *memcached.TapArguments) (*memcached.TapFeed, error) {
	if cp == nil {
		return nil, errNoPool
	}
	mc, err := cp.Get()
	if err != nil {
		return nil, err
	}

	// A connection can't be used after TAP; Dont' count it against the
	// connection pool capacity
	<-cp.createsem

	return mc.StartTapFeed(*args)
}

const DEFAULT_WINDOW_SIZE = 20 * 1024 * 1024 // 20 Mb

func (cp *connectionPool) StartUprFeed(name string, sequence uint32, dcp_buffer_size uint32, data_chan_size int) (*memcached.UprFeed, error) {
	if cp == nil {
		return nil, errNoPool
	}
	mc, err := cp.Get()
	if err != nil {
		return nil, err
	}

	// A connection can't be used after it has been allocated to UPR;
	// Dont' count it against the connection pool capacity
	<-cp.createsem

	uf, err := mc.NewUprFeed()
	if err != nil {
		return nil, err
	}

	if err := uf.UprOpen(name, sequence, dcp_buffer_size); err != nil {
		return nil, err
	}

	if err := uf.StartFeedWithConfig(data_chan_size); err != nil {
		return nil, err
	}

	return uf, nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"regexp"
	"strings"

	"github.com/dfordsoft/golib/ebook"
	"github.com/golang/freetype/truetype"
	flags "github.com/jessevdk/go-flags"
)

// Options for all command line options
type Options struct {
	Format          string  `short:"f" long:"format" description:"set generated file format, candidate values: mobi, epub, pdf"`
	List            bool    `short:"l" long:"list" description:"list supported novel websites"`
	LeftMargin      float64 `long:"leftMargin" description:"set left margin for PDF format"`
	TopMargin       float64 `long:"topMargin" description:"set top margin for PDF format"`
	PageType        string  `short:"p" long:"pageType" description:"set page type for PDF format, candidate values: a0, a1, a2, a3, a4, a5, a6, b0, b1, b2, b3, b4, b5, b6, c0, c1, c2, c3, c4, c5, c6, dxg(=a4), 6inch(90mm x 117mm), 7inch, 10inch(=a4), pc(=a4 & 25.4mm left margin & 31.7mm top margin & 16 point title font size & 12 point content font size)"`
	TitleFontSize   int     `long:"titleFontSize" description:"set title font point size for PDF format"`
	ContentFontSize int     `long:"contentFontSize" description:"set content font point size for PDF format"`
	LineSpacing     float64 `long:"lineSpacing" description:"set line spacing rate for PDF format"`
	PagesPerFile    int     `long:"pagesPerFile" description:"split the big single PDF file to several smaller PDF files, how many pages should be included in a file, 0 means don't split"`
	ChaptersPerFile int     `long:"chaptersPerFile" description:"split the big signle PDF file to several smaller PDF files, how many chapters should be included in a file, 0 means don't split"`
	FontFile        string  `long:"fontFile" description:"set TTF font file path"`
	RetryCount      int     `short:"r" long:"retries" description:"download retry count"`
	Timeout         int     `short:"t" long:"timeout" description:"download timeout seconds"`
	ParallelCount   int64   `long:"parallel" description:"parallel count for downloading"`
	ConfigFile      string  `short:"c" long:"config" description:"read configurations from local file"`
	OutputFile      string  `short:"o" long:"output" description:"output file path"`
	FontFamily      string
}

type tocPattern struct {
	host            string
	bookTitle       string
	bookTitlePos    int
	item            string
	articleTitlePos int
	articleURLPos   int
	isAbsoluteURL   bool
}

type pageContentMarker struct {
	host  string
	start []byte
	end   []byte
}

type novelSiteHandler struct {
	Title         string
	MatchPatterns []string
	Download      func(string)
}

var (
	novelSiteHandlers []*novelSiteHandler
	gen               ebook.IBook
	opts              Options
)

func registerNovelSiteHandler(h *novelSiteHandler) {
	novelSiteHandlers = append(novelSiteHandlers, h)
}

func listCommandHandler() {
	fmt.Println("支持小说网站：")
	for _, h := range novelSiteHandlers {
		urlMap := make(map[string]struct{})
		for _, p := range h.MatchPatterns {
			u := strings.Replace(p, `\`, ``, -1)
			idxStart := strings.Index(u, `www.`)
			idxEnd := strings.Index(u[idxStart:], `/`)
			u = u[:idxStart+idxEnd]
			urlMap[u] = struct{}{}
		}
		var urls []string
		for u := range urlMap {
			urls = append(urls, u)
		}
		fmt.Println("\t" + h.Title + ": " + strings.Join(urls, ", "))
	}
}

func readConfigFile(opts *Options) bool {
	if opts.ConfigFile != "" {
		contentFd, err := os.OpenFile(opts.ConfigFile, os.O_RDONLY, 0644)
		if err != nil {
			log.Println("opening config file ", opts.ConfigFile, " for reading failed ", err)
			return false
		}

		contentC, err := ioutil.ReadAll(contentFd)
		if err != nil {
			log.Println("reading config file ", opts.ConfigFile, " failed ", err)
			return false
		}
		contentFd.Close()

		var options map[string]interface{}
		if err = json.Unmarshal(contentC, &options); err != nil {
			log.Println("unmarshall configurations failed", err)
			return false
		}

		if f, ok := options["format"]; ok {
			if v := f.(string); len(v) > 0 {
				opts.Format = v
			}
		}
		if f, ok := options["pageType"]; ok {
			if v := f.(string); len(v) > 0 {
				opts.PageType = v
			}
		}
		if f, ok := options["fontFamily"]; ok {
			if v := f.(string); len(v) > 0 {
				opts.FontFamily = v
			}
		}
		if f, ok := options["fontFile"]; ok {
			if v := f.(string); len(v) > 0 {
				opts.FontFile = v
			}
		}

		if f, ok := options["leftMargin"]; ok {
			if v := f.(float64); v > 0 {
				opts.LeftMargin = v
			}
		}
		if f, ok := options["topMargin"]; ok {
			if v := f.(float64); v > 0 {
				opts.TopMargin = v
			}
		}
		if f, ok := options["lineSpacing"]; ok {
			if v := f.(float64); v > 0 {
				opts.LineSpacing = v
			}
		}
		if f, ok := options["titleFontSize"]; ok {
			if v := f.(int); v > 0 {
				opts.TitleFontSize = v
			}
		}
		if f, ok := options["contentFontSize"]; ok {
			if v := f.(int); v > 0 {
				opts.ContentFontSize = v
			}
		}
		if f, ok := options["pagesPerFile"]; ok {
			if v := f.(int); v > 0 {
				opts.PagesPerFile = v
			}
		}
		if f, ok := options["chaptersPerFile"]; ok {
			if v := f.(int); v > 0 {
				opts.ChaptersPerFile = v
			}
		}
		if f, ok := options["retries"]; ok {
			if v := f.(int); v > 0 {
				opts.RetryCount = v
			}
		}
		if f, ok := options["timeout"]; ok {
			if v := f.(int); v > 0 {
				opts.Timeout = v
			}
		}
		if f, ok := options["parallel"]; ok {
			if v := f.(int64); v > 0 {
				opts.ParallelCount = v
			}
		}
	}
	return true
}

func main() {
	if len(os.Args) < 2 {
		fmt.Println("使用方法：\n\tgetnovel 小说目录网址")
		listCommandHandler()
		return
	}

	opts = Options{
		Format:          "mobi",
		List:            false,
		LeftMargin:      10,
		TopMargin:       10,
		PageType:        "a4",
		TitleFontSize:   24,
		ContentFontSize: 18,
		LineSpacing:     1.2,
		PagesPerFile:    0,
		ChaptersPerFile: 0,
		FontFile:        "fonts/CustomFont.ttf",
		RetryCount:      3,
		Timeout:         60,
		ParallelCount:   10,
	}

	args, err := flags.Parse(&opts)
	if err != nil {
		return
	}

	if opts.List {
		listCommandHandler()
		return
	}

	if !readConfigFile(&opts) {
		return
	}

	// check font files
	fontFd, err := os.OpenFile(opts.FontFile, os.O_RDONLY, 0644)
	if err != nil {
		log.Fatalln("can't find font file", opts.FontFile, err)
		return
	}
	defer fontFd.Close()

	fontContent, err := ioutil.ReadAll(fontFd)
	if err != nil {
		log.Fatalln("can't read font file", err)
		return
	}

	font, err := truetype.Parse(fontContent)
	if err != nil {
		log.Fatalln("can't parse TTF font", err)
		return
	}
	opts.FontFamily = font.Name(truetype.NameIDFontFamily)

	downloaded := false
	for _, novelURL := range args {
		_, e := url.Parse(novelURL)
		if e != nil {
			fmt.Println("invalid URL", novelURL)
			continue
		}
		for _, h := range novelSiteHandlers {
			for _, pattern := range h.MatchPatterns {
				r, _ := regexp.Compile(pattern)
				if r.MatchString(novelURL) {
					gen = ebook.NewBook(opts.Format)
					gen.SetFontSize(opts.TitleFontSize, opts.ContentFontSize)
					gen.SetLineSpacing(opts.LineSpacing)
					gen.PagesPerFile(opts.PagesPerFile)
					gen.ChaptersPerFile(opts.ChaptersPerFile)
					gen.SetMargins(opts.LeftMargin, opts.TopMargin)
					gen.SetPageType(opts.PageType)
					gen.SetFontFamily(opts.FontFamily)
					gen.SetFontFile(opts.FontFile)
					gen.Info()
					h.Download(novelURL)
					downloaded = true
				}
			}
		}
	}
	if !downloaded {
		fmt.Println("使用方法：\n\tgetnovel 小说目录网址")
		listCommandHandler()
	}
}
<commit_msg>(*)refactoring<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"regexp"
	"strings"

	"github.com/dfordsoft/golib/ebook"
	flags "github.com/jessevdk/go-flags"
)

// Options for all command line options
type Options struct {
	Format          string  `short:"f" long:"format" description:"set generated file format, candidate values: mobi, epub, pdf"`
	List            bool    `short:"l" long:"list" description:"list supported novel websites"`
	LeftMargin      float64 `long:"leftMargin" description:"set left margin for PDF format"`
	TopMargin       float64 `long:"topMargin" description:"set top margin for PDF format"`
	PageType        string  `short:"p" long:"pageType" description:"set page type for PDF format, candidate values: a0, a1, a2, a3, a4, a5, a6, b0, b1, b2, b3, b4, b5, b6, c0, c1, c2, c3, c4, c5, c6, dxg(=a4), 6inch(90mm x 117mm), 7inch, 10inch(=a4), pc(=a4 & 25.4mm left margin & 31.7mm top margin & 16 point title font size & 12 point content font size)"`
	TitleFontSize   int     `long:"titleFontSize" description:"set title font point size for PDF format"`
	ContentFontSize int     `long:"contentFontSize" description:"set content font point size for PDF format"`
	LineSpacing     float64 `long:"lineSpacing" description:"set line spacing rate for PDF format"`
	PagesPerFile    int     `long:"pagesPerFile" description:"split the big single PDF file to several smaller PDF files, how many pages should be included in a file, 0 means don't split"`
	ChaptersPerFile int     `long:"chaptersPerFile" description:"split the big signle PDF file to several smaller PDF files, how many chapters should be included in a file, 0 means don't split"`
	FontFile        string  `long:"fontFile" description:"set TTF font file path"`
	RetryCount      int     `short:"r" long:"retries" description:"download retry count"`
	Timeout         int     `short:"t" long:"timeout" description:"download timeout seconds"`
	ParallelCount   int64   `long:"parallel" description:"parallel count for downloading"`
	ConfigFile      string  `short:"c" long:"config" description:"read configurations from local file"`
	OutputFile      string  `short:"o" long:"output" description:"output file path"`
}

type tocPattern struct {
	host            string
	bookTitle       string
	bookTitlePos    int
	item            string
	articleTitlePos int
	articleURLPos   int
	isAbsoluteURL   bool
}

type pageContentMarker struct {
	host  string
	start []byte
	end   []byte
}

type novelSiteHandler struct {
	Title         string
	MatchPatterns []string
	Download      func(string)
}

var (
	novelSiteHandlers []*novelSiteHandler
	gen               ebook.IBook
	opts              Options
)

func registerNovelSiteHandler(h *novelSiteHandler) {
	novelSiteHandlers = append(novelSiteHandlers, h)
}

func listCommandHandler() {
	fmt.Println("支持小说网站：")
	for _, h := range novelSiteHandlers {
		urlMap := make(map[string]struct{})
		for _, p := range h.MatchPatterns {
			u := strings.Replace(p, `\`, ``, -1)
			idxStart := strings.Index(u, `www.`)
			idxEnd := strings.Index(u[idxStart:], `/`)
			u = u[:idxStart+idxEnd]
			urlMap[u] = struct{}{}
		}
		var urls []string
		for u := range urlMap {
			urls = append(urls, u)
		}
		fmt.Println("\t" + h.Title + ": " + strings.Join(urls, ", "))
	}
}

func readConfigFile(opts *Options) bool {
	if opts.ConfigFile != "" {
		contentFd, err := os.OpenFile(opts.ConfigFile, os.O_RDONLY, 0644)
		if err != nil {
			log.Println("opening config file ", opts.ConfigFile, " for reading failed ", err)
			return false
		}

		contentC, err := ioutil.ReadAll(contentFd)
		if err != nil {
			log.Println("reading config file ", opts.ConfigFile, " failed ", err)
			return false
		}
		contentFd.Close()

		var options map[string]interface{}
		if err = json.Unmarshal(contentC, &options); err != nil {
			log.Println("unmarshall configurations failed", err)
			return false
		}

		if f, ok := options["format"]; ok {
			if v := f.(string); len(v) > 0 {
				opts.Format = v
			}
		}
		if f, ok := options["pageType"]; ok {
			if v := f.(string); len(v) > 0 {
				opts.PageType = v
			}
		}
		if f, ok := options["fontFile"]; ok {
			if v := f.(string); len(v) > 0 {
				opts.FontFile = v
			}
		}

		if f, ok := options["leftMargin"]; ok {
			if v := f.(float64); v > 0 {
				opts.LeftMargin = v
			}
		}
		if f, ok := options["topMargin"]; ok {
			if v := f.(float64); v > 0 {
				opts.TopMargin = v
			}
		}
		if f, ok := options["lineSpacing"]; ok {
			if v := f.(float64); v > 0 {
				opts.LineSpacing = v
			}
		}
		if f, ok := options["titleFontSize"]; ok {
			if v := f.(int); v > 0 {
				opts.TitleFontSize = v
			}
		}
		if f, ok := options["contentFontSize"]; ok {
			if v := f.(int); v > 0 {
				opts.ContentFontSize = v
			}
		}
		if f, ok := options["pagesPerFile"]; ok {
			if v := f.(int); v > 0 {
				opts.PagesPerFile = v
			}
		}
		if f, ok := options["chaptersPerFile"]; ok {
			if v := f.(int); v > 0 {
				opts.ChaptersPerFile = v
			}
		}
		if f, ok := options["retries"]; ok {
			if v := f.(int); v > 0 {
				opts.RetryCount = v
			}
		}
		if f, ok := options["timeout"]; ok {
			if v := f.(int); v > 0 {
				opts.Timeout = v
			}
		}
		if f, ok := options["parallel"]; ok {
			if v := f.(int64); v > 0 {
				opts.ParallelCount = v
			}
		}
	}
	return true
}

func main() {
	if len(os.Args) < 2 {
		fmt.Println("使用方法：\n\tgetnovel 小说目录网址")
		listCommandHandler()
		return
	}

	opts = Options{
		Format:          "mobi",
		List:            false,
		LeftMargin:      10,
		TopMargin:       10,
		PageType:        "a4",
		TitleFontSize:   24,
		ContentFontSize: 18,
		LineSpacing:     1.2,
		PagesPerFile:    0,
		ChaptersPerFile: 0,
		FontFile:        "fonts/CustomFont.ttf",
		RetryCount:      3,
		Timeout:         60,
		ParallelCount:   10,
	}

	args, err := flags.Parse(&opts)
	if err != nil {
		return
	}

	if opts.List {
		listCommandHandler()
		return
	}

	if !readConfigFile(&opts) {
		return
	}

	downloaded := false
	for _, novelURL := range args {
		_, e := url.Parse(novelURL)
		if e != nil {
			fmt.Println("invalid URL", novelURL)
			continue
		}
		for _, h := range novelSiteHandlers {
			for _, pattern := range h.MatchPatterns {
				r, _ := regexp.Compile(pattern)
				if r.MatchString(novelURL) {
					gen = ebook.NewBook(opts.Format)
					gen.SetFontSize(opts.TitleFontSize, opts.ContentFontSize)
					gen.SetLineSpacing(opts.LineSpacing)
					gen.PagesPerFile(opts.PagesPerFile)
					gen.ChaptersPerFile(opts.ChaptersPerFile)
					gen.SetMargins(opts.LeftMargin, opts.TopMargin)
					gen.SetPageType(opts.PageType)
					gen.SetFontFile(opts.FontFile)
					gen.Info()
					h.Download(novelURL)
					downloaded = true
				}
			}
		}
	}
	if !downloaded {
		fmt.Println("使用方法：\n\tgetnovel 小说目录网址")
		listCommandHandler()
	}
}
<|endoftext|>


<commit_before>// Package c2go contains the main function for running the executable.
//
// Installation
//
//     go get -u github.com/elliotchance/c2go
//
// Usage
//
//     c2go myfile.c
//
package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"runtime"
	"strings"

	"errors"

	"github.com/elliotchance/c2go/ast"
	"github.com/elliotchance/c2go/preprocessor"
	"github.com/elliotchance/c2go/program"
	"github.com/elliotchance/c2go/transpiler"
)

// Version can be requested through the command line with:
//
//     c2go -v
//
// See https://github.com/elliotchance/c2go/wiki/Release-Process
const Version = "v0.19.2 Uranium 2017-12-21"

var stderr io.Writer = os.Stderr

// ProgramArgs defines the options available when processing the program. There
// is no constructor since the zeroed out values are the appropriate defaults -
// you need only set the options you need.
//
// TODO: Better separation on CLI modes
// https://github.com/elliotchance/c2go/issues/134
//
// Do not instantiate this directly. Instead use DefaultProgramArgs(); then
// modify any specific attributes.
type ProgramArgs struct {
	verbose     bool
	ast         bool
	inputFiles  []string
	clangFlags  []string
	outputFile  string
	packageName string

	// A private option to output the Go as a *_test.go file.
	outputAsTest bool
}

// DefaultProgramArgs default value of ProgramArgs
func DefaultProgramArgs() ProgramArgs {
	return ProgramArgs{
		verbose:      false,
		ast:          false,
		packageName:  "main",
		clangFlags:   []string{},
		outputAsTest: false,
	}
}

func readAST(data []byte) []string {
	return strings.Split(string(data), "\n")
}

type treeNode struct {
	indent int
	node   ast.Node
}

func convertLinesToNodes(lines []string) []treeNode {
	nodes := make([]treeNode, len(lines))
	var counter int
	for _, line := range lines {
		if strings.TrimSpace(line) == "" {
			continue
		}

		// It is tempting to discard null AST nodes, but these may
		// have semantic importance: for example, they represent omitted
		// for-loop conditions, as in for(;;).
		line = strings.Replace(line, "<<<NULL>>>", "NullStmt", 1)
		trimmed := strings.TrimLeft(line, "|\\- `")
		node := ast.Parse(trimmed)
		indentLevel := (len(line) - len(trimmed)) / 2
		nodes[counter] = treeNode{indentLevel, node}
		counter++
	}
	nodes = nodes[0:counter]

	return nodes
}

func convertLinesToNodesParallel(lines []string) []treeNode {
	// function f separate full list on 2 parts and
	// then each part can recursive run function f
	var f func([]string, int) []treeNode

	f = func(lines []string, deep int) []treeNode {
		deep = deep - 2
		part := len(lines) / 2

		var tr1 = make(chan []treeNode)
		var tr2 = make(chan []treeNode)

		go func(lines []string, deep int) {
			if deep <= 0 || len(lines) < deep {
				tr1 <- convertLinesToNodes(lines)
				return
			}
			tr1 <- f(lines, deep)
		}(lines[0:part], deep)

		go func(lines []string, deep int) {
			if deep <= 0 || len(lines) < deep {
				tr2 <- convertLinesToNodes(lines)
				return
			}
			tr2 <- f(lines, deep)
		}(lines[part:], deep)

		defer close(tr1)
		defer close(tr2)

		return append(<-tr1, <-tr2...)
	}

	// Parameter of deep - can be any, but effective to use
	// same amount of CPU
	return f(lines, runtime.NumCPU())
}

// buildTree converts an array of nodes, each prefixed with a depth into a tree.
func buildTree(nodes []treeNode, depth int) []ast.Node {
	if len(nodes) == 0 {
		return []ast.Node{}
	}

	// Split the list into sections, treat each section as a tree with its own
	// root.
	sections := [][]treeNode{}
	for _, node := range nodes {
		if node.indent == depth {
			sections = append(sections, []treeNode{node})
		} else {
			sections[len(sections)-1] = append(sections[len(sections)-1], node)
		}
	}

	results := []ast.Node{}
	for _, section := range sections {
		slice := []treeNode{}
		for _, n := range section {
			if n.indent > depth {
				slice = append(slice, n)
			}
		}

		children := buildTree(slice, depth+1)
		for _, child := range children {
			section[0].node.AddChild(child)
		}
		results = append(results, section[0].node)
	}

	return results
}

// Start begins transpiling an input file.
func Start(args ProgramArgs) (err error) {
	if args.verbose {
		fmt.Println("Start tanspiling ...")
	}

	if os.Getenv("GOPATH") == "" {
		return fmt.Errorf("The $GOPATH must be set")
	}

	// 1. Compile it first (checking for errors)
	for _, in := range args.inputFiles {
		_, err := os.Stat(in)
		if err != nil {
			return fmt.Errorf("Input file %s is not found", in)
		}
	}

	// 2. Preprocess
	if args.verbose {
		fmt.Println("Running clang preprocessor...")
	}

	pp, err := preprocessor.Analyze(args.inputFiles, args.clangFlags)
	if err != nil {
		return err
	}

	if args.verbose {
		fmt.Println("Writing preprocessor ...")
	}
	dir, err := ioutil.TempDir("", "c2go")
	if err != nil {
		return fmt.Errorf("Cannot create temp folder: %v", err)
	}
	defer os.RemoveAll(dir) // clean up

	ppFilePath := path.Join(dir, "pp.c")
	err = ioutil.WriteFile(ppFilePath, pp, 0644)
	if err != nil {
		return fmt.Errorf("writing to %s failed: %v", ppFilePath, err)
	}

	// 3. Generate JSON from AST
	if args.verbose {
		fmt.Println("Running clang for AST tree...")
	}
	astPP, err := exec.Command("clang", "-Xclang", "-ast-dump", "-fsyntax-only", "-fno-color-diagnostics", ppFilePath).Output()
	if err != nil {
		// If clang fails it still prints out the AST, so we have to run it
		// again to get the real error.
		errBody, _ := exec.Command("clang", ppFilePath).CombinedOutput()

		panic("clang failed: " + err.Error() + ":\n\n" + string(errBody))
	}

	if args.verbose {
		fmt.Println("Reading clang AST tree...")
	}
	lines := readAST(astPP)
	if args.ast {
		for _, l := range lines {
			fmt.Println(l)
		}
		fmt.Println()

		return nil
	}

	p := program.NewProgram()
	p.Verbose = args.verbose
	p.OutputAsTest = args.outputAsTest

	// Converting to nodes
	if args.verbose {
		fmt.Println("Converting to nodes...")
	}
	nodes := convertLinesToNodesParallel(lines)

	// build tree
	if args.verbose {
		fmt.Println("Building tree...")
	}
	tree := buildTree(nodes, 0)
	ast.FixPositions(tree)

	// Repair the floating literals. See RepairFloatingLiteralsFromSource for
	// more information.
	floatingErrors := ast.RepairFloatingLiteralsFromSource(tree[0], ppFilePath)

	for _, fErr := range floatingErrors {
		message := fmt.Sprintf("could not read exact floating literal: %s",
			fErr.Err.Error())
		p.AddMessage(p.GenerateWarningMessage(errors.New(message), fErr.Node))
	}

	outputFilePath := args.outputFile

	if outputFilePath == "" {
		// Choose inputFile for creating name of output file
		input := args.inputFiles[0]
		// We choose name for output Go code at the base
		// on filename for choosed input file
		cleanFileName := filepath.Clean(filepath.Base(input))
		extension := filepath.Ext(input)
		outputFilePath = cleanFileName[0:len(cleanFileName)-len(extension)] + ".go"
	}

	// transpile ast tree
	if args.verbose {
		fmt.Println("Transpiling tree...")
	}

	err = transpiler.TranspileAST(args.outputFile, args.packageName, p, tree[0].(ast.Node))
	if err != nil {
		return fmt.Errorf("cannot transpile AST : %v", err)
	}

	// write the output Go code
	if args.verbose {
		fmt.Println("Writing the output Go code...")
	}
	err = ioutil.WriteFile(outputFilePath, []byte(p.String()), 0644)
	if err != nil {
		return fmt.Errorf("writing Go output file failed: %v", err)
	}

	return nil
}

type inputDataFlags []string

func (i *inputDataFlags) String() (s string) {
	for pos, item := range *i {
		s += fmt.Sprintf("Flag %d. %s\n", pos, item)
	}
	return
}

func (i *inputDataFlags) Set(value string) error {
	*i = append(*i, value)
	return nil
}

var clangFlags inputDataFlags

func init() {
	transpileCommand.Var(&clangFlags, "clang-flag", "Pass arguments to clang. You may provide multiple -clang-flag items.")
}

var (
	versionFlag       = flag.Bool("v", false, "print the version and exit")
	transpileCommand  = flag.NewFlagSet("transpile", flag.ContinueOnError)
	verboseFlag       = transpileCommand.Bool("V", false, "print progress as comments")
	outputFlag        = transpileCommand.String("o", "", "output Go generated code to the specified file")
	packageFlag       = transpileCommand.String("p", "main", "set the name of the generated package")
	transpileHelpFlag = transpileCommand.Bool("h", false, "print help information")
	astCommand        = flag.NewFlagSet("ast", flag.ContinueOnError)
	astHelpFlag       = astCommand.Bool("h", false, "print help information")
)

func main() {
	code := runCommand()
	if code != 0 {
		os.Exit(code)
	}
}

func runCommand() int {

	flag.Usage = func() {
		usage := "Usage: %s [-v] [<command>] [<flags>] file1.c ...\n\n"
		usage += "Commands:\n"
		usage += "  transpile\ttranspile an input C source file or files to Go\n"
		usage += "  ast\t\tprint AST before translated Go code\n\n"

		usage += "Flags:\n"
		fmt.Fprintf(stderr, usage, os.Args[0])
		flag.PrintDefaults()
	}

	transpileCommand.SetOutput(stderr)
	astCommand.SetOutput(stderr)

	flag.Parse()

	if *versionFlag {
		// Simply print out the version and exit.
		fmt.Println(Version)
		return 0
	}

	if flag.NArg() < 1 {
		flag.Usage()
		return 1
	}

	args := DefaultProgramArgs()

	switch os.Args[1] {
	case "ast":
		err := astCommand.Parse(os.Args[2:])
		if err != nil {
			fmt.Printf("ast command cannot parse: %v", err)
			return 1
		}

		if *astHelpFlag || astCommand.NArg() == 0 {
			fmt.Fprintf(stderr, "Usage: %s ast file.c\n", os.Args[0])
			astCommand.PrintDefaults()
			return 1
		}

		args.ast = true
		args.inputFiles = astCommand.Args()
	case "transpile":
		err := transpileCommand.Parse(os.Args[2:])
		if err != nil {
			fmt.Printf("transpile command cannot parse: %v", err)
			return 1
		}

		if *transpileHelpFlag || transpileCommand.NArg() == 0 {
			fmt.Fprintf(stderr, "Usage: %s transpile [-V] [-o file.go] [-p package] file1.c ...\n", os.Args[0])
			transpileCommand.PrintDefaults()
			return 1
		}

		args.inputFiles = transpileCommand.Args()
		args.outputFile = *outputFlag
		args.packageName = *packageFlag
		args.verbose = *verboseFlag
		args.clangFlags = clangFlags
	default:
		flag.Usage()
		return 1
	}

	if err := Start(args); err != nil {
		fmt.Printf("Error: %v\n", err)
		return 1
	}

	return 0
}
<commit_msg>Bump version: v0.19.3 Uranium 2017-12-26<commit_after>// Package c2go contains the main function for running the executable.
//
// Installation
//
//     go get -u github.com/elliotchance/c2go
//
// Usage
//
//     c2go myfile.c
//
package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"runtime"
	"strings"

	"errors"

	"github.com/elliotchance/c2go/ast"
	"github.com/elliotchance/c2go/preprocessor"
	"github.com/elliotchance/c2go/program"
	"github.com/elliotchance/c2go/transpiler"
)

// Version can be requested through the command line with:
//
//     c2go -v
//
// See https://github.com/elliotchance/c2go/wiki/Release-Process
const Version = "v0.19.3 Uranium 2017-12-26"

var stderr io.Writer = os.Stderr

// ProgramArgs defines the options available when processing the program. There
// is no constructor since the zeroed out values are the appropriate defaults -
// you need only set the options you need.
//
// TODO: Better separation on CLI modes
// https://github.com/elliotchance/c2go/issues/134
//
// Do not instantiate this directly. Instead use DefaultProgramArgs(); then
// modify any specific attributes.
type ProgramArgs struct {
	verbose     bool
	ast         bool
	inputFiles  []string
	clangFlags  []string
	outputFile  string
	packageName string

	// A private option to output the Go as a *_test.go file.
	outputAsTest bool
}

// DefaultProgramArgs default value of ProgramArgs
func DefaultProgramArgs() ProgramArgs {
	return ProgramArgs{
		verbose:      false,
		ast:          false,
		packageName:  "main",
		clangFlags:   []string{},
		outputAsTest: false,
	}
}

func readAST(data []byte) []string {
	return strings.Split(string(data), "\n")
}

type treeNode struct {
	indent int
	node   ast.Node
}

func convertLinesToNodes(lines []string) []treeNode {
	nodes := make([]treeNode, len(lines))
	var counter int
	for _, line := range lines {
		if strings.TrimSpace(line) == "" {
			continue
		}

		// It is tempting to discard null AST nodes, but these may
		// have semantic importance: for example, they represent omitted
		// for-loop conditions, as in for(;;).
		line = strings.Replace(line, "<<<NULL>>>", "NullStmt", 1)
		trimmed := strings.TrimLeft(line, "|\\- `")
		node := ast.Parse(trimmed)
		indentLevel := (len(line) - len(trimmed)) / 2
		nodes[counter] = treeNode{indentLevel, node}
		counter++
	}
	nodes = nodes[0:counter]

	return nodes
}

func convertLinesToNodesParallel(lines []string) []treeNode {
	// function f separate full list on 2 parts and
	// then each part can recursive run function f
	var f func([]string, int) []treeNode

	f = func(lines []string, deep int) []treeNode {
		deep = deep - 2
		part := len(lines) / 2

		var tr1 = make(chan []treeNode)
		var tr2 = make(chan []treeNode)

		go func(lines []string, deep int) {
			if deep <= 0 || len(lines) < deep {
				tr1 <- convertLinesToNodes(lines)
				return
			}
			tr1 <- f(lines, deep)
		}(lines[0:part], deep)

		go func(lines []string, deep int) {
			if deep <= 0 || len(lines) < deep {
				tr2 <- convertLinesToNodes(lines)
				return
			}
			tr2 <- f(lines, deep)
		}(lines[part:], deep)

		defer close(tr1)
		defer close(tr2)

		return append(<-tr1, <-tr2...)
	}

	// Parameter of deep - can be any, but effective to use
	// same amount of CPU
	return f(lines, runtime.NumCPU())
}

// buildTree converts an array of nodes, each prefixed with a depth into a tree.
func buildTree(nodes []treeNode, depth int) []ast.Node {
	if len(nodes) == 0 {
		return []ast.Node{}
	}

	// Split the list into sections, treat each section as a tree with its own
	// root.
	sections := [][]treeNode{}
	for _, node := range nodes {
		if node.indent == depth {
			sections = append(sections, []treeNode{node})
		} else {
			sections[len(sections)-1] = append(sections[len(sections)-1], node)
		}
	}

	results := []ast.Node{}
	for _, section := range sections {
		slice := []treeNode{}
		for _, n := range section {
			if n.indent > depth {
				slice = append(slice, n)
			}
		}

		children := buildTree(slice, depth+1)
		for _, child := range children {
			section[0].node.AddChild(child)
		}
		results = append(results, section[0].node)
	}

	return results
}

// Start begins transpiling an input file.
func Start(args ProgramArgs) (err error) {
	if args.verbose {
		fmt.Println("Start tanspiling ...")
	}

	if os.Getenv("GOPATH") == "" {
		return fmt.Errorf("The $GOPATH must be set")
	}

	// 1. Compile it first (checking for errors)
	for _, in := range args.inputFiles {
		_, err := os.Stat(in)
		if err != nil {
			return fmt.Errorf("Input file %s is not found", in)
		}
	}

	// 2. Preprocess
	if args.verbose {
		fmt.Println("Running clang preprocessor...")
	}

	pp, err := preprocessor.Analyze(args.inputFiles, args.clangFlags)
	if err != nil {
		return err
	}

	if args.verbose {
		fmt.Println("Writing preprocessor ...")
	}
	dir, err := ioutil.TempDir("", "c2go")
	if err != nil {
		return fmt.Errorf("Cannot create temp folder: %v", err)
	}
	defer os.RemoveAll(dir) // clean up

	ppFilePath := path.Join(dir, "pp.c")
	err = ioutil.WriteFile(ppFilePath, pp, 0644)
	if err != nil {
		return fmt.Errorf("writing to %s failed: %v", ppFilePath, err)
	}

	// 3. Generate JSON from AST
	if args.verbose {
		fmt.Println("Running clang for AST tree...")
	}
	astPP, err := exec.Command("clang", "-Xclang", "-ast-dump", "-fsyntax-only", "-fno-color-diagnostics", ppFilePath).Output()
	if err != nil {
		// If clang fails it still prints out the AST, so we have to run it
		// again to get the real error.
		errBody, _ := exec.Command("clang", ppFilePath).CombinedOutput()

		panic("clang failed: " + err.Error() + ":\n\n" + string(errBody))
	}

	if args.verbose {
		fmt.Println("Reading clang AST tree...")
	}
	lines := readAST(astPP)
	if args.ast {
		for _, l := range lines {
			fmt.Println(l)
		}
		fmt.Println()

		return nil
	}

	p := program.NewProgram()
	p.Verbose = args.verbose
	p.OutputAsTest = args.outputAsTest

	// Converting to nodes
	if args.verbose {
		fmt.Println("Converting to nodes...")
	}
	nodes := convertLinesToNodesParallel(lines)

	// build tree
	if args.verbose {
		fmt.Println("Building tree...")
	}
	tree := buildTree(nodes, 0)
	ast.FixPositions(tree)

	// Repair the floating literals. See RepairFloatingLiteralsFromSource for
	// more information.
	floatingErrors := ast.RepairFloatingLiteralsFromSource(tree[0], ppFilePath)

	for _, fErr := range floatingErrors {
		message := fmt.Sprintf("could not read exact floating literal: %s",
			fErr.Err.Error())
		p.AddMessage(p.GenerateWarningMessage(errors.New(message), fErr.Node))
	}

	outputFilePath := args.outputFile

	if outputFilePath == "" {
		// Choose inputFile for creating name of output file
		input := args.inputFiles[0]
		// We choose name for output Go code at the base
		// on filename for choosed input file
		cleanFileName := filepath.Clean(filepath.Base(input))
		extension := filepath.Ext(input)
		outputFilePath = cleanFileName[0:len(cleanFileName)-len(extension)] + ".go"
	}

	// transpile ast tree
	if args.verbose {
		fmt.Println("Transpiling tree...")
	}

	err = transpiler.TranspileAST(args.outputFile, args.packageName, p, tree[0].(ast.Node))
	if err != nil {
		return fmt.Errorf("cannot transpile AST : %v", err)
	}

	// write the output Go code
	if args.verbose {
		fmt.Println("Writing the output Go code...")
	}
	err = ioutil.WriteFile(outputFilePath, []byte(p.String()), 0644)
	if err != nil {
		return fmt.Errorf("writing Go output file failed: %v", err)
	}

	return nil
}

type inputDataFlags []string

func (i *inputDataFlags) String() (s string) {
	for pos, item := range *i {
		s += fmt.Sprintf("Flag %d. %s\n", pos, item)
	}
	return
}

func (i *inputDataFlags) Set(value string) error {
	*i = append(*i, value)
	return nil
}

var clangFlags inputDataFlags

func init() {
	transpileCommand.Var(&clangFlags, "clang-flag", "Pass arguments to clang. You may provide multiple -clang-flag items.")
}

var (
	versionFlag       = flag.Bool("v", false, "print the version and exit")
	transpileCommand  = flag.NewFlagSet("transpile", flag.ContinueOnError)
	verboseFlag       = transpileCommand.Bool("V", false, "print progress as comments")
	outputFlag        = transpileCommand.String("o", "", "output Go generated code to the specified file")
	packageFlag       = transpileCommand.String("p", "main", "set the name of the generated package")
	transpileHelpFlag = transpileCommand.Bool("h", false, "print help information")
	astCommand        = flag.NewFlagSet("ast", flag.ContinueOnError)
	astHelpFlag       = astCommand.Bool("h", false, "print help information")
)

func main() {
	code := runCommand()
	if code != 0 {
		os.Exit(code)
	}
}

func runCommand() int {

	flag.Usage = func() {
		usage := "Usage: %s [-v] [<command>] [<flags>] file1.c ...\n\n"
		usage += "Commands:\n"
		usage += "  transpile\ttranspile an input C source file or files to Go\n"
		usage += "  ast\t\tprint AST before translated Go code\n\n"

		usage += "Flags:\n"
		fmt.Fprintf(stderr, usage, os.Args[0])
		flag.PrintDefaults()
	}

	transpileCommand.SetOutput(stderr)
	astCommand.SetOutput(stderr)

	flag.Parse()

	if *versionFlag {
		// Simply print out the version and exit.
		fmt.Println(Version)
		return 0
	}

	if flag.NArg() < 1 {
		flag.Usage()
		return 1
	}

	args := DefaultProgramArgs()

	switch os.Args[1] {
	case "ast":
		err := astCommand.Parse(os.Args[2:])
		if err != nil {
			fmt.Printf("ast command cannot parse: %v", err)
			return 1
		}

		if *astHelpFlag || astCommand.NArg() == 0 {
			fmt.Fprintf(stderr, "Usage: %s ast file.c\n", os.Args[0])
			astCommand.PrintDefaults()
			return 1
		}

		args.ast = true
		args.inputFiles = astCommand.Args()
	case "transpile":
		err := transpileCommand.Parse(os.Args[2:])
		if err != nil {
			fmt.Printf("transpile command cannot parse: %v", err)
			return 1
		}

		if *transpileHelpFlag || transpileCommand.NArg() == 0 {
			fmt.Fprintf(stderr, "Usage: %s transpile [-V] [-o file.go] [-p package] file1.c ...\n", os.Args[0])
			transpileCommand.PrintDefaults()
			return 1
		}

		args.inputFiles = transpileCommand.Args()
		args.outputFile = *outputFlag
		args.packageName = *packageFlag
		args.verbose = *verboseFlag
		args.clangFlags = clangFlags
	default:
		flag.Usage()
		return 1
	}

	if err := Start(args); err != nil {
		fmt.Printf("Error: %v\n", err)
		return 1
	}

	return 0
}
<|endoftext|>