<commit_before>package main

import (
	"net/http"
	"os"

	"github.com/gempir/gempbot/internal/auth"
	"github.com/gempir/gempbot/internal/bot"
	"github.com/gempir/gempbot/internal/channelpoint"
	"github.com/gempir/gempbot/internal/config"
	"github.com/gempir/gempbot/internal/emotechief"
	"github.com/gempir/gempbot/internal/emoteservice"
	"github.com/gempir/gempbot/internal/eventsub"
	"github.com/gempir/gempbot/internal/helixclient"
	"github.com/gempir/gempbot/internal/log"
	"github.com/gempir/gempbot/internal/server"
	"github.com/gempir/gempbot/internal/store"
	"github.com/gempir/gempbot/internal/user"
	"github.com/gempir/gempbot/internal/ws"
	"github.com/rs/cors"
)

func main() {
	cfg := config.FromEnv()
	db := store.NewDatabase(cfg)

	argsWithoutProg := os.Args[1:]
	if len(argsWithoutProg) == 1 && argsWithoutProg[0] == "migrate" {
		db.Migrate()
		os.Exit(0)
		return
	}

	os.Exit(0)

	helixClient := helixclient.NewClient(cfg, db)
	go helixClient.StartRefreshTokenRoutine()

	userAdmin := user.NewUserAdmin(cfg, db, helixClient, nil)
	authClient := auth.NewAuth(cfg, db, helixClient)

	bot := bot.NewBot(cfg, db, helixClient)
	go bot.Connect()

	seventvClient := emoteservice.NewSevenTvClient(db)

	emoteChief := emotechief.NewEmoteChief(cfg, db, helixClient, bot.ChatClient, seventvClient)
	eventsubManager := eventsub.NewEventsubManager(cfg, helixClient, db, emoteChief, bot.ChatClient)
	eventsubSubscriptionManager := eventsub.NewSubscriptionManager(cfg, db, helixClient)
	channelPointManager := channelpoint.NewChannelPointManager(cfg, helixClient, db)

	wsHandler := ws.NewWsHandler(authClient)

	apiHandlers := server.NewApi(cfg, db, helixClient, userAdmin, authClient, bot, emoteChief, eventsubManager, eventsubSubscriptionManager, channelPointManager, seventvClient, wsHandler)

	mux := http.NewServeMux()

	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		_, _ = w.Write([]byte("<img src=\"data:image/webp;base64,UklGRk4LAABXRUJQVlA4WAoAAAAQAAAASgAASgAAQUxQSG0DAAABoLZtkyHJlv9G5B7btm3btm3btm3btm3btu0zqoj3w3RnVWVMREwAWhdg2DTb3HrbNfc8/uDVJ++1/AxjAkiCiAJZ5Pz3/2fX3u/efeTUlUcFVMpTjLLrVyS9Mzv2P7fr+ICWljDhFX10c3Z1NzOSL20+FkQKEk0Y92bSWNvNOXDXfICUIgpg0rvozkaN/GC7MaBlCKAz7/4W3dmwO/svHRdagiAtcvl3pLFFI68YF9qaCKY95RvSja06edkY0JYEstz9Fc3ZttPPHhXSimLErd8inQU6/WRAWlCMdeg3dGeRzr6toM0Jxji/l85SjT8uj9SUIJ1KOss1vjY5UmNb/Utjyc4zh4M2olj+OzoL+28LDGtCMPbzNBZu/HB6DNeAYme6l0bnGYL6gknepDHAN3NBayn2ozOg8VLUFoz3Li2Cs2duaI2ETWkMWfEIpBqCm1jFML41EiRLMcvf9BjO3jmgWQk7s2JQ46ZIWcC9cSqekKeY+W96nFuhOQlr0RnnAUiO4iRaGOPLI0C6CYZ7klWgr8fJm+gjWhjn75NDc+b9mx7IFspRLF8FonM5pG4JqwdbK29HOuNW3CdvT1qoo4eSy4aSK/L2CHblUHI1NGfXYFfl7RjsCqScTYJdnreC0SNdmqOY9adYB2FYN8GYb9EibZ4DwU2RjOsiZSTsFsjpC2Qp5viTHuf3yaAZgmEP0aIYPx4FkoGE3eJUvBuCXMWUX9ODGPdHyoLiLFoMZ/8i0LyEJXroIYzvjQXJE6R7aCEqXgxFTcWCv9MjODdEqgPBBSGMn0wMqaWY/2t6hHMhaPLUAM6ehaENKGb5nF6a8blhkAYg2KOHXtyeSGhm1HtoZRm/mATSCBI2GKAXVfFEKAZLLcHoz9AKcTPS+dtMHaSGAEDCxmVY5Rxc8VwoOtTXYViVXoCT/PuDj2n8YaZOTSbgJFp7zkf2WmlqrGzs2QqK2pIUognY6S96exV3BLDQ6+QZgNQR0aQAMNZZvXS27+w9d88n+sinxoGiqwwSVQwec7JVHiGdBX8+JxSZokkApMlXOPjmV74wurNA50Or7nbxTTcfMRMUmQIA4yx97AOf9XGwsUjjPugsyB5hqg2v/WSAJM3MnGU6314egCRF9q0v/UySZuYs2fnfzcsK6pL0ypzFG/nrUtAaAFZQOCC6BwAAkCEAnQEqSwBLAD4xEodCoiEMLYsqEAGCWMAuVMmBkyLK8/j95hNt2tz1NuB5j/Nm9InkzejN7L3oNdLX5N2aAf0Dta/u3gv4XfZvuDyEol/av+z4d9sTfEQB7s/2A6H/Dbj7/6Xqf6HXqX2B/1b/4/Yh9DP9WnDcUFps+qBiO8Psh83ftVnEPTtqhcTWtyxZTOaUCyk8o8bnZ5E7JPJ33+vXn4Ypi4Iwb2JiPFmHaAsVL6K+FiEYVOnwqJkxe0d19Q9enVcm/nx5lsPTGBfr7p2Cv+6cTJ0wwCcgZgWQ1lK8zHzUtBGqupRCkNGZJsIj44zM89QznNhCjaEET+IhU6vCLcOiabrOG60hHy/1X6Bkp/lgAAD+/7kOaP/F9RJ7XdWUAD+9ubasj4fv8mDsDXmv4jmd+0rNsp7wPbQOO5cigFdRd+F9a9amR9FX70UzQcfxrB2iejTaIQ84B343d3cL1/qE74HvC7DHpqfbk0d9KZw5q0FEyECLvmTni+u7l0F65S3CLwf8rFfvG7fEKRNBf//TheuFiaA9+Oitfw5r+Yi6XPCJYqLfDK1qGZ/a8hVUuem0O7L+smpJ7HBdGEhbgoZmXBsk5FDTJcuQjILsoJl1wnUvC+ZgxFZ3Y1LLM5xl4GhGEvdjUWM3bkjdBUnwGxzWBSCUfBZCKTnhrSlpHhaFwYQ4wwb794gYwvBQGUmckjRi2lhtC6WIdLIKcq//Ang7L13zTW9vyFq291iP6N3MJ55FvCvTjPuwK/xLO6bYWm49U/9oEzPeqv/wMLvd9/zMx+VsgmMWLJRKMyoT/diA7MoQssyYxb8N6lXH2aih9/TCCpgtaBuMEcBHuyjv/znDaMzUPeAN0SYnd9122LWXWn3uJ78fM/jtffCXWVRfwhN6TQFG0OJ/g+z+Cy3HijEuflchfEry3qm1/HPnRd/eiu9uYczPsMOuz5cVYoHzZU7XOjysYqoa91cJ1r+cN170ReE6iYWbNGN5c8hR8YwugKPmhwc/Cu0E9v8x691QBMcXfulWFQn7WXCQNMi+mc6fnE3NiW2h6Bnllxhjhulv/kPX80DMrVXHJiboegcy5IJofjr/WsBVweRJ//V2vh+OWGaI5CHeVKedOaPpTsUqG/UGUAmwZG8RIIoU6pEWUVGIKCeUrAi/y77XJFjcQ9KecsLN0rQWhZTuuw1lDBXhlPRXead2JFun5Ya0o2KgIwRvpU8+9om46x/waADb0f3nx+ccROnXV/VTNyBIFOwjy2pyE66aituP6JSYcSG/msNC8OK80gqzazZ0KvsPOSKON4tJFL6Vv/0bcPRM+mJ8PMAs/5ZRFg3E78kLrYklLTYj5DUn6Gom//xLhFpwDYBLbFPXzmpmBB+Q/H8Lt0Z2ars0dh06F3hZr9AEL0tVsXkDn0I2oiXuN0jD6u+kn82eW4OqQEoPP9d9s7lIPv2UhWAcSzHKSth3dnW0DOtcrnZt3rhOPeGEtphO6mGw3ehCeQGiAy2qaMfZlJKxFcKkXGZ/gj4N5nMF3SLXvkzG0bfmPRhbu32g3yg4jW9Tu1E6TK3bCgP+Im9c0iSjTQPb/7c7peH5/q6+WPPD2jKkyK+RDLnGHnDgTJiQzU7uDYMSfZjSEAs+zBmj7LM+14aVEsH++ZasHz74yiAwIMHRUHkAmBQa8L39hxU23KDXRPOKNOqztCZ186FsrI5OO2CwOxOBpyEDRpOHZec5RcK5QDINahXSE07kX2cb7bczkDozDXt51fMQSpMgLXvPfzAdcSsJWteOYTOf7ViiQBgwmy33kGHlkxPR4Lh/p2cYp0WP5XIvdDPQmHjUUaO9SMH5a61ezb+3BuTVpsmJdval5bUcM5p08sCwHxzXj/Y5GVIqGcZCXeRu2vfSu0C8NX9nN3Dw+iBJ3DLfh9A/qm85aVyFhKSCwJSucrNHY4zvznmXXKifhlnDXO9gsTm0X05dPsbDoxPgRyT/IgGvfzH+DMF0mtoGY/BHakUzOVsgluNbVUUPplPtxzNQ/Yrral404M0g+QLu5XmU+ULuMAZ2G3rgk3Yvhtnzo3iQBgrCUmEjUUKEUweatHRXb+0nYWEF8FNJzAxUj7UPPxQwqsKj1GWw4RXn138A7yzbmOjTsj4coEWgtU+ctzl/AZJUoV1nuHZ8F63gfIjNZpPoIhdGVDOXJUPXhEMXkvyLpSUyFmSEKbe20g4N1a78GAy5kWo25ZNJUC/C3QMNhVL5qYGgqZYE70FFxxA2Fqg8lYAf0aWWcLg9N2AEJNv/hDwQM9APYu4KAGD9jzVvl7U2oeWAUmxqpeCbh3zLvrspN3QEKMk/PoEmvpQVhoR6qXFgf+h9OqnEnPOegYCeR6x0H8zi4cnOdQmqgRyHknn9Yyytj52hyxcqYACIfutFcK3khwjtfmDdMr9TIu4RkS/x3fGv2rNQ5T4XUg4M9NKZ5EURWAic222yrYg1nQK1nzTZjGDyXGmT8YlkKtdsnYzLssVP4huhLhaK77xcVP0BNYMeJHr4TcWgwFzOIDXZ3FpzG/FiTeKXcBo63uEwENAH47qQouDnaIAjiB+Jg2QE1gf4GuPebn+WiJirfb4Ci30T3p9zrMnffxsAG+eiTiCGU6xS2A/mocAAAA==\" />"))
		w.WriteHeader(http.StatusOK)
	})
	mux.HandleFunc("/api/blocks", apiHandlers.BlocksHandler)
	mux.HandleFunc("/api/botconfig", apiHandlers.BotConfigHandler)
	mux.HandleFunc("/api/callback", apiHandlers.CallbackHandler)
	mux.HandleFunc("/api/emotehistory", apiHandlers.EmoteHistoryHandler)
	mux.HandleFunc("/api/eventsub", apiHandlers.EventSubHandler)
	mux.HandleFunc("/api/reward", apiHandlers.RewardHandler)
	mux.HandleFunc("/api/subscriptions", apiHandlers.SubscriptionsHandler)
	mux.HandleFunc("/api/userconfig", apiHandlers.UserConfigHandler)
	mux.HandleFunc("/api/ws", wsHandler.HandleWs)

	handler := cors.New(cors.Options{
		AllowedOrigins:   []string{cfg.WebBaseUrl},
		AllowedMethods:   []string{"GET", "POST", "PUT", "DELETE", "PATCH"},
		AllowedHeaders:   []string{"*"},
		AllowCredentials: true,
	}).Handler(mux)
	err := http.ListenAndServe(":3010", handler)
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>prevent reboot<commit_after>package main

import (
	"net/http"
	"os"

	"github.com/gempir/gempbot/internal/auth"
	"github.com/gempir/gempbot/internal/bot"
	"github.com/gempir/gempbot/internal/channelpoint"
	"github.com/gempir/gempbot/internal/config"
	"github.com/gempir/gempbot/internal/emotechief"
	"github.com/gempir/gempbot/internal/emoteservice"
	"github.com/gempir/gempbot/internal/eventsub"
	"github.com/gempir/gempbot/internal/helixclient"
	"github.com/gempir/gempbot/internal/log"
	"github.com/gempir/gempbot/internal/server"
	"github.com/gempir/gempbot/internal/store"
	"github.com/gempir/gempbot/internal/user"
	"github.com/gempir/gempbot/internal/ws"
	"github.com/rs/cors"
)

func main() {
	cfg := config.FromEnv()
	db := store.NewDatabase(cfg)

	argsWithoutProg := os.Args[1:]
	if len(argsWithoutProg) == 1 && argsWithoutProg[0] == "migrate" {
		db.Migrate()
		os.Exit(0)
		return
	}

	helixClient := helixclient.NewClient(cfg, db)
	go helixClient.StartRefreshTokenRoutine()

	userAdmin := user.NewUserAdmin(cfg, db, helixClient, nil)
	authClient := auth.NewAuth(cfg, db, helixClient)

	bot := bot.NewBot(cfg, db, helixClient)
	go bot.Connect()

	seventvClient := emoteservice.NewSevenTvClient(db)

	emoteChief := emotechief.NewEmoteChief(cfg, db, helixClient, bot.ChatClient, seventvClient)
	eventsubManager := eventsub.NewEventsubManager(cfg, helixClient, db, emoteChief, bot.ChatClient)
	eventsubSubscriptionManager := eventsub.NewSubscriptionManager(cfg, db, helixClient)
	channelPointManager := channelpoint.NewChannelPointManager(cfg, helixClient, db)

	wsHandler := ws.NewWsHandler(authClient)

	apiHandlers := server.NewApi(cfg, db, helixClient, userAdmin, authClient, bot, emoteChief, eventsubManager, eventsubSubscriptionManager, channelPointManager, seventvClient, wsHandler)

	mux := http.NewServeMux()

	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		_, _ = w.Write([]byte("<img src=\"data:image/webp;base64,UklGRk4LAABXRUJQVlA4WAoAAAAQAAAASgAASgAAQUxQSG0DAAABoLZtkyHJlv9G5B7btm3btm3btm3btm3btu0zqoj3w3RnVWVMREwAWhdg2DTb3HrbNfc8/uDVJ++1/AxjAkiCiAJZ5Pz3/2fX3u/efeTUlUcFVMpTjLLrVyS9Mzv2P7fr+ICWljDhFX10c3Z1NzOSL20+FkQKEk0Y92bSWNvNOXDXfICUIgpg0rvozkaN/GC7MaBlCKAz7/4W3dmwO/svHRdagiAtcvl3pLFFI68YF9qaCKY95RvSja06edkY0JYEstz9Fc3ZttPPHhXSimLErd8inQU6/WRAWlCMdeg3dGeRzr6toM0Jxji/l85SjT8uj9SUIJ1KOss1vjY5UmNb/Utjyc4zh4M2olj+OzoL+28LDGtCMPbzNBZu/HB6DNeAYme6l0bnGYL6gknepDHAN3NBayn2ozOg8VLUFoz3Li2Cs2duaI2ETWkMWfEIpBqCm1jFML41EiRLMcvf9BjO3jmgWQk7s2JQ46ZIWcC9cSqekKeY+W96nFuhOQlr0RnnAUiO4iRaGOPLI0C6CYZ7klWgr8fJm+gjWhjn75NDc+b9mx7IFspRLF8FonM5pG4JqwdbK29HOuNW3CdvT1qoo4eSy4aSK/L2CHblUHI1NGfXYFfl7RjsCqScTYJdnreC0SNdmqOY9adYB2FYN8GYb9EibZ4DwU2RjOsiZSTsFsjpC2Qp5viTHuf3yaAZgmEP0aIYPx4FkoGE3eJUvBuCXMWUX9ODGPdHyoLiLFoMZ/8i0LyEJXroIYzvjQXJE6R7aCEqXgxFTcWCv9MjODdEqgPBBSGMn0wMqaWY/2t6hHMhaPLUAM6ehaENKGb5nF6a8blhkAYg2KOHXtyeSGhm1HtoZRm/mATSCBI2GKAXVfFEKAZLLcHoz9AKcTPS+dtMHaSGAEDCxmVY5Rxc8VwoOtTXYViVXoCT/PuDj2n8YaZOTSbgJFp7zkf2WmlqrGzs2QqK2pIUognY6S96exV3BLDQ6+QZgNQR0aQAMNZZvXS27+w9d88n+sinxoGiqwwSVQwec7JVHiGdBX8+JxSZokkApMlXOPjmV74wurNA50Or7nbxTTcfMRMUmQIA4yx97AOf9XGwsUjjPugsyB5hqg2v/WSAJM3MnGU6314egCRF9q0v/UySZuYs2fnfzcsK6pL0ypzFG/nrUtAaAFZQOCC6BwAAkCEAnQEqSwBLAD4xEodCoiEMLYsqEAGCWMAuVMmBkyLK8/j95hNt2tz1NuB5j/Nm9InkzejN7L3oNdLX5N2aAf0Dta/u3gv4XfZvuDyEol/av+z4d9sTfEQB7s/2A6H/Dbj7/6Xqf6HXqX2B/1b/4/Yh9DP9WnDcUFps+qBiO8Psh83ftVnEPTtqhcTWtyxZTOaUCyk8o8bnZ5E7JPJ33+vXn4Ypi4Iwb2JiPFmHaAsVL6K+FiEYVOnwqJkxe0d19Q9enVcm/nx5lsPTGBfr7p2Cv+6cTJ0wwCcgZgWQ1lK8zHzUtBGqupRCkNGZJsIj44zM89QznNhCjaEET+IhU6vCLcOiabrOG60hHy/1X6Bkp/lgAAD+/7kOaP/F9RJ7XdWUAD+9ubasj4fv8mDsDXmv4jmd+0rNsp7wPbQOO5cigFdRd+F9a9amR9FX70UzQcfxrB2iejTaIQ84B343d3cL1/qE74HvC7DHpqfbk0d9KZw5q0FEyECLvmTni+u7l0F65S3CLwf8rFfvG7fEKRNBf//TheuFiaA9+Oitfw5r+Yi6XPCJYqLfDK1qGZ/a8hVUuem0O7L+smpJ7HBdGEhbgoZmXBsk5FDTJcuQjILsoJl1wnUvC+ZgxFZ3Y1LLM5xl4GhGEvdjUWM3bkjdBUnwGxzWBSCUfBZCKTnhrSlpHhaFwYQ4wwb794gYwvBQGUmckjRi2lhtC6WIdLIKcq//Ang7L13zTW9vyFq291iP6N3MJ55FvCvTjPuwK/xLO6bYWm49U/9oEzPeqv/wMLvd9/zMx+VsgmMWLJRKMyoT/diA7MoQssyYxb8N6lXH2aih9/TCCpgtaBuMEcBHuyjv/znDaMzUPeAN0SYnd9122LWXWn3uJ78fM/jtffCXWVRfwhN6TQFG0OJ/g+z+Cy3HijEuflchfEry3qm1/HPnRd/eiu9uYczPsMOuz5cVYoHzZU7XOjysYqoa91cJ1r+cN170ReE6iYWbNGN5c8hR8YwugKPmhwc/Cu0E9v8x691QBMcXfulWFQn7WXCQNMi+mc6fnE3NiW2h6Bnllxhjhulv/kPX80DMrVXHJiboegcy5IJofjr/WsBVweRJ//V2vh+OWGaI5CHeVKedOaPpTsUqG/UGUAmwZG8RIIoU6pEWUVGIKCeUrAi/y77XJFjcQ9KecsLN0rQWhZTuuw1lDBXhlPRXead2JFun5Ya0o2KgIwRvpU8+9om46x/waADb0f3nx+ccROnXV/VTNyBIFOwjy2pyE66aituP6JSYcSG/msNC8OK80gqzazZ0KvsPOSKON4tJFL6Vv/0bcPRM+mJ8PMAs/5ZRFg3E78kLrYklLTYj5DUn6Gom//xLhFpwDYBLbFPXzmpmBB+Q/H8Lt0Z2ars0dh06F3hZr9AEL0tVsXkDn0I2oiXuN0jD6u+kn82eW4OqQEoPP9d9s7lIPv2UhWAcSzHKSth3dnW0DOtcrnZt3rhOPeGEtphO6mGw3ehCeQGiAy2qaMfZlJKxFcKkXGZ/gj4N5nMF3SLXvkzG0bfmPRhbu32g3yg4jW9Tu1E6TK3bCgP+Im9c0iSjTQPb/7c7peH5/q6+WPPD2jKkyK+RDLnGHnDgTJiQzU7uDYMSfZjSEAs+zBmj7LM+14aVEsH++ZasHz74yiAwIMHRUHkAmBQa8L39hxU23KDXRPOKNOqztCZ186FsrI5OO2CwOxOBpyEDRpOHZec5RcK5QDINahXSE07kX2cb7bczkDozDXt51fMQSpMgLXvPfzAdcSsJWteOYTOf7ViiQBgwmy33kGHlkxPR4Lh/p2cYp0WP5XIvdDPQmHjUUaO9SMH5a61ezb+3BuTVpsmJdval5bUcM5p08sCwHxzXj/Y5GVIqGcZCXeRu2vfSu0C8NX9nN3Dw+iBJ3DLfh9A/qm85aVyFhKSCwJSucrNHY4zvznmXXKifhlnDXO9gsTm0X05dPsbDoxPgRyT/IgGvfzH+DMF0mtoGY/BHakUzOVsgluNbVUUPplPtxzNQ/Yrral404M0g+QLu5XmU+ULuMAZ2G3rgk3Yvhtnzo3iQBgrCUmEjUUKEUweatHRXb+0nYWEF8FNJzAxUj7UPPxQwqsKj1GWw4RXn138A7yzbmOjTsj4coEWgtU+ctzl/AZJUoV1nuHZ8F63gfIjNZpPoIhdGVDOXJUPXhEMXkvyLpSUyFmSEKbe20g4N1a78GAy5kWo25ZNJUC/C3QMNhVL5qYGgqZYE70FFxxA2Fqg8lYAf0aWWcLg9N2AEJNv/hDwQM9APYu4KAGD9jzVvl7U2oeWAUmxqpeCbh3zLvrspN3QEKMk/PoEmvpQVhoR6qXFgf+h9OqnEnPOegYCeR6x0H8zi4cnOdQmqgRyHknn9Yyytj52hyxcqYACIfutFcK3khwjtfmDdMr9TIu4RkS/x3fGv2rNQ5T4XUg4M9NKZ5EURWAic222yrYg1nQK1nzTZjGDyXGmT8YlkKtdsnYzLssVP4huhLhaK77xcVP0BNYMeJHr4TcWgwFzOIDXZ3FpzG/FiTeKXcBo63uEwENAH47qQouDnaIAjiB+Jg2QE1gf4GuPebn+WiJirfb4Ci30T3p9zrMnffxsAG+eiTiCGU6xS2A/mocAAAA==\" />"))
		w.WriteHeader(http.StatusOK)
	})
	mux.HandleFunc("/api/blocks", apiHandlers.BlocksHandler)
	mux.HandleFunc("/api/botconfig", apiHandlers.BotConfigHandler)
	mux.HandleFunc("/api/callback", apiHandlers.CallbackHandler)
	mux.HandleFunc("/api/emotehistory", apiHandlers.EmoteHistoryHandler)
	mux.HandleFunc("/api/eventsub", apiHandlers.EventSubHandler)
	mux.HandleFunc("/api/reward", apiHandlers.RewardHandler)
	mux.HandleFunc("/api/subscriptions", apiHandlers.SubscriptionsHandler)
	mux.HandleFunc("/api/userconfig", apiHandlers.UserConfigHandler)
	mux.HandleFunc("/api/ws", wsHandler.HandleWs)

	handler := cors.New(cors.Options{
		AllowedOrigins:   []string{cfg.WebBaseUrl},
		AllowedMethods:   []string{"GET", "POST", "PUT", "DELETE", "PATCH"},
		AllowedHeaders:   []string{"*"},
		AllowCredentials: true,
	}).Handler(mux)
	err := http.ListenAndServe(":3010", handler)
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/vaughan0/go-ini"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"

	//	"errors"
	//	"log"
	//	"net"
	//	"crypto/sha512"
	//	"encoding/hex"
)

//	defaults:
var (
	WORKERS 	int64	= 10    //Number of workers
	IMGDIR  	string	= "img" //default download directory
	TAG     	string	= ""    //default string is empty, it can only ge extracted from command line
	STARTPAGE	int		= 1		//default start page, derpiboo.ru 1-indexed
	STOPPAGE	int		= 0		//default stop page, would stop parsing json when it ends
	)

func main() {

	fmt.Println("Derpiboo.ru Downloader version 0.0.5 \nWorking")

	config, err := ini.LoadFile("config.ini") // Loading default config file and checking for various errors.
	if os.IsNotExist(err) {
		panic("config.ini does not exist, create it")
	}

	if err != nil {
		panic(err)
	}

	//Getting stuff from config, overwriting defaults

	key, ok := config.Get("main", "key")
	if !ok {
		panic("'key' variable missing from 'main' section")
	}

	W_temp, _ := config.Get("main", "workers")
	if W_temp != "" {
		WORKERS, err = strconv.ParseInt(W_temp, 10, 0)
		if err != nil {
			fmt.Println("Wrong configuration: Amount of workers is not a number")
			os.Exit(1)
		}
	}

	ID_temp, _ := config.Get("main", "downdir")
	if ID_temp != "" {
		IMGDIR = ID_temp
	}

	//here shall be flag parser

	flag.StringVar(&TAG, "t", TAG, "Tag to download: Replace spaces with \"+\".")
	flag.Parse()

	length := flag.NArg()
	if length == 0 {
		fmt.Println("Nothing to download, bye!")
		os.Exit(0)
	}

	//	creating directory for downloads if not yet done
	if err := os.MkdirAll(IMGDIR, 0777); err != nil {
		panic(err)
	}

	imgdat := make(chan Image, 10)

	if TAG == "" {

		imgid := flag.Arg(length - 1)

		//	fmt.Println(key) //Just checking that I am not wrong

		_, err = strconv.ParseInt(imgid, 10, 0)
		if err != nil {
			fmt.Println("Wrong input: can not parse", imgid, "as a number")
			os.Exit(1)
		}

		fmt.Println("Processing image No " + imgid)

		parseImg(imgdat, imgid, key)

	}	else	{
	//	and here we parse tag and abuse stuff horribly

		fmt.Println("trying to process tags")
		parseTag(imgdat, TAG, key)
	}

	dlimage(imgdat)

}

type Image struct {
	url      string
	filename string
	hash     string
}

func parseImg(imgchan chan<- Image, imgid string, key string) {

	source := "http://derpiboo.ru/" + imgid + ".json?nofav=&nocomments="
	if key != "" {
		source = source + "&key=" + key
	}

	//	fmt.Println(source)

	resp, err := http.Get(source) //Getting our nice http response. Needs checking for 404 and other responses that are... less expected
	if err != nil {
		panic(err)
	}

	defer resp.Body.Close() //and not forgetting to close it when it's done

	var dat map[string]interface{}

	body, err := ioutil.ReadAll(resp.Body) //stolen from official documentation
	if err != nil {
		panic(err)
	}

	if err := json.Unmarshal(body, &dat); //transforming json into native map

	err != nil {
		panic(err)
	}
	var imgdata Image
	imgdata.url = "http:" + dat["image"].(string)
	imgdata.hash = dat["sha512_hash"].(string) //for the future and checking that we got file right
	imgdata.filename = strconv.FormatFloat(dat["id_number"].(float64), 'f', -1, 64) + "." + dat["file_name"].(string) + "." + dat["original_format"].(string)

	//	fmt.Println(strconv.FormatFloat(dat["id_number"].(float64), 'f', -1, 64))

	//	fmt.Println(dat)

	//	for now and troubleshooting
	//	fmt.Println(imgdata.url)
	//	fmt.Println(imgdata.hash)
	//	fmt.Println(imgdata.filename)

	imgchan <- imgdata

	return
}

func dlimage(imgchan <-chan Image) {
	//	fmt.Println("reading channel")

	imgdata := <-imgchan

	fmt.Println("Saving as ", imgdata.filename)
	PathSep, _ := strconv.Unquote(strconv.QuoteRune(os.PathSeparator))

	output, err := os.Create(IMGDIR + PathSep + imgdata.filename)
	if err != err {
		panic(err)
	}
	defer output.Close()

	response, err := http.Get(imgdata.url)
	if err != nil {
		panic(err)
		fmt.Println("Error while downloading", imgdata.url, "-", err)
		return
	}
	defer response.Body.Close()

	//	hash := sha512.New()

	io.Copy(output, response.Body)

	/*	io.Copy(hash, response.Body)
		b := make([]byte, hash.Size())
		hash.Sum(b[:0])

		fmt.Println("\n", hex.EncodeToString(b), "\n", imgdata.hash )

		if hex.EncodeToString(b) == imgdata.hash {
			fmt.Println("Hash correct")
		}	else {
			fmt.Println("Hash wrong")
		}

		fmt.Println("\n", hex.EncodeToString(hash.Sum(nil)), "\n", imgdata.hash )
	*/

}

func parseTag(imgchan chan<- Image, tag string, key string) {

}
<commit_msg>Added attempt to parse json from search query. It's horribly borked<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/vaughan0/go-ini"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"

	//	"errors"
	//	"log"
	//	"net"
	//	"crypto/sha512"
	//	"encoding/hex"
)

//	defaults:
var (
	WORKERS 	int64	= 10    //Number of workers
	IMGDIR  	string	= "img" //default download directory
	TAG     	string	= ""    //default string is empty, it can only ge extracted from command line
	STARTPAGE	int		= 1		//default start page, derpiboo.ru 1-indexed
	STOPPAGE	int		= 0		//default stop page, would stop parsing json when it ends
	)

func main() {

	fmt.Println("Derpiboo.ru Downloader version 0.0.5 \nWorking")

	config, err := ini.LoadFile("config.ini") // Loading default config file and checking for various errors.
	if os.IsNotExist(err) {
		panic("config.ini does not exist, create it")
	}

	if err != nil {
		panic(err)
	}

	//Getting stuff from config, overwriting defaults

	key, ok := config.Get("main", "key")
	if !ok {
		panic("'key' variable missing from 'main' section")
	}

	W_temp, _ := config.Get("main", "workers")
	if W_temp != "" {
		WORKERS, err = strconv.ParseInt(W_temp, 10, 0)
		if err != nil {
			fmt.Println("Wrong configuration: Amount of workers is not a number")
			os.Exit(1)
		}
	}

	ID_temp, _ := config.Get("main", "downdir")
	if ID_temp != "" {
		IMGDIR = ID_temp
	}

	//here shall be flag parser

	flag.StringVar(&TAG, "t", TAG, "Tag to download: Replace spaces with \"+\".")
	flag.Parse()

	length := flag.NArg()
	if length == 0 && TAG == "" {
		fmt.Println("Nothing to download, bye!")
		os.Exit(0)
	}

	//	creating directory for downloads if not yet done
	if err := os.MkdirAll(IMGDIR, 0777); err != nil {
		panic(err)
	}

	imgdat := make(chan Image, WORKERS)

	if TAG == "" {

		imgid := flag.Arg(length - 1)

		//	fmt.Println(key) //Just checking that I am not wrong

		_, err = strconv.ParseInt(imgid, 10, 0)
		if err != nil {
			fmt.Println("Wrong input: can not parse", imgid, "as a number")
			os.Exit(1)
		}

		fmt.Println("Processing image No " + imgid)

		parseImg(imgdat, imgid, key)

	}	else	{
	//	and here we parse tag and abuse stuff horribly

		fmt.Println("trying to process tags")
		parseTag(imgdat, TAG, key)
	}

	dlimage(imgdat)

}

type Image struct {
	url      string
	filename string
	hash     string
}

func parseImg(imgchan chan<- Image, imgid string, key string) {

	source := "http://derpiboo.ru/" + imgid + ".json?nofav=&nocomments="
	if key != "" {
		source = source + "&key=" + key
	}

	//	fmt.Println(source)

	resp, err := http.Get(source) //Getting our nice http response. Needs checking for 404 and other responses that are... less expected
	if err != nil {
		panic(err)
	}

	defer resp.Body.Close() //and not forgetting to close it when it's done

	var dat map[string]interface{}

	body, err := ioutil.ReadAll(resp.Body) //stolen from official documentation
	if err != nil {
		panic(err)
	}

	fmt.Println(body)

	if err := json.Unmarshal(body, &dat); //transforming json into native map

	err != nil {
		panic(err)
	}
	var imgdata Image
	imgdata.url = "http:" + dat["image"].(string)
	imgdata.hash = dat["sha512_hash"].(string) //for the future and checking that we got file right
	imgdata.filename = strconv.FormatFloat(dat["id_number"].(float64), 'f', -1, 64) + "." + dat["file_name"].(string) + "." + dat["original_format"].(string)

	//	fmt.Println(strconv.FormatFloat(dat["id_number"].(float64), 'f', -1, 64))

	//	fmt.Println(dat)

	//	for now and troubleshooting
	//	fmt.Println(imgdata.url)
	//	fmt.Println(imgdata.hash)
	//	fmt.Println(imgdata.filename)

	imgchan <- imgdata

	return
}

func dlimage(imgchan <-chan Image) {
	//	fmt.Println("reading channel")

	imgdata := <-imgchan

	fmt.Println("Saving as ", imgdata.filename)
	PathSep, _ := strconv.Unquote(strconv.QuoteRune(os.PathSeparator))

	output, err := os.Create(IMGDIR + PathSep + imgdata.filename)
	if err != err {
		panic(err)
	}
	defer output.Close()

	response, err := http.Get(imgdata.url)
	if err != nil {
		panic(err)
		fmt.Println("Error while downloading", imgdata.url, "-", err)
		return
	}
	defer response.Body.Close()

	io.Copy(output, response.Body)

	/*	hash := sha512.New()
		io.Copy(hash, response.Body)
		b := make([]byte, hash.Size())
		hash.Sum(b[:0])

		fmt.Println("\n", hex.EncodeToString(b), "\n", imgdata.hash )

		if hex.EncodeToString(b) == imgdata.hash {
			fmt.Println("Hash correct")
		}	else {
			fmt.Println("Hash wrong")
		}

		fmt.Println("\n", hex.EncodeToString(hash.Sum(nil)), "\n", imgdata.hash )
	*/

}

func parseTag(imgchan chan<- Image, tag string, key string) {

	if tag == "" { fmt.Println("Something has gone horribly wrong, no tag found?"); os.Exit(1) }
	source := "http://derpiboo.ru/search.json?nofav=&nocomments=&utf8=false"
	if key != "" { source = source + "&key=" + key }

	fmt.Println(source + "&q=" + tag)

	resp, err := http.Get(source + "&q=" + tag) //Getting our nice http response. Needs checking for 404 and other responses that are... less expected
	if err != nil {
		panic(err)
	}

	defer resp.Body.Close() //and not forgetting to close it when it's done

	var dat map[string]interface{}

	//fmt.Println(resp)

	body, err := ioutil.ReadAll(resp.Body) //stolen from official documentation
	if err != nil {
		panic(err)
	}

	//fmt.Println(body)

	if err := json.Unmarshal(body, &dat); //transforming json into native map

	err != nil {
		panic(err)

	}
	fmt.Println(dat)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"errors"
	"flag"
	"fmt"
	"log"
	"os"
	"os/user"
	"path/filepath"
	"strings"

	"github.com/alyu/configparser"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/iam"
	"github.com/aws/aws-sdk-go/service/sts"
)

func main() {

	sourceProfile := flag.String("i", "default", "Source Profile")
	targetProfile := flag.String("t", "default", "Destination Profile")
	rotateKeys := flag.Bool("rotate-identity-keys", false, "Boolean flag to rotate keys")
	overwrite := flag.Bool("o", false, "Boolean flag to overwrite profile")
	credFile := flag.String("c", filepath.Join(getCredentialPath(), ".aws", "credentials"), "Full path to credentials file")
	duration := flag.Int64("d", 28800, "Token Duration")
	flag.Parse()

	if sourceProfile == targetProfile && !*overwrite {
		fmt.Println("Source equals target and will overwrite it you probably don't want to do this")
		return
	}
	//Get Current Credentials
	exists, err := checkProfileExists(credFile, sourceProfile)
	if err != nil || !exists {
		fmt.Println(err.Error())
		return
	}
	sess := CreateSession(sourceProfile)

	user, err := getUserMFA(sess)
	if err != nil {
		fmt.Println(err.Error())
		return
	}

	//Get MFA Code
	mfa, err := getMFACode()
	if err != nil {
		fmt.Println(err.Error())
		return
	}
	tempCreds, err := getSTSCredentials(sess, mfa, duration, user)
	if err != nil {
		fmt.Println(err.Error())
		return
	}
	writeNewProfile(credFile, targetProfile, sourceProfile, tempCreds)

	if *rotateKeys {
		newKeys, err := rotateCredentialKeys(sess)
		if err != nil {
			fmt.Println(err.Error())
			return
		}
		writeNewKeys(credFile, sourceProfile, newKeys)
	}
}

func getMFACode() (string, error) {
	var mfa string
	fmt.Print("Enter MFA Token: ")
	reader := bufio.NewReader(os.Stdin)
	mfa, err := reader.ReadString('\n')
	if err != nil {
		return mfa, errors.New("failed to get token")
	}
	return strings.TrimSpace(mfa), nil
}

//CreateSession Creates AWS Session with specified profile
func CreateSession(profileName *string) *session.Session {
	profileNameValue := *profileName
	sess := session.Must(session.NewSessionWithOptions(session.Options{
		Profile: profileNameValue,
	}))
	return sess
}

func getUserMFA(sess *session.Session) (*string, error) {
	var newToken *string

	svc := iam.New(sess)

	params := &iam.GetUserInput{}
	resp, err := svc.GetUser(params)

	if err != nil {
		// Print the error, cast err to awserr.Error to get the Code and
		// Message from an error.
		fmt.Println(err.Error())
		return newToken, errors.New("failed to Fetch User")
	}
	userName := *resp.User.UserName
	mfaparams := &iam.ListMFADevicesInput{
		MaxItems: aws.Int64(1),
		UserName: aws.String(userName),
	}
	mfaresp, err := svc.ListMFADevices(mfaparams)

	if err != nil {
		// Print the error, cast err to awserr.Error to get the Code and
		// Message from an error.
		fmt.Println(err.Error())
		return newToken, errors.New("failed to Fetch User")
	}

	if len(mfaresp.MFADevices) == 0 {
		return nil, errors.New("unable to find a MFA device for this identity")
	}

	return mfaresp.MFADevices[0].SerialNumber, nil
}

func getCredentialPath() string {
	usr, err := user.Current()
	if err != nil {
		log.Fatal(err)
	}
	return usr.HomeDir
}

func writeNewProfile(credFile *string, profileName *string, sourceProfile *string, sessionDetails *sts.GetSessionTokenOutput) {
	config, err := configparser.Read(*credFile)
	sourceSection, err := config.Section(*sourceProfile)
	region := sourceSection.ValueOf("region")
	section, err := config.Section(*profileName)
	if err != nil {
		section = config.NewSection(*profileName)
	}
	section.Add("region", region)
	section.Add("aws_access_key_id", *sessionDetails.Credentials.AccessKeyId)
	section.Add("aws_secret_access_key", *sessionDetails.Credentials.SecretAccessKey)
	section.Add("aws_session_token", *sessionDetails.Credentials.SessionToken)
	section.Add("awsmfa_expiration", (*sessionDetails.Credentials.Expiration).String())
	err = configparser.Save(config, *credFile)
	if err != nil {
		log.Fatal(err)
	}
}

func writeNewKeys(credFile *string, profileName *string, newKeys *iam.CreateAccessKeyOutput) {
	config, err := configparser.Read(*credFile)
	sourceSection, err := config.Section(*profileName)
	region := sourceSection.ValueOf("region")
	section, err := config.Section(*profileName)
	if err != nil {
		section = config.NewSection(*profileName)
	}
	section.Add("region", region)
	section.Add("aws_access_key_id", *newKeys.AccessKey.AccessKeyId)
	section.Add("aws_secret_access_key", *newKeys.AccessKey.SecretAccessKey)
	err = configparser.Save(config, *credFile)
	if err != nil {
		log.Fatal(err)
	}
}

func checkProfileExists(credFile *string, profileName *string) (bool, error) {
	config, err := configparser.Read(*credFile)
	if err != nil {
		fmt.Println("Could not find credentials file")
		fmt.Println(err.Error())
		return false, err
	}
	section, err := config.Section(*profileName)
	if err != nil {
		fmt.Println("Could not find profile in credentials file")
		return false, nil
	}
	if !section.Exists("aws_access_key_id") {
		fmt.Println("Could not find access key in profile")
		return false, nil
	}

	return true, nil
}

func getSTSCredentials(sess *session.Session, tokenCode string, duration *int64, device *string) (*sts.GetSessionTokenOutput, error) {
	svc := sts.New(sess)
	params := &sts.GetSessionTokenInput{
		DurationSeconds: aws.Int64(*duration),
		SerialNumber:    aws.String(*device),
		TokenCode:       aws.String(tokenCode),
	}
	resp, err := svc.GetSessionToken(params)

	if err != nil {
		return nil, err
	}
	return resp, nil
}

func rotateCredentialKeys(sess *session.Session) (*iam.CreateAccessKeyOutput, error) {
	svc := iam.New(sess)
	input := &iam.ListAccessKeysInput{}
	var currentAccessKey *iam.AccessKeyMetadata
	var createResult *iam.CreateAccessKeyOutput
	result, err := svc.ListAccessKeys(input)
	if err != nil {
		fmt.Println(err.Error())
		return nil, err
	}
	currentCreds, err := sess.Config.Credentials.Get()
	for _, accessKey := range result.AccessKeyMetadata {
		if *accessKey.AccessKeyId == currentCreds.AccessKeyID {
			currentAccessKey = accessKey
		}
	}
	if currentAccessKey != nil {
		deleteKeyInput := &iam.DeleteAccessKeyInput{
			AccessKeyId: currentAccessKey.AccessKeyId,
		}
		_, err := svc.DeleteAccessKey(deleteKeyInput)
		if err != nil {
			fmt.Println(err.Error())
			return nil, err
		}
		createKeyInput := &iam.CreateAccessKeyInput{}
		createResult, err = svc.CreateAccessKey(createKeyInput)
		if err != nil {
			fmt.Println(err.Error())
			return nil, err
		}
		fmt.Printf("Replacing %s with %s", currentCreds.AccessKeyID, *createResult.AccessKey.AccessKeyId)
		return createResult, nil
	}
	return nil, errors.New("unable to find a current access key for this Identity")
}
<commit_msg>Merging in master<commit_after>package main

import (
	"bufio"
	"errors"
	"flag"
	"fmt"
	"log"
	"os"
	"os/user"
	"path/filepath"
	"strings"

	"github.com/alyu/configparser"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/iam"
	"github.com/aws/aws-sdk-go/service/sts"
)

func main() {

	sourceProfile := flag.String("i", "default", "Source Profile")
	targetProfile := flag.String("t", "default", "Destination Profile")
	rotateKeys := flag.Bool("rotate-identity-keys", false, "Boolean flag to rotate keys")
	overwrite := flag.Bool("o", false, "Boolean flag to overwrite profile")
	credFile := flag.String("c", filepath.Join(getCredentialPath(), ".aws", "credentials"), "Full path to credentials file")
	duration := flag.Int64("d", 28800, "Token Duration")
	flag.Parse()

	if sourceProfile == targetProfile && !*overwrite {
		fmt.Println("Source equals target and will overwrite it you probably don't want to do this")
		return
	}
	//Get Current Credentials
	exists, err := checkProfileExists(credFile, sourceProfile)
	if err != nil || !exists {
		fmt.Println(err.Error())
		return
	}
	sess := CreateSession(sourceProfile)

	user, err := getUserMFA(sess)
	if err != nil {
		fmt.Println(err.Error())
		return
	}

	//Get MFA Code
	mfa, err := getMFACode()
	if err != nil {
		fmt.Println(err.Error())
		return
	}

	tempCreds := getSTSCredentials(sess, mfa, duration, user)
	if tempCreds != nil {
		writeNewProfile(credFile, targetProfile, sourceProfile, tempCreds)
	}
	if *rotateKeys {
		newKeys, err := rotateCredentialKeys(sess)
		if err != nil {
			fmt.Println(err.Error())
			return
		}
		writeNewKeys(credFile, sourceProfile, newKeys)
	}
}

func getMFACode() (string, error) {
	var mfa string
	fmt.Print("Enter MFA Token: ")
	reader := bufio.NewReader(os.Stdin)
	mfa, err := reader.ReadString('\n')
	if err != nil {
		return mfa, errors.New("failed to get token")
	}
	return strings.TrimSpace(mfa), nil
}

//CreateSession Creates AWS Session with specified profile
func CreateSession(profileName *string) *session.Session {
	profileNameValue := *profileName
	sess := session.Must(session.NewSessionWithOptions(session.Options{
		Profile: profileNameValue,
	}))
	return sess
}

func getUserMFA(sess *session.Session) (*string, error) {
	var newToken *string

	svc := iam.New(sess)

	params := &iam.GetUserInput{}
	resp, err := svc.GetUser(params)

	if err != nil {
		// Print the error, cast err to awserr.Error to get the Code and
		// Message from an error.
		fmt.Println(err.Error())
		return newToken, errors.New("failed to Fetch User")
	}
	userName := *resp.User.UserName
	mfaparams := &iam.ListMFADevicesInput{
		MaxItems: aws.Int64(1),
		UserName: aws.String(userName),
	}
	mfaresp, err := svc.ListMFADevices(mfaparams)

	if err != nil {
		// Print the error, cast err to awserr.Error to get the Code and
		// Message from an error.
		fmt.Println(err.Error())
		return newToken, errors.New("failed to Fetch User")
	}

	if len(mfaresp.MFADevices) == 0 {
		return nil, errors.New("unable to find a MFA device for this identity")
	}

	return mfaresp.MFADevices[0].SerialNumber, nil
}

func getCredentialPath() string {
	usr, err := user.Current()
	if err != nil {
		log.Fatal(err)
	}
	return usr.HomeDir
}

func writeNewProfile(credFile *string, profileName *string, sourceProfile *string, sessionDetails *sts.GetSessionTokenOutput) {
	config, err := configparser.Read(*credFile)
	sourceSection, err := config.Section(*sourceProfile)
	region := sourceSection.ValueOf("region")
	section, err := config.Section(*profileName)
	if err != nil {
		section = config.NewSection(*profileName)
	}
	section.Add("region", region)
	section.Add("aws_access_key_id", *sessionDetails.Credentials.AccessKeyId)
	section.Add("aws_secret_access_key", *sessionDetails.Credentials.SecretAccessKey)
	section.Add("aws_session_token", *sessionDetails.Credentials.SessionToken)
	section.Add("awsmfa_expiration", (*sessionDetails.Credentials.Expiration).String())
	err = configparser.Save(config, *credFile)
	if err != nil {
		log.Fatal(err)
	}
}

func writeNewKeys(credFile *string, profileName *string, newKeys *iam.CreateAccessKeyOutput) {
	config, err := configparser.Read(*credFile)
	sourceSection, err := config.Section(*profileName)
	region := sourceSection.ValueOf("region")
	section, err := config.Section(*profileName)
	if err != nil {
		section = config.NewSection(*profileName)
	}
	section.Add("region", region)
	section.Add("aws_access_key_id", *newKeys.AccessKey.AccessKeyId)
	section.Add("aws_secret_access_key", *newKeys.AccessKey.SecretAccessKey)
	err = configparser.Save(config, *credFile)
	if err != nil {
		log.Fatal(err)
	}
}

func checkProfileExists(credFile *string, profileName *string) (bool, error) {
	config, err := configparser.Read(*credFile)
	if err != nil {
		fmt.Println("Could not find credentials file")
		fmt.Println(err.Error())
		return false, err
	}
	section, err := config.Section(*profileName)
	if err != nil {
		fmt.Println("Could not find profile in credentials file")
		return false, nil
	}
	if !section.Exists("aws_access_key_id") {
		fmt.Println("Could not find access key in profile")
		return false, nil
	}

	return true, nil
}

func getSTSCredentials(sess *session.Session, tokenCode string, duration *int64, device *string) *sts.GetSessionTokenOutput {
	svc := sts.New(sess)
	params := &sts.GetSessionTokenInput{
		DurationSeconds: aws.Int64(*duration),
		SerialNumber:    aws.String(*device),
		TokenCode:       aws.String(tokenCode),
	}
	resp, err := svc.GetSessionToken(params)

	if err != nil {
		fmt.Println(err.Error())
		return nil
	}
	return resp
}

func rotateCredentialKeys(sess *session.Session) (*iam.CreateAccessKeyOutput, error) {
	svc := iam.New(sess)
	input := &iam.ListAccessKeysInput{}
	var currentAccessKey *iam.AccessKeyMetadata
	var createResult *iam.CreateAccessKeyOutput
	result, err := svc.ListAccessKeys(input)
	if err != nil {
		fmt.Println(err.Error())
		return nil, err
	}
	currentCreds, err := sess.Config.Credentials.Get()
	for _, accessKey := range result.AccessKeyMetadata {
		if *accessKey.AccessKeyId == currentCreds.AccessKeyID {
			currentAccessKey = accessKey
		}
	}
	if currentAccessKey != nil {
		deleteKeyInput := &iam.DeleteAccessKeyInput{
			AccessKeyId: currentAccessKey.AccessKeyId,
		}
		_, err := svc.DeleteAccessKey(deleteKeyInput)
		if err != nil {
			fmt.Println(err.Error())
			return nil, err
		}
		createKeyInput := &iam.CreateAccessKeyInput{}
		createResult, err = svc.CreateAccessKey(createKeyInput)
		if err != nil {
			fmt.Println(err.Error())
			return nil, err
		}
		fmt.Printf("Replacing %s with %s", currentCreds.AccessKeyID, *createResult.AccessKey.AccessKeyId)
		return createResult, nil
	}
	return nil, errors.New("unable to find a current access key for this Identity")
}
<|endoftext|>


<commit_before>// Copyright 2017, 2018 Liam Breck
// Published at https://github.com/networkimprov/mnm
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/

package main

import (
   "fmt"
   "io/ioutil"
   "encoding/json"
   "net"
   "os"
   pNtp "github.com/beevik/ntp"
   pQ "github.com/networkimprov/mnm/qlib"
   "os/signal"
   "strconv"
   "strings"
   "time"
   "crypto/tls"
)

const kVersionA, kVersionB, kVersionC = 0, 0, 0
const kVersionDate = "(unreleased)"
const kConfigFile = "mnm.config"

var sConfig tConfig

func main() { os.Exit(mainResult()) }

func mainResult() int {
   // return 2 reserved for use by Go internals
   var err error

   aTcNum := 0
   if len(os.Args) == 2 {
      aTcNum, err = strconv.Atoi(os.Args[1])
      if err != nil || aTcNum < 2 || aTcNum > 1000 {
         fmt.Fprintf(os.Stderr, "testclient count must be 2-1000\n")
         return 1
      }
   } else {
      err = sConfig.load()
      if err != nil {
         if !os.IsNotExist(err) {
            fmt.Fprintf(os.Stderr, "config load: %s\n", err.Error())
         } else {
            fmt.Fprintf(os.Stderr, "config load: %s missing; see mnm.conf for example\n", kConfigFile)
         }
         return 1
      }
   }

   fmt.Printf("mnm tmtp server v%d.%d.%d %s\nntp time %v\n",
              kVersionA, kVersionB, kVersionC, kVersionDate, sConfig.Ntp.time.UTC())

   aDbName := "userdb"; if aTcNum != 0 { aDbName += "-test-qlib" }
   pQ.UDb, err = NewUserDb(aDbName)
   if err != nil {
      fmt.Fprintf(os.Stderr, "%s\n", err.Error())
      return 1
   }

   aQstore := "qstore"; if aTcNum != 0 { aQstore += "-test" }
   pQ.Init(aQstore, sConfig.Ntp.time)

   if aTcNum != 0 {
      fmt.Printf("Starting Test Pass\n")
      if TestUserDb("userdb-test-unit") {
         pQ.LocalTest(aTcNum)
      }
      pQ.UDb.Erase()
   } else {
      err = startServer(&sConfig)
      if err != nil {
         fmt.Fprintf(os.Stderr, "server exit: %s\n", err.Error())
         return 1
      }
      fmt.Printf("server done\n")
   }

   return 0
}

type tConfig struct {
   Ntp struct {
      Hosts []string
      Retries uint8
      time time.Time
   }
   Listen struct {
      Net string
      Laddr string
      CertPath, KeyPath string
   }
}

func (o *tConfig) load() error {
   aBuf, err := ioutil.ReadFile(kConfigFile)
   if err != nil { return err }
   err = json.Unmarshal(aBuf, o)
   if err != nil { return err }

   for _, aHost := range o.Ntp.Hosts {
      for a := uint8(0); a < o.Ntp.Retries; a++ {
         o.Ntp.time, err = pNtp.Time(aHost)
         if err == nil {
            return nil
         }
         fmt.Fprintf(os.Stderr, "ntp site %s error: %s\n", aHost, err.Error())
         time.Sleep(time.Second / 2)
      }
   }
   return tError("ntp not available")
}

func startServer(iConf *tConfig) error {
   var err error
   aCfgTcp := net.ListenConfig{KeepAlive: -1}
   aListener, err := aCfgTcp.Listen(nil, iConf.Listen.Net, iConf.Listen.Laddr)
   if err != nil { return err }
   aCert, err := tls.LoadX509KeyPair(iConf.Listen.CertPath, iConf.Listen.KeyPath)
   if err != nil { return err }
   aCfgTls := tls.Config{Certificates: []tls.Certificate{aCert}}
   aListener = tls.NewListener(aListener, &aCfgTls)

   aIntWatch := make(chan os.Signal, 1)
   signal.Notify(aIntWatch, os.Interrupt)
   go func() {
      <-aIntWatch
      aListener.Close()
   }()

   var aConn net.Conn
   const kPauseMin, kPauseMax = time.Millisecond, time.Second
   aPause := kPauseMin
   for {
      aConn, err = aListener.Accept()
      if err != nil {
         if !err.(net.Error).Temporary() {
            pQ.Suspend()
            if strings.Contains(err.Error(), "use of closed network connection") {
               return nil
            }
            return err
         }
         if aPause > kPauseMax {
            aPause = kPauseMax
            fmt.Fprintf(os.Stderr, "listener recurring error %s\n", err.Error())
         }
         time.Sleep(aPause)
         aPause *= 2
         continue
      }
      aPause = kPauseMin
      pQ.NewLink(aConn)
   }
}

type tError string
func (o tError) Error() string { return string(o) }

<commit_msg>main: Add command-line flag -version<commit_after>// Copyright 2017, 2018 Liam Breck
// Published at https://github.com/networkimprov/mnm
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/

package main

import (
   "flag"
   "fmt"
   "io/ioutil"
   "encoding/json"
   "net"
   "os"
   pNtp "github.com/beevik/ntp"
   pQ "github.com/networkimprov/mnm/qlib"
   "os/signal"
   "strconv"
   "strings"
   "time"
   "crypto/tls"
)

const kVersionA, kVersionB, kVersionC = 0, 0, 0
const kVersionDate = "(unreleased)"
const kConfigFile = "mnm.config"

var sConfig tConfig

func main() {
   aVersionQuit := flag.Bool("version", false, "print version and quit")
   flag.Parse() // may os.Exit(2)
   if *aVersionQuit {
      fmt.Printf("mnm tmtp server v%d.%d.%d %s\n", kVersionA, kVersionB, kVersionC, kVersionDate)
      os.Exit(0)
   }
   os.Exit(mainResult())
}

func mainResult() int {
   // return 2 reserved for use by Go internals
   var err error

   aTcNum := 0
   if len(os.Args) == 2 {
      aTcNum, err = strconv.Atoi(os.Args[1])
      if err != nil || aTcNum < 2 || aTcNum > 1000 {
         fmt.Fprintf(os.Stderr, "testclient count must be 2-1000\n")
         return 1
      }
   } else {
      err = sConfig.load()
      if err != nil {
         if !os.IsNotExist(err) {
            fmt.Fprintf(os.Stderr, "config load: %s\n", err.Error())
         } else {
            fmt.Fprintf(os.Stderr, "config load: %s missing; see mnm.conf for example\n", kConfigFile)
         }
         return 1
      }
   }

   fmt.Printf("mnm tmtp server v%d.%d.%d %s\nntp time %v\n",
              kVersionA, kVersionB, kVersionC, kVersionDate, sConfig.Ntp.time.UTC())

   aDbName := "userdb"; if aTcNum != 0 { aDbName += "-test-qlib" }
   pQ.UDb, err = NewUserDb(aDbName)
   if err != nil {
      fmt.Fprintf(os.Stderr, "%s\n", err.Error())
      return 1
   }

   aQstore := "qstore"; if aTcNum != 0 { aQstore += "-test" }
   pQ.Init(aQstore, sConfig.Ntp.time)

   if aTcNum != 0 {
      fmt.Printf("Starting Test Pass\n")
      if TestUserDb("userdb-test-unit") {
         pQ.LocalTest(aTcNum)
      }
      pQ.UDb.Erase()
   } else {
      err = startServer(&sConfig)
      if err != nil {
         fmt.Fprintf(os.Stderr, "server exit: %s\n", err.Error())
         return 1
      }
      fmt.Printf("server done\n")
   }

   return 0
}

type tConfig struct {
   Ntp struct {
      Hosts []string
      Retries uint8
      time time.Time
   }
   Listen struct {
      Net string
      Laddr string
      CertPath, KeyPath string
   }
}

func (o *tConfig) load() error {
   aBuf, err := ioutil.ReadFile(kConfigFile)
   if err != nil { return err }
   err = json.Unmarshal(aBuf, o)
   if err != nil { return err }

   for _, aHost := range o.Ntp.Hosts {
      for a := uint8(0); a < o.Ntp.Retries; a++ {
         o.Ntp.time, err = pNtp.Time(aHost)
         if err == nil {
            return nil
         }
         fmt.Fprintf(os.Stderr, "ntp site %s error: %s\n", aHost, err.Error())
         time.Sleep(time.Second / 2)
      }
   }
   return tError("ntp not available")
}

func startServer(iConf *tConfig) error {
   var err error
   aCfgTcp := net.ListenConfig{KeepAlive: -1}
   aListener, err := aCfgTcp.Listen(nil, iConf.Listen.Net, iConf.Listen.Laddr)
   if err != nil { return err }
   aCert, err := tls.LoadX509KeyPair(iConf.Listen.CertPath, iConf.Listen.KeyPath)
   if err != nil { return err }
   aCfgTls := tls.Config{Certificates: []tls.Certificate{aCert}}
   aListener = tls.NewListener(aListener, &aCfgTls)

   aIntWatch := make(chan os.Signal, 1)
   signal.Notify(aIntWatch, os.Interrupt)
   go func() {
      <-aIntWatch
      aListener.Close()
   }()

   var aConn net.Conn
   const kPauseMin, kPauseMax = time.Millisecond, time.Second
   aPause := kPauseMin
   for {
      aConn, err = aListener.Accept()
      if err != nil {
         if !err.(net.Error).Temporary() {
            pQ.Suspend()
            if strings.Contains(err.Error(), "use of closed network connection") {
               return nil
            }
            return err
         }
         if aPause > kPauseMax {
            aPause = kPauseMax
            fmt.Fprintf(os.Stderr, "listener recurring error %s\n", err.Error())
         }
         time.Sleep(aPause)
         aPause *= 2
         continue
      }
      aPause = kPauseMin
      pQ.NewLink(aConn)
   }
}

type tError string
func (o tError) Error() string { return string(o) }

<|endoftext|>


<commit_before>// Package c2go contains the main function for running the executable.
//
// Installation
//
//     go get -u github.com/elliotchance/c2go
//
// Usage
//
//     c2go myfile.c
//
package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"strings"

	"errors"
	"reflect"

	"github.com/elliotchance/c2go/ast"
	"github.com/elliotchance/c2go/program"
	"github.com/elliotchance/c2go/transpiler"
)

// Version can be requested through the command line with:
//
//     c2go -v
//
// See https://github.com/elliotchance/c2go/wiki/Release-Process
const Version = "v0.16.2 Radium 2017-09-18"

var stderr io.Writer = os.Stderr

// ProgramArgs defines the options available when processing the program. There
// is no constructor since the zeroed out values are the appropriate defaults -
// you need only set the options you need.
//
// TODO: Better separation on CLI modes
// https://github.com/elliotchance/c2go/issues/134
type ProgramArgs struct {
	verbose     bool
	ast         bool
	inputFile   string
	outputFile  string
	packageName string

	// A private option to output the Go as a *_test.go file.
	outputAsTest bool
}

func readAST(data []byte) []string {
	uncolored := regexp.MustCompile(`\x1b\[[\d;]+m`).ReplaceAll(data, []byte{})
	return strings.Split(string(uncolored), "\n")
}

type treeNode struct {
	indent int
	node   ast.Node
}

func convertLinesToNodes(lines []string) []treeNode {
	nodes := make([]treeNode, len(lines))
	var counter int
	for _, line := range lines {
		if strings.TrimSpace(line) == "" {
			continue
		}

		// It is tempting to discard null AST nodes, but these may
		// have semantic importance: for example, they represent omitted
		// for-loop conditions, as in for(;;).
		line = strings.Replace(line, "<<<NULL>>>", "NullStmt", 1)
		trimmed := strings.TrimLeft(line, "|\\- `")
		node := ast.Parse(trimmed)
		indentLevel := (len(line) - len(trimmed)) / 2
		nodes[counter] = treeNode{indentLevel, node}
		counter++
	}
	nodes = nodes[0:counter]

	return nodes
}

// buildTree converts an array of nodes, each prefixed with a depth into a tree.
func buildTree(nodes []treeNode, depth int) []ast.Node {
	if len(nodes) == 0 {
		return []ast.Node{}
	}

	// Split the list into sections, treat each section as a tree with its own
	// root.
	sections := [][]treeNode{}
	for _, node := range nodes {
		if node.indent == depth {
			sections = append(sections, []treeNode{node})
		} else {
			sections[len(sections)-1] = append(sections[len(sections)-1], node)
		}
	}

	results := []ast.Node{}
	for _, section := range sections {
		slice := []treeNode{}
		for _, n := range section {
			if n.indent > depth {
				slice = append(slice, n)
			}
		}

		children := buildTree(slice, depth+1)
		for _, child := range children {
			section[0].node.AddChild(child)
		}
		results = append(results, section[0].node)
	}

	return results
}

func toJSON(tree []interface{}) []map[string]interface{} {
	r := make([]map[string]interface{}, len(tree))

	for j, n := range tree {
		rn := reflect.ValueOf(n).Elem()
		r[j] = make(map[string]interface{})
		r[j]["node"] = rn.Type().Name()

		for i := 0; i < rn.NumField(); i++ {
			name := strings.ToLower(rn.Type().Field(i).Name)
			value := rn.Field(i).Interface()

			if name == "children" {
				v := value.([]interface{})

				if len(v) == 0 {
					continue
				}

				value = toJSON(v)
			}

			r[j][name] = value
		}
	}

	return r
}

func check(prefix string, e error) {
	if e != nil {
		panic(prefix + e.Error())
	}
}

// Start begins transpiling an input file.
func Start(args ProgramArgs) error {
	if os.Getenv("GOPATH") == "" {
		return fmt.Errorf("The $GOPATH must be set")
	}

	// 1. Compile it first (checking for errors)
	_, err := os.Stat(args.inputFile)
	if err != nil {
		return fmt.Errorf("Input file is not found")
	}

	// 2. Preprocess
	var pp []byte
	{
		// See : https://clang.llvm.org/docs/CommandGuide/clang.html
		// clang -E <file>    Run the preprocessor stage.
		cmd := exec.Command("clang", "-E", args.inputFile)
		var out bytes.Buffer
		var stderr bytes.Buffer
		cmd.Stdout = &out
		cmd.Stderr = &stderr
		err = cmd.Run()
		if err != nil {
			return fmt.Errorf("preprocess failed: %v\nStdErr = %v", err, stderr.String())
		}
		pp = []byte(out.String())
	}

	ppFilePath := path.Join("/tmp", "pp.c")
	err = ioutil.WriteFile(ppFilePath, pp, 0644)
	if err != nil {
		return fmt.Errorf("writing to %s failed: %v", ppFilePath, err)
	}

	// 3. Generate JSON from AST
	astPP, err := exec.Command("clang", "-Xclang", "-ast-dump", "-fsyntax-only", ppFilePath).Output()
	if err != nil {
		// If clang fails it still prints out the AST, so we have to run it
		// again to get the real error.
		errBody, _ := exec.Command("clang", ppFilePath).CombinedOutput()

		panic("clang failed: " + err.Error() + ":\n\n" + string(errBody))
	}

	lines := readAST(astPP)
	if args.ast {
		for _, l := range lines {
			fmt.Println(l)
		}
		fmt.Println()

		return nil
	}

	p := program.NewProgram()
	p.Verbose = args.verbose
	p.OutputAsTest = true // args.outputAsTest

	nodes := convertLinesToNodes(lines)
	tree := buildTree(nodes, 0)
	ast.FixPositions(tree)

	// Repair the floating literals. See RepairFloatingLiteralsFromSource for
	// more information.
	floatingErrors := ast.RepairFloatingLiteralsFromSource(tree[0], ppFilePath)

	for _, fErr := range floatingErrors {
		message := fmt.Sprintf("could not read exact floating literal: %s",
			fErr.Err.Error())
		p.AddMessage(p.GenerateWarningMessage(errors.New(message), fErr.Node))
	}

	err = transpiler.TranspileAST(args.inputFile, args.packageName, p, tree[0].(ast.Node))
	if err != nil {
		panic(err)
	}

	outputFilePath := args.outputFile

	if outputFilePath == "" {
		cleanFileName := filepath.Clean(filepath.Base(args.inputFile))
		extension := filepath.Ext(args.inputFile)
		outputFilePath = cleanFileName[0:len(cleanFileName)-len(extension)] + ".go"
	}

	err = ioutil.WriteFile(outputFilePath, []byte(p.String()), 0755)
	if err != nil {
		return fmt.Errorf("writing C output file failed: %v", err)
	}

	return nil
}

// newTempFile - returns temp file
func newTempFile(dir, prefix, suffix string) (*os.File, error) {
	for index := 1; index < 10000; index++ {
		path := filepath.Join(dir, fmt.Sprintf("%s%03d%s", prefix, index, suffix))
		if _, err := os.Stat(path); err != nil {
			return os.Create(path)
		}
	}
	return nil, fmt.Errorf("could not create file: %s%03d%s", prefix, 1, suffix)
}

var (
	versionFlag       = flag.Bool("v", false, "print the version and exit")
	transpileCommand  = flag.NewFlagSet("transpile", flag.ContinueOnError)
	verboseFlag       = transpileCommand.Bool("V", false, "print progress as comments")
	outputFlag        = transpileCommand.String("o", "", "output Go generated code to the specified file")
	packageFlag       = transpileCommand.String("p", "main", "set the name of the generated package")
	transpileHelpFlag = transpileCommand.Bool("h", false, "print help information")
	astCommand        = flag.NewFlagSet("ast", flag.ContinueOnError)
	astHelpFlag       = astCommand.Bool("h", false, "print help information")
)

func main() {
	code := runCommand()
	if code != 0 {
		os.Exit(code)
	}
}

func runCommand() int {
	flag.Usage = func() {
		usage := "Usage: %s [-v] [<command>] [<flags>] file.c\n\n"
		usage += "Commands:\n"
		usage += "  transpile\ttranspile an input C source file to Go\n"
		usage += "  ast\t\tprint AST before translated Go code\n\n"

		usage += "Flags:\n"
		fmt.Fprintf(stderr, usage, os.Args[0])
		flag.PrintDefaults()
	}

	transpileCommand.SetOutput(stderr)
	astCommand.SetOutput(stderr)

	flag.Parse()

	if *versionFlag {
		// Simply print out the version and exit.
		fmt.Println(Version)
		return 0
	}

	if flag.NArg() < 1 {
		flag.Usage()
		return 1
	}

	args := ProgramArgs{verbose: *verboseFlag, ast: false}

	switch os.Args[1] {
	case "ast":
		err := astCommand.Parse(os.Args[2:])
		if err != nil {
			fmt.Printf("ast command cannot parse: %v", err)
			return 1
		}

		if *astHelpFlag || astCommand.NArg() == 0 {
			fmt.Fprintf(stderr, "Usage: %s ast file.c\n", os.Args[0])
			astCommand.PrintDefaults()
			return 1
		}

		args.ast = true
		args.inputFile = astCommand.Arg(0)
	case "transpile":
		err := transpileCommand.Parse(os.Args[2:])
		if err != nil {
			fmt.Printf("transpile command cannot parse: %v", err)
			return 1
		}

		if *transpileHelpFlag || transpileCommand.NArg() == 0 {
			fmt.Fprintf(stderr, "Usage: %s transpile [-V] [-o file.go] [-p package] file.c\n", os.Args[0])
			transpileCommand.PrintDefaults()
			return 1
		}

		args.inputFile = transpileCommand.Arg(0)
		args.outputFile = *outputFlag
		args.packageName = *packageFlag
	default:
		flag.Usage()
		return 1
	}

	if err := Start(args); err != nil {
		fmt.Printf("Error: %v", err)
		return 1
	}

	return 0
}
<commit_msg>Bump version: v0.16.3 Radium 2017-10-17<commit_after>// Package c2go contains the main function for running the executable.
//
// Installation
//
//     go get -u github.com/elliotchance/c2go
//
// Usage
//
//     c2go myfile.c
//
package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"strings"

	"errors"
	"reflect"

	"github.com/elliotchance/c2go/ast"
	"github.com/elliotchance/c2go/program"
	"github.com/elliotchance/c2go/transpiler"
)

// Version can be requested through the command line with:
//
//     c2go -v
//
// See https://github.com/elliotchance/c2go/wiki/Release-Process
const Version = "v0.16.3 Radium 2017-10-17"

var stderr io.Writer = os.Stderr

// ProgramArgs defines the options available when processing the program. There
// is no constructor since the zeroed out values are the appropriate defaults -
// you need only set the options you need.
//
// TODO: Better separation on CLI modes
// https://github.com/elliotchance/c2go/issues/134
type ProgramArgs struct {
	verbose     bool
	ast         bool
	inputFile   string
	outputFile  string
	packageName string

	// A private option to output the Go as a *_test.go file.
	outputAsTest bool
}

func readAST(data []byte) []string {
	uncolored := regexp.MustCompile(`\x1b\[[\d;]+m`).ReplaceAll(data, []byte{})
	return strings.Split(string(uncolored), "\n")
}

type treeNode struct {
	indent int
	node   ast.Node
}

func convertLinesToNodes(lines []string) []treeNode {
	nodes := make([]treeNode, len(lines))
	var counter int
	for _, line := range lines {
		if strings.TrimSpace(line) == "" {
			continue
		}

		// It is tempting to discard null AST nodes, but these may
		// have semantic importance: for example, they represent omitted
		// for-loop conditions, as in for(;;).
		line = strings.Replace(line, "<<<NULL>>>", "NullStmt", 1)
		trimmed := strings.TrimLeft(line, "|\\- `")
		node := ast.Parse(trimmed)
		indentLevel := (len(line) - len(trimmed)) / 2
		nodes[counter] = treeNode{indentLevel, node}
		counter++
	}
	nodes = nodes[0:counter]

	return nodes
}

// buildTree converts an array of nodes, each prefixed with a depth into a tree.
func buildTree(nodes []treeNode, depth int) []ast.Node {
	if len(nodes) == 0 {
		return []ast.Node{}
	}

	// Split the list into sections, treat each section as a tree with its own
	// root.
	sections := [][]treeNode{}
	for _, node := range nodes {
		if node.indent == depth {
			sections = append(sections, []treeNode{node})
		} else {
			sections[len(sections)-1] = append(sections[len(sections)-1], node)
		}
	}

	results := []ast.Node{}
	for _, section := range sections {
		slice := []treeNode{}
		for _, n := range section {
			if n.indent > depth {
				slice = append(slice, n)
			}
		}

		children := buildTree(slice, depth+1)
		for _, child := range children {
			section[0].node.AddChild(child)
		}
		results = append(results, section[0].node)
	}

	return results
}

func toJSON(tree []interface{}) []map[string]interface{} {
	r := make([]map[string]interface{}, len(tree))

	for j, n := range tree {
		rn := reflect.ValueOf(n).Elem()
		r[j] = make(map[string]interface{})
		r[j]["node"] = rn.Type().Name()

		for i := 0; i < rn.NumField(); i++ {
			name := strings.ToLower(rn.Type().Field(i).Name)
			value := rn.Field(i).Interface()

			if name == "children" {
				v := value.([]interface{})

				if len(v) == 0 {
					continue
				}

				value = toJSON(v)
			}

			r[j][name] = value
		}
	}

	return r
}

func check(prefix string, e error) {
	if e != nil {
		panic(prefix + e.Error())
	}
}

// Start begins transpiling an input file.
func Start(args ProgramArgs) error {
	if os.Getenv("GOPATH") == "" {
		return fmt.Errorf("The $GOPATH must be set")
	}

	// 1. Compile it first (checking for errors)
	_, err := os.Stat(args.inputFile)
	if err != nil {
		return fmt.Errorf("Input file is not found")
	}

	// 2. Preprocess
	var pp []byte
	{
		// See : https://clang.llvm.org/docs/CommandGuide/clang.html
		// clang -E <file>    Run the preprocessor stage.
		cmd := exec.Command("clang", "-E", args.inputFile)
		var out bytes.Buffer
		var stderr bytes.Buffer
		cmd.Stdout = &out
		cmd.Stderr = &stderr
		err = cmd.Run()
		if err != nil {
			return fmt.Errorf("preprocess failed: %v\nStdErr = %v", err, stderr.String())
		}
		pp = []byte(out.String())
	}

	ppFilePath := path.Join("/tmp", "pp.c")
	err = ioutil.WriteFile(ppFilePath, pp, 0644)
	if err != nil {
		return fmt.Errorf("writing to %s failed: %v", ppFilePath, err)
	}

	// 3. Generate JSON from AST
	astPP, err := exec.Command("clang", "-Xclang", "-ast-dump", "-fsyntax-only", ppFilePath).Output()
	if err != nil {
		// If clang fails it still prints out the AST, so we have to run it
		// again to get the real error.
		errBody, _ := exec.Command("clang", ppFilePath).CombinedOutput()

		panic("clang failed: " + err.Error() + ":\n\n" + string(errBody))
	}

	lines := readAST(astPP)
	if args.ast {
		for _, l := range lines {
			fmt.Println(l)
		}
		fmt.Println()

		return nil
	}

	p := program.NewProgram()
	p.Verbose = args.verbose
	p.OutputAsTest = true // args.outputAsTest

	nodes := convertLinesToNodes(lines)
	tree := buildTree(nodes, 0)
	ast.FixPositions(tree)

	// Repair the floating literals. See RepairFloatingLiteralsFromSource for
	// more information.
	floatingErrors := ast.RepairFloatingLiteralsFromSource(tree[0], ppFilePath)

	for _, fErr := range floatingErrors {
		message := fmt.Sprintf("could not read exact floating literal: %s",
			fErr.Err.Error())
		p.AddMessage(p.GenerateWarningMessage(errors.New(message), fErr.Node))
	}

	err = transpiler.TranspileAST(args.inputFile, args.packageName, p, tree[0].(ast.Node))
	if err != nil {
		panic(err)
	}

	outputFilePath := args.outputFile

	if outputFilePath == "" {
		cleanFileName := filepath.Clean(filepath.Base(args.inputFile))
		extension := filepath.Ext(args.inputFile)
		outputFilePath = cleanFileName[0:len(cleanFileName)-len(extension)] + ".go"
	}

	err = ioutil.WriteFile(outputFilePath, []byte(p.String()), 0755)
	if err != nil {
		return fmt.Errorf("writing C output file failed: %v", err)
	}

	return nil
}

// newTempFile - returns temp file
func newTempFile(dir, prefix, suffix string) (*os.File, error) {
	for index := 1; index < 10000; index++ {
		path := filepath.Join(dir, fmt.Sprintf("%s%03d%s", prefix, index, suffix))
		if _, err := os.Stat(path); err != nil {
			return os.Create(path)
		}
	}
	return nil, fmt.Errorf("could not create file: %s%03d%s", prefix, 1, suffix)
}

var (
	versionFlag       = flag.Bool("v", false, "print the version and exit")
	transpileCommand  = flag.NewFlagSet("transpile", flag.ContinueOnError)
	verboseFlag       = transpileCommand.Bool("V", false, "print progress as comments")
	outputFlag        = transpileCommand.String("o", "", "output Go generated code to the specified file")
	packageFlag       = transpileCommand.String("p", "main", "set the name of the generated package")
	transpileHelpFlag = transpileCommand.Bool("h", false, "print help information")
	astCommand        = flag.NewFlagSet("ast", flag.ContinueOnError)
	astHelpFlag       = astCommand.Bool("h", false, "print help information")
)

func main() {
	code := runCommand()
	if code != 0 {
		os.Exit(code)
	}
}

func runCommand() int {
	flag.Usage = func() {
		usage := "Usage: %s [-v] [<command>] [<flags>] file.c\n\n"
		usage += "Commands:\n"
		usage += "  transpile\ttranspile an input C source file to Go\n"
		usage += "  ast\t\tprint AST before translated Go code\n\n"

		usage += "Flags:\n"
		fmt.Fprintf(stderr, usage, os.Args[0])
		flag.PrintDefaults()
	}

	transpileCommand.SetOutput(stderr)
	astCommand.SetOutput(stderr)

	flag.Parse()

	if *versionFlag {
		// Simply print out the version and exit.
		fmt.Println(Version)
		return 0
	}

	if flag.NArg() < 1 {
		flag.Usage()
		return 1
	}

	args := ProgramArgs{verbose: *verboseFlag, ast: false}

	switch os.Args[1] {
	case "ast":
		err := astCommand.Parse(os.Args[2:])
		if err != nil {
			fmt.Printf("ast command cannot parse: %v", err)
			return 1
		}

		if *astHelpFlag || astCommand.NArg() == 0 {
			fmt.Fprintf(stderr, "Usage: %s ast file.c\n", os.Args[0])
			astCommand.PrintDefaults()
			return 1
		}

		args.ast = true
		args.inputFile = astCommand.Arg(0)
	case "transpile":
		err := transpileCommand.Parse(os.Args[2:])
		if err != nil {
			fmt.Printf("transpile command cannot parse: %v", err)
			return 1
		}

		if *transpileHelpFlag || transpileCommand.NArg() == 0 {
			fmt.Fprintf(stderr, "Usage: %s transpile [-V] [-o file.go] [-p package] file.c\n", os.Args[0])
			transpileCommand.PrintDefaults()
			return 1
		}

		args.inputFile = transpileCommand.Arg(0)
		args.outputFile = *outputFlag
		args.packageName = *packageFlag
	default:
		flag.Usage()
		return 1
	}

	if err := Start(args); err != nil {
		fmt.Printf("Error: %v", err)
		return 1
	}

	return 0
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"github.com/zenazn/goji/bind"
	"github.com/zenazn/goji/web"
	"github.com/zenazn/goji/web/middleware"
	"log"
	"net/http"
	"net/http/pprof"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"time"
)

var clusterNames = flag.String("cluster-name", "default", "The user-visible names for the clusters")
var resourceManagerURL = flag.String("resource-manager-url", "http://localhost:8088", "The HTTP URL to access the resource manager.")
var historyServerURL = flag.String("history-server-url", "http://localhost:19888", "The HTTP URL to access the history server.")
var proxyServerURL = flag.String("proxy-server-url", "", "The HTTP URL to access the proxy server, if separate from the resource manager.")
var namenodeAddress = flag.String("namenode-address", "localhost:9000", "The host:port to access the Namenode metadata service.")
var yarnLogDir = flag.String("yarn-logs-dir", "/tmp/logs", "The HDFS path where YARN stores logs. This is the controlled by the hadoop property yarn.nodemanager.remote-app-log-dir.")
var yarnHistoryDir = flag.String("yarn-history-dir", "/tmp/staging/history/done", "The HDFS path where YARN stores finished job history files. This is the controlled by the hadoop property mapreduce.jobhistory.done-dir.")
var httpTimeout = flag.Duration("http-timeout", time.Second*2, "The timeout used for connecting to YARN API. Pass values like: 2s")
var pollInterval = flag.Duration("poll-interval", time.Second*5, "How often should we poll the job APIs. Pass values like: 2s")
var enableDebug = flag.Bool("pprof", false, "Enable pprof debugging tools at /debug.")
var s3BucketName = flag.String("s3-bucket", "", "S3 bucket to fetch old jobs from")
var s3Region = flag.String("s3-region", "", "AWS region for the job storage S3 bucket")
var s3JobsPrefix = flag.String("s3-jobs-prefix", "", "S3 key prefix (\"folder\") where jobs are stored")
var s3FlowPrefix = flag.String("s3-flow-prefix", "", "S3 key prefix (\"folder\") where cascading flows are stored")

var jts map[string]*jobTracker
var persistedJobClient PersistedJobClient

var rootPath, staticPath string

var mux *web.Mux

func init() {
	bind.WithFlag()
	mux = web.New()
	mux.Use(middleware.RequestID)
	mux.Use(middleware.Logger)
	mux.Use(middleware.Recoverer)
	mux.Use(middleware.AutomaticOptions)
}

func index(c web.C, w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, filepath.Join(rootPath, "index.html"))
}

func getJobs(c web.C, w http.ResponseWriter, r *http.Request) {
	// We only need the details for listing pages.
	var jobs []*job
	for clusterName, tracker := range jts {
		for _, j := range tracker.jobs {
			jobs = append(jobs, &job{
				Cluster: tracker.clusterName,
				Details: j.Details,
				conf: conf{
					Input:         j.conf.Input,
					Output:        j.conf.Output,
					ScaldingSteps: j.conf.ScaldingSteps,
					name:          j.conf.name,
				},
			})
		}
		log.Printf("Appending %d jobs for Cluster %s: %s %s\n", len(jobs), clusterName, tracker.hs, tracker.rm)
	}

	jsonBytes, err := json.Marshal(jobs)
	if err != nil {
		log.Println("error:", err)
		w.WriteHeader(500)
		return
	}

	w.Write(jsonBytes)
}

func getNumClusters(c web.C, w http.ResponseWriter, r *http.Request) {
	jsonBytes, err := json.Marshal(len(jts))
	if err != nil {
		log.Println("error:", err)
		w.WriteHeader(500)
		return
	}
	w.Write(jsonBytes)
}

func getConf(c web.C, w http.ResponseWriter, r *http.Request) {
	id := c.URLParams["id"]
	log.Printf("Getting job conf for %s", id)

	job := getJob(id)
	if job == nil {
		w.WriteHeader(404)
		return
	}

	jsonBytes, err := json.Marshal(jobConf{
		Conf: job.conf,
		ID:   job.Details.ID,
		Name: job.Details.Name,
	})
	if err != nil {
		log.Println("could not marshal:", err)
		w.WriteHeader(500)
		return
	}

	w.Write(jsonBytes)
}

func getJob(rawJobID string) *job {
	// check if we have it in memory
	for clusterName, jt := range jts {
		if _, ok := jt.jobs[jobID(rawJobID)]; ok {
			job := jt.reifyJob(rawJobID)
			job.Cluster = clusterName
			return job
		}
	}

	// check if we have it in long-term storage
	job, _ := persistedJobClient.FetchJob(rawJobID)

	return job
}

func getJobAPIHandler(c web.C, w http.ResponseWriter, r *http.Request) {
	job := getJob(c.URLParams["id"])

	if job == nil {
		w.WriteHeader(404)
		return
	}

	jsonBytes, err := json.Marshal(job)
	if err != nil {
		log.Println("error serializing job:", err)
		w.WriteHeader(500)
		return
	}

	w.Write(jsonBytes)
}

func getJobIdsAPIHandler(c web.C, w http.ResponseWriter, r *http.Request) {
	jobIds, err := persistedJobClient.FetchFlowJobIds(c.URLParams["flowID"])
	if err != nil {
		log.Println("FetchFlowJobIds error:", err)
		w.WriteHeader(500)
		return
	}

	jsonBytes, err := json.Marshal(jobIds)
	if err != nil {
		log.Println("JSON marshal error:", err)
		w.WriteHeader(500)
		return
	}

	w.Write(jsonBytes)
}

func killJob(c web.C, w http.ResponseWriter, r *http.Request) {
	id := c.URLParams["id"]
	jobID := jobID(id)

	for _, jt := range jts {
		if _, ok := jt.jobs[jobID]; ok {
			err := jt.killJob(id)
			if err != nil {
				log.Println("error: ", err)
				w.WriteHeader(500)
				return
			}
			w.WriteHeader(204)
			return
		}
	}

	w.WriteHeader(404)
}

func init() {
	binPath, err := filepath.Abs(filepath.Dir(os.Args[0]))
	if err != nil {
		log.Fatal(err)
	}

	rootPath = filepath.Join(binPath, "..")
	staticPath = filepath.Join(rootPath, "static")
}

func main() {
	flag.Parse()

	var clusterNames = strings.Split(*clusterNames, ",")
	var resourceManagerURLs = strings.Split(*resourceManagerURL, ",")
	var historyServerURLs = strings.Split(*historyServerURL, ",")
	var proxyServerURLs = strings.Split(*proxyServerURL, ",")
	var namenodeAddresses = strings.Split(*namenodeAddress, ",")

	if len(resourceManagerURLs) != len(historyServerURLs) {
		log.Fatal("resource-manager-url and history-server-url are not 1:1")
	}
	if !reflect.DeepEqual(proxyServerURLs, []string{""}) && len(proxyServerURLs) != len(resourceManagerURLs) {
		log.Fatal("proxy-server-url exists and is not 1:1 with resource-manager-url")
	}
	if len(resourceManagerURLs) != len(namenodeAddresses) {
		log.Fatal("resource-manager-url and namenode-address are not 1:1")
	}
	if len(resourceManagerURLs) != len(clusterNames) {
		log.Fatal("cluster-names and resource-manager-url are not 1:1")
	}

	persistedJobClient = NewS3JobClient(*s3Region, *s3BucketName, *s3JobsPrefix, *s3FlowPrefix)
	jts = make(map[string]*jobTracker)
	for i := range resourceManagerURLs {
		var proxyServerURL string
		if reflect.DeepEqual(proxyServerURLs, []string{""}) {
			proxyServerURL = resourceManagerURLs[i]
		} else {
			proxyServerURL = proxyServerURLs[i]
		}
		log.Printf("Creating new JT [%d]: %s %s %s\n", i, resourceManagerURLs[i], historyServerURLs[i], proxyServerURL)
		jts[clusterNames[i]] = newJobTracker(
			clusterNames[i],
			newRecentJobClient(
				resourceManagerURLs[i],
				historyServerURLs[i],
				proxyServerURL,
				namenodeAddresses[i],
			),
		)
	}

	log.Println("initiating JT loop")

	for clusterName, jt := range jts {
		go jt.Loop()
		if err := jt.testLogsDir(); err != nil {
			log.Printf("WARNING: Could not read yarn logs directory for cluster %s. Error message: `%s`\n", clusterName, err)
			log.Println("\tYou can change the path with --yarn-logs-dir=HDFS_PATH.")
			log.Println("\tTo talk to HDFS, Timberlake needs to be able to access the namenode (--namenode-address) and datanodes.")
		}
	}

	sse := newSSE()
	go sse.Loop()

	static := http.StripPrefix("/static/", http.FileServer(http.Dir(staticPath)))
	log.Println("serving static files from", staticPath)

	mux.Get("/static/*", static)
	mux.Get("/", index)
	mux.Get("/jobs/", getJobs)
	mux.Get("/numClusters/", getNumClusters)
	mux.Get("/sse", sse)
	mux.Get("/jobIds/:flowID", getJobIdsAPIHandler)
	mux.Get("/jobs/:id", getJobAPIHandler)
	mux.Get("/jobs/:id/conf", getConf)
	mux.Post("/jobs/:id/kill", killJob)

	if *enableDebug {
		mux.Get("/debug/pprof/*", pprof.Index)
		mux.Get("/debug/pprof/cmdline", pprof.Cmdline)
		mux.Get("/debug/pprof/profile", pprof.Profile)
		mux.Get("/debug/pprof/symbol", pprof.Symbol)
		mux.Get("/debug/pprof/trace", pprof.Trace)
	}

	for _, jt := range jts {
		go jt.sendUpdates(sse)
	}

	http.Serve(bind.Default(), mux)
}
<commit_msg>Make error lines searchable<commit_after>package main

import (
	"encoding/json"
	"flag"
	"github.com/zenazn/goji/bind"
	"github.com/zenazn/goji/web"
	"github.com/zenazn/goji/web/middleware"
	"log"
	"net/http"
	"net/http/pprof"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"time"
)

var clusterNames = flag.String("cluster-name", "default", "The user-visible names for the clusters")
var resourceManagerURL = flag.String("resource-manager-url", "http://localhost:8088", "The HTTP URL to access the resource manager.")
var historyServerURL = flag.String("history-server-url", "http://localhost:19888", "The HTTP URL to access the history server.")
var proxyServerURL = flag.String("proxy-server-url", "", "The HTTP URL to access the proxy server, if separate from the resource manager.")
var namenodeAddress = flag.String("namenode-address", "localhost:9000", "The host:port to access the Namenode metadata service.")
var yarnLogDir = flag.String("yarn-logs-dir", "/tmp/logs", "The HDFS path where YARN stores logs. This is the controlled by the hadoop property yarn.nodemanager.remote-app-log-dir.")
var yarnHistoryDir = flag.String("yarn-history-dir", "/tmp/staging/history/done", "The HDFS path where YARN stores finished job history files. This is the controlled by the hadoop property mapreduce.jobhistory.done-dir.")
var httpTimeout = flag.Duration("http-timeout", time.Second*2, "The timeout used for connecting to YARN API. Pass values like: 2s")
var pollInterval = flag.Duration("poll-interval", time.Second*5, "How often should we poll the job APIs. Pass values like: 2s")
var enableDebug = flag.Bool("pprof", false, "Enable pprof debugging tools at /debug.")
var s3BucketName = flag.String("s3-bucket", "", "S3 bucket to fetch old jobs from")
var s3Region = flag.String("s3-region", "", "AWS region for the job storage S3 bucket")
var s3JobsPrefix = flag.String("s3-jobs-prefix", "", "S3 key prefix (\"folder\") where jobs are stored")
var s3FlowPrefix = flag.String("s3-flow-prefix", "", "S3 key prefix (\"folder\") where cascading flows are stored")

var jts map[string]*jobTracker
var persistedJobClient PersistedJobClient

var rootPath, staticPath string

var mux *web.Mux

func init() {
	bind.WithFlag()
	mux = web.New()
	mux.Use(middleware.RequestID)
	mux.Use(middleware.Logger)
	mux.Use(middleware.Recoverer)
	mux.Use(middleware.AutomaticOptions)
}

func index(c web.C, w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, filepath.Join(rootPath, "index.html"))
}

func getJobs(c web.C, w http.ResponseWriter, r *http.Request) {
	// We only need the details for listing pages.
	var jobs []*job
	for clusterName, tracker := range jts {
		for _, j := range tracker.jobs {
			jobs = append(jobs, &job{
				Cluster: tracker.clusterName,
				Details: j.Details,
				conf: conf{
					Input:         j.conf.Input,
					Output:        j.conf.Output,
					ScaldingSteps: j.conf.ScaldingSteps,
					name:          j.conf.name,
				},
			})
		}
		log.Printf("Appending %d jobs for Cluster %s: %s %s\n", len(jobs), clusterName, tracker.hs, tracker.rm)
	}

	jsonBytes, err := json.Marshal(jobs)
	if err != nil {
		log.Println("getJobs error:", err)
		w.WriteHeader(500)
		return
	}

	w.Write(jsonBytes)
}

func getNumClusters(c web.C, w http.ResponseWriter, r *http.Request) {
	jsonBytes, err := json.Marshal(len(jts))
	if err != nil {
		log.Println("getNumbClusters error:", err)
		w.WriteHeader(500)
		return
	}
	w.Write(jsonBytes)
}

func getConf(c web.C, w http.ResponseWriter, r *http.Request) {
	id := c.URLParams["id"]
	log.Printf("Getting job conf for %s", id)

	job := getJob(id)
	if job == nil {
		w.WriteHeader(404)
		return
	}

	jsonBytes, err := json.Marshal(jobConf{
		Conf: job.conf,
		ID:   job.Details.ID,
		Name: job.Details.Name,
	})
	if err != nil {
		log.Println("could not marshal:", err)
		w.WriteHeader(500)
		return
	}

	w.Write(jsonBytes)
}

func getJob(rawJobID string) *job {
	// check if we have it in memory
	for clusterName, jt := range jts {
		if _, ok := jt.jobs[jobID(rawJobID)]; ok {
			job := jt.reifyJob(rawJobID)
			job.Cluster = clusterName
			return job
		}
	}

	// check if we have it in long-term storage
	job, _ := persistedJobClient.FetchJob(rawJobID)

	return job
}

func getJobAPIHandler(c web.C, w http.ResponseWriter, r *http.Request) {
	job := getJob(c.URLParams["id"])

	if job == nil {
		w.WriteHeader(404)
		return
	}

	jsonBytes, err := json.Marshal(job)
	if err != nil {
		log.Println("error serializing job:", err)
		w.WriteHeader(500)
		return
	}

	w.Write(jsonBytes)
}

func getJobIdsAPIHandler(c web.C, w http.ResponseWriter, r *http.Request) {
	jobIds, err := persistedJobClient.FetchFlowJobIds(c.URLParams["flowID"])
	if err != nil {
		log.Println("FetchFlowJobIds error:", err)
		w.WriteHeader(500)
		return
	}

	jsonBytes, err := json.Marshal(jobIds)
	if err != nil {
		log.Println("JSON marshal error:", err)
		w.WriteHeader(500)
		return
	}

	w.Write(jsonBytes)
}

func killJob(c web.C, w http.ResponseWriter, r *http.Request) {
	id := c.URLParams["id"]
	jobID := jobID(id)

	for _, jt := range jts {
		if _, ok := jt.jobs[jobID]; ok {
			err := jt.killJob(id)
			if err != nil {
				log.Println("killJob error: ", err)
				w.WriteHeader(500)
				return
			}
			w.WriteHeader(204)
			return
		}
	}

	w.WriteHeader(404)
}

func init() {
	binPath, err := filepath.Abs(filepath.Dir(os.Args[0]))
	if err != nil {
		log.Fatal(err)
	}

	rootPath = filepath.Join(binPath, "..")
	staticPath = filepath.Join(rootPath, "static")
}

func main() {
	flag.Parse()

	var clusterNames = strings.Split(*clusterNames, ",")
	var resourceManagerURLs = strings.Split(*resourceManagerURL, ",")
	var historyServerURLs = strings.Split(*historyServerURL, ",")
	var proxyServerURLs = strings.Split(*proxyServerURL, ",")
	var namenodeAddresses = strings.Split(*namenodeAddress, ",")

	if len(resourceManagerURLs) != len(historyServerURLs) {
		log.Fatal("resource-manager-url and history-server-url are not 1:1")
	}
	if !reflect.DeepEqual(proxyServerURLs, []string{""}) && len(proxyServerURLs) != len(resourceManagerURLs) {
		log.Fatal("proxy-server-url exists and is not 1:1 with resource-manager-url")
	}
	if len(resourceManagerURLs) != len(namenodeAddresses) {
		log.Fatal("resource-manager-url and namenode-address are not 1:1")
	}
	if len(resourceManagerURLs) != len(clusterNames) {
		log.Fatal("cluster-names and resource-manager-url are not 1:1")
	}

	persistedJobClient = NewS3JobClient(*s3Region, *s3BucketName, *s3JobsPrefix, *s3FlowPrefix)
	jts = make(map[string]*jobTracker)
	for i := range resourceManagerURLs {
		var proxyServerURL string
		if reflect.DeepEqual(proxyServerURLs, []string{""}) {
			proxyServerURL = resourceManagerURLs[i]
		} else {
			proxyServerURL = proxyServerURLs[i]
		}
		log.Printf("Creating new JT [%d]: %s %s %s\n", i, resourceManagerURLs[i], historyServerURLs[i], proxyServerURL)
		jts[clusterNames[i]] = newJobTracker(
			clusterNames[i],
			newRecentJobClient(
				resourceManagerURLs[i],
				historyServerURLs[i],
				proxyServerURL,
				namenodeAddresses[i],
			),
		)
	}

	log.Println("initiating JT loop")

	for clusterName, jt := range jts {
		go jt.Loop()
		if err := jt.testLogsDir(); err != nil {
			log.Printf("WARNING: Could not read yarn logs directory for cluster %s. Error message: `%s`\n", clusterName, err)
			log.Println("\tYou can change the path with --yarn-logs-dir=HDFS_PATH.")
			log.Println("\tTo talk to HDFS, Timberlake needs to be able to access the namenode (--namenode-address) and datanodes.")
		}
	}

	sse := newSSE()
	go sse.Loop()

	static := http.StripPrefix("/static/", http.FileServer(http.Dir(staticPath)))
	log.Println("serving static files from", staticPath)

	mux.Get("/static/*", static)
	mux.Get("/", index)
	mux.Get("/jobs/", getJobs)
	mux.Get("/numClusters/", getNumClusters)
	mux.Get("/sse", sse)
	mux.Get("/jobIds/:flowID", getJobIdsAPIHandler)
	mux.Get("/jobs/:id", getJobAPIHandler)
	mux.Get("/jobs/:id/conf", getConf)
	mux.Post("/jobs/:id/kill", killJob)

	if *enableDebug {
		mux.Get("/debug/pprof/*", pprof.Index)
		mux.Get("/debug/pprof/cmdline", pprof.Cmdline)
		mux.Get("/debug/pprof/profile", pprof.Profile)
		mux.Get("/debug/pprof/symbol", pprof.Symbol)
		mux.Get("/debug/pprof/trace", pprof.Trace)
	}

	for _, jt := range jts {
		go jt.sendUpdates(sse)
	}

	http.Serve(bind.Default(), mux)
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/garyburd/redigo/redis"
	"github.com/realglobe-Inc/edo/driver"
	"github.com/realglobe-Inc/edo/util/crypto"
	logutil "github.com/realglobe-Inc/edo/util/log"
	"github.com/realglobe-Inc/edo/util/server"
	"github.com/realglobe-Inc/go-lib-rg/erro"
	"github.com/realglobe-Inc/go-lib-rg/rglog"
	"net/http"
	"os"
	"time"
)

var exitCode = 0

func exit() {
	if exitCode != 0 {
		os.Exit(exitCode)
	}
}

func main() {
	defer exit()
	defer rglog.Flush()

	logutil.InitConsole("github.com/realglobe-Inc")

	param, err := parseParameters(os.Args...)
	if err != nil {
		log.Err(erro.Unwrap(err))
		log.Debug(err)
		exitCode = 1
		return
	}

	logutil.SetupConsole("github.com/realglobe-Inc", param.consLv)
	if err := logutil.Setup("github.com/realglobe-Inc", param.logType, param.logLv, param); err != nil {
		log.Err(erro.Unwrap(err))
		log.Debug(err)
		exitCode = 1
		return
	}

	if err := mainCore(param); err != nil {
		err = erro.Wrap(err)
		log.Err(erro.Unwrap(err))
		log.Debug(err)
		exitCode = 1
		return
	}

	log.Info("Shut down.")
}

// system を準備する。
func mainCore(param *parameters) error {
	key, err := crypto.ReadPrivateKey(param.keyPath)
	if err != nil {
		return erro.Wrap(err)
	}

	const (
		connNum = 5
		idlDur  = 10 * time.Minute
	)
	redPools := map[string]*redis.Pool{} // 同じ redis-server ならコネクションプールを共有する。

	var taCont taContainer
	switch param.taContType {
	case "file":
		taCont = newFileTaContainer(param.taContPath, param.caStaleDur, param.caExpiDur)
		log.Info("Use file TA container " + param.taContPath)
	case "mongo":
		taCont = newMongoTaContainer(param.taContUrl, param.taContDb, param.taContColl, param.caStaleDur, param.caExpiDur)
		log.Info("Use mongodb TA container " + param.taContUrl)
	default:
		return erro.New("invalid TA container type " + param.taContType)
	}

	var accCont accountContainer
	switch param.accContType {
	case "file":
		accCont = newFileAccountContainer(param.accContPath, param.accNameContPath, param.caStaleDur, param.caExpiDur)
		log.Info("Use file account container " + param.accContPath + "," + param.accNameContPath)
	case "mongo":
		accCont = newMongoAccountContainer(param.accContUrl, param.accContDb, param.accContColl, param.caStaleDur, param.caExpiDur)
		log.Info("Use mongodb account container " + param.accContUrl)
	default:
		return erro.New("invalid account container type " + param.accContType)
	}

	var consCont consentContainer
	switch param.consContType {
	case "file":
		consCont = newFileConsentContainer(param.consContPath, param.caStaleDur, param.caExpiDur)
		log.Info("Use file consent container " + param.consContPath)
	case "mongo":
		consCont = newMongoConsentContainer(param.consContUrl, param.consContDb, param.consContColl, param.caStaleDur, param.caExpiDur)
		log.Info("Use mongodb consent container " + param.consContUrl)
	default:
		return erro.New("invalid consent container type " + param.consContType)
	}

	var sessCont sessionContainer
	switch param.sessContType {
	case "memory":
		sessCont = newMemorySessionContainer(param.sessIdLen, param.procId, param.caStaleDur, param.caExpiDur)
		log.Info("Use memory session container.")
	case "file":
		sessCont = newFileSessionContainer(param.sessIdLen, param.procId, param.sessContPath, param.sessExpiContPath, param.caStaleDur, param.caExpiDur)
		log.Info("Use file session container " + param.sessContPath + "," + param.sessExpiContPath)
	case "redis":
		if redPools[param.sessContUrl] == nil {
			redPools[param.sessContUrl] = driver.NewRedisPool(param.sessContUrl, connNum, idlDur)
		}
		sessCont = newRedisSessionContainer(param.sessIdLen, param.procId, redPools[param.sessContUrl], param.sessContPrefix, param.caStaleDur, param.caExpiDur)
		log.Info("Use redis session container " + param.sessContUrl)
	default:
		return erro.New("invalid session container type " + param.sessContType)
	}

	var codCont codeContainer
	switch param.codContType {
	case "memory":
		codCont = newMemoryCodeContainer(param.codIdLen, param.procId, param.codTicExpDur, param.codSavDur, param.caStaleDur, param.caExpiDur)
		log.Info("Use memory code container.")
	case "file":
		codCont = newFileCodeContainer(param.codIdLen, param.procId, param.codTicExpDur, param.codSavDur, param.codContPath, param.codExpiContPath, param.caStaleDur, param.caExpiDur)
		log.Info("Use file code container " + param.codContPath + "," + param.codExpiContPath)
	case "redis":
		if redPools[param.codContUrl] == nil {
			redPools[param.codContUrl] = driver.NewRedisPool(param.codContUrl, connNum, idlDur)
		}
		codCont = newRedisCodeContainer(param.codIdLen, param.procId, param.codTicExpDur, param.codSavDur, redPools[param.codContUrl], param.codContPrefix, param.caStaleDur, param.caExpiDur)
		log.Info("Use redis code container " + param.codContUrl)
	default:
		return erro.New("invalid code container type " + param.codContType)
	}

	var tokCont tokenContainer
	switch param.tokContType {
	case "memory":
		tokCont = newMemoryTokenContainer(param.tokIdLen, param.procId, param.tokSavDur, param.caStaleDur, param.caExpiDur)
		log.Info("Use memory token container.")
	case "file":
		tokCont = newFileTokenContainer(param.tokIdLen, param.procId, param.tokSavDur, param.tokContPath, param.tokExpiContPath, param.caStaleDur, param.caExpiDur)
		log.Info("Use file token container " + param.tokContPath + "," + param.tokExpiContPath)
	case "redis":
		if redPools[param.tokContUrl] == nil {
			redPools[param.tokContUrl] = driver.NewRedisPool(param.tokContUrl, connNum, idlDur)
		}
		tokCont = newRedisTokenContainer(param.tokIdLen, param.procId, param.tokSavDur, redPools[param.tokContUrl], param.tokContPrefix, param.caStaleDur, param.caExpiDur)
		log.Info("Use redis token container " + param.tokContUrl)
	default:
		return erro.New("invalid token container type " + param.tokContType)
	}

	sys := &system{
		param.selfId,
		param.secCook,
		param.codIdLen / 2,
		param.codIdLen / 2,
		param.uiUri,
		param.uiPath,
		taCont,
		accCont,
		consCont,
		sessCont,
		codCont,
		tokCont,
		param.codExpiDur,
		param.tokExpiDur,
		param.idTokExpiDur,
		param.sessExpiDur,
		param.sigAlg,
		param.kid,
		key,
	}
	return serve(sys, param.socType, param.socPath, param.socPort, param.protType, nil)
}

// 振り分ける。
func serve(sys *system, socType, socPath string, socPort int, protType string, shutCh chan struct{}) error {
	routes := map[string]server.HandlerFunc{
		authPath: func(w http.ResponseWriter, r *http.Request) error {
			return authPage(w, r, sys)
		},
		loginPath: func(w http.ResponseWriter, r *http.Request) error {
			return loginPage(w, r, sys)
		},
		selPath: func(w http.ResponseWriter, r *http.Request) error {
			return selectPage(w, r, sys)
		},
		consPath: func(w http.ResponseWriter, r *http.Request) error {
			return consentPage(w, r, sys)
		},
		tokPath: func(w http.ResponseWriter, r *http.Request) error {
			return tokenApi(w, r, sys)
		},
		accInfPath: func(w http.ResponseWriter, r *http.Request) error {
			return accountInfoApi(w, r, sys)
		},
	}
	if routes["/"] == nil {
		routes["/"] = func(w http.ResponseWriter, r *http.Request) error {
			return newIdpError(errInvReq, "invalid endpoint", http.StatusNotFound, nil)
		}
	}
	if sys.uiPath != "" {
		// ファイル配信も自前でやる。
		fileHndl := http.StripPrefix(sys.uiUri, http.FileServer(http.Dir(sys.uiPath)))
		for _, uri := range []string{sys.uiUri, sys.uiUri + "/"} {
			routes[uri] = func(w http.ResponseWriter, r *http.Request) error {
				fileHndl.ServeHTTP(w, r)
				return nil
			}
		}
	}
	return server.TerminableServe(socType, socPath, socPort, protType, routes, shutCh, panicErrorWrapper)
}
<commit_msg>グローバルな変数を減らした<commit_after>package main

import (
	"github.com/garyburd/redigo/redis"
	"github.com/realglobe-Inc/edo/driver"
	"github.com/realglobe-Inc/edo/util/crypto"
	logutil "github.com/realglobe-Inc/edo/util/log"
	"github.com/realglobe-Inc/edo/util/server"
	"github.com/realglobe-Inc/go-lib-rg/erro"
	"github.com/realglobe-Inc/go-lib-rg/rglog"
	"net/http"
	"os"
	"time"
)

func main() {
	var exitCode = 0
	defer func() {
		if exitCode != 0 {
			os.Exit(exitCode)
		}
	}()

	defer rglog.Flush()

	logutil.InitConsole("github.com/realglobe-Inc")

	param, err := parseParameters(os.Args...)
	if err != nil {
		log.Err(erro.Unwrap(err))
		log.Debug(err)
		exitCode = 1
		return
	}

	logutil.SetupConsole("github.com/realglobe-Inc", param.consLv)
	if err := logutil.Setup("github.com/realglobe-Inc", param.logType, param.logLv, param); err != nil {
		log.Err(erro.Unwrap(err))
		log.Debug(err)
		exitCode = 1
		return
	}

	if err := mainCore(param); err != nil {
		err = erro.Wrap(err)
		log.Err(erro.Unwrap(err))
		log.Debug(err)
		exitCode = 1
		return
	}

	log.Info("Shut down.")
}

// system を準備する。
func mainCore(param *parameters) error {
	key, err := crypto.ReadPrivateKey(param.keyPath)
	if err != nil {
		return erro.Wrap(err)
	}

	const (
		connNum = 5
		idlDur  = 10 * time.Minute
	)
	redPools := map[string]*redis.Pool{} // 同じ redis-server ならコネクションプールを共有する。

	var taCont taContainer
	switch param.taContType {
	case "file":
		taCont = newFileTaContainer(param.taContPath, param.caStaleDur, param.caExpiDur)
		log.Info("Use file TA container " + param.taContPath)
	case "mongo":
		taCont = newMongoTaContainer(param.taContUrl, param.taContDb, param.taContColl, param.caStaleDur, param.caExpiDur)
		log.Info("Use mongodb TA container " + param.taContUrl)
	default:
		return erro.New("invalid TA container type " + param.taContType)
	}

	var accCont accountContainer
	switch param.accContType {
	case "file":
		accCont = newFileAccountContainer(param.accContPath, param.accNameContPath, param.caStaleDur, param.caExpiDur)
		log.Info("Use file account container " + param.accContPath + "," + param.accNameContPath)
	case "mongo":
		accCont = newMongoAccountContainer(param.accContUrl, param.accContDb, param.accContColl, param.caStaleDur, param.caExpiDur)
		log.Info("Use mongodb account container " + param.accContUrl)
	default:
		return erro.New("invalid account container type " + param.accContType)
	}

	var consCont consentContainer
	switch param.consContType {
	case "file":
		consCont = newFileConsentContainer(param.consContPath, param.caStaleDur, param.caExpiDur)
		log.Info("Use file consent container " + param.consContPath)
	case "mongo":
		consCont = newMongoConsentContainer(param.consContUrl, param.consContDb, param.consContColl, param.caStaleDur, param.caExpiDur)
		log.Info("Use mongodb consent container " + param.consContUrl)
	default:
		return erro.New("invalid consent container type " + param.consContType)
	}

	var sessCont sessionContainer
	switch param.sessContType {
	case "memory":
		sessCont = newMemorySessionContainer(param.sessIdLen, param.procId, param.caStaleDur, param.caExpiDur)
		log.Info("Use memory session container.")
	case "file":
		sessCont = newFileSessionContainer(param.sessIdLen, param.procId, param.sessContPath, param.sessExpiContPath, param.caStaleDur, param.caExpiDur)
		log.Info("Use file session container " + param.sessContPath + "," + param.sessExpiContPath)
	case "redis":
		if redPools[param.sessContUrl] == nil {
			redPools[param.sessContUrl] = driver.NewRedisPool(param.sessContUrl, connNum, idlDur)
		}
		sessCont = newRedisSessionContainer(param.sessIdLen, param.procId, redPools[param.sessContUrl], param.sessContPrefix, param.caStaleDur, param.caExpiDur)
		log.Info("Use redis session container " + param.sessContUrl)
	default:
		return erro.New("invalid session container type " + param.sessContType)
	}

	var codCont codeContainer
	switch param.codContType {
	case "memory":
		codCont = newMemoryCodeContainer(param.codIdLen, param.procId, param.codTicExpDur, param.codSavDur, param.caStaleDur, param.caExpiDur)
		log.Info("Use memory code container.")
	case "file":
		codCont = newFileCodeContainer(param.codIdLen, param.procId, param.codTicExpDur, param.codSavDur, param.codContPath, param.codExpiContPath, param.caStaleDur, param.caExpiDur)
		log.Info("Use file code container " + param.codContPath + "," + param.codExpiContPath)
	case "redis":
		if redPools[param.codContUrl] == nil {
			redPools[param.codContUrl] = driver.NewRedisPool(param.codContUrl, connNum, idlDur)
		}
		codCont = newRedisCodeContainer(param.codIdLen, param.procId, param.codTicExpDur, param.codSavDur, redPools[param.codContUrl], param.codContPrefix, param.caStaleDur, param.caExpiDur)
		log.Info("Use redis code container " + param.codContUrl)
	default:
		return erro.New("invalid code container type " + param.codContType)
	}

	var tokCont tokenContainer
	switch param.tokContType {
	case "memory":
		tokCont = newMemoryTokenContainer(param.tokIdLen, param.procId, param.tokSavDur, param.caStaleDur, param.caExpiDur)
		log.Info("Use memory token container.")
	case "file":
		tokCont = newFileTokenContainer(param.tokIdLen, param.procId, param.tokSavDur, param.tokContPath, param.tokExpiContPath, param.caStaleDur, param.caExpiDur)
		log.Info("Use file token container " + param.tokContPath + "," + param.tokExpiContPath)
	case "redis":
		if redPools[param.tokContUrl] == nil {
			redPools[param.tokContUrl] = driver.NewRedisPool(param.tokContUrl, connNum, idlDur)
		}
		tokCont = newRedisTokenContainer(param.tokIdLen, param.procId, param.tokSavDur, redPools[param.tokContUrl], param.tokContPrefix, param.caStaleDur, param.caExpiDur)
		log.Info("Use redis token container " + param.tokContUrl)
	default:
		return erro.New("invalid token container type " + param.tokContType)
	}

	sys := &system{
		param.selfId,
		param.secCook,
		param.codIdLen / 2,
		param.codIdLen / 2,
		param.uiUri,
		param.uiPath,
		taCont,
		accCont,
		consCont,
		sessCont,
		codCont,
		tokCont,
		param.codExpiDur,
		param.tokExpiDur,
		param.idTokExpiDur,
		param.sessExpiDur,
		param.sigAlg,
		param.kid,
		key,
	}
	return serve(sys, param.socType, param.socPath, param.socPort, param.protType, nil)
}

// 振り分ける。
func serve(sys *system, socType, socPath string, socPort int, protType string, shutCh chan struct{}) error {
	routes := map[string]server.HandlerFunc{
		authPath: func(w http.ResponseWriter, r *http.Request) error {
			return authPage(w, r, sys)
		},
		loginPath: func(w http.ResponseWriter, r *http.Request) error {
			return loginPage(w, r, sys)
		},
		selPath: func(w http.ResponseWriter, r *http.Request) error {
			return selectPage(w, r, sys)
		},
		consPath: func(w http.ResponseWriter, r *http.Request) error {
			return consentPage(w, r, sys)
		},
		tokPath: func(w http.ResponseWriter, r *http.Request) error {
			return tokenApi(w, r, sys)
		},
		accInfPath: func(w http.ResponseWriter, r *http.Request) error {
			return accountInfoApi(w, r, sys)
		},
	}
	if routes["/"] == nil {
		routes["/"] = func(w http.ResponseWriter, r *http.Request) error {
			return newIdpError(errInvReq, "invalid endpoint", http.StatusNotFound, nil)
		}
	}
	if sys.uiPath != "" {
		// ファイル配信も自前でやる。
		fileHndl := http.StripPrefix(sys.uiUri, http.FileServer(http.Dir(sys.uiPath)))
		for _, uri := range []string{sys.uiUri, sys.uiUri + "/"} {
			routes[uri] = func(w http.ResponseWriter, r *http.Request) error {
				fileHndl.ServeHTTP(w, r)
				return nil
			}
		}
	}
	return server.TerminableServe(socType, socPath, socPort, protType, routes, shutCh, panicErrorWrapper)
}
<|endoftext|>


<commit_before>package main

import (
	"yap/app"
	"github.com/gonuts/commander"
	_ "net/http/pprof"

	"fmt"
	"os"
)

var cmd *commander.Commander

func init() {
	cmd = app.AllCommands()
}

func main() {
	err := cmd.Flag.Parse(os.Args[1:])
	if err != nil {
		fmt.Printf("**err**: %v\n", err)
		os.Exit(1)
	}

	args := cmd.Flag.Args()
	err = cmd.Run(args)
	if err != nil {
		fmt.Printf("**err**: %v\n", err)
		os.Exit(1)
	}

	return
}
<commit_msg>Add build tag for appengine<commit_after>// +build !appengine

package main

import (
	"github.com/gonuts/commander"
	_ "net/http/pprof"
	"yap/app"

	"fmt"
	"os"
)

var cmd *commander.Commander

func init() {
	cmd = app.AllCommands()
}

func main() {
	err := cmd.Flag.Parse(os.Args[1:])
	if err != nil {
		fmt.Printf("**err**: %v\n", err)
		os.Exit(1)
	}

	args := cmd.Flag.Args()
	err = cmd.Run(args)
	if err != nil {
		fmt.Printf("**err**: %v\n", err)
		os.Exit(1)
	}

	return
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/codegangsta/negroni"
	"github.com/luismesas/goPi/piface"
	"github.com/luismesas/goPi/spi"
	"github.com/unrolled/render"
	"net/http"
	"time"
)

var progress int

func main() {
	// Render engine
	r := render.New(render.Options{
		Layout: "layout",
	})

	// Handlers
	mux := http.NewServeMux()
	mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		r.HTML(w, http.StatusOK, "home", progress)
	})
	mux.HandleFunc("/about", func(w http.ResponseWriter, req *http.Request) {
		r.HTML(w, http.StatusOK, "about", nil)
	})
	mux.HandleFunc("/blink", func(w http.ResponseWriter, req *http.Request) {
		go testLed()
		r.Text(w, http.StatusOK, "OK")
	})

	// HTTP Server
	n := negroni.Classic()
	n.UseHandler(mux)
	n.Run(":3000")
}

func testLed() {
	// creates a new pifacedigital instance
	pfd := piface.NewPiFaceDigital(spi.DEFAULT_HARDWARE_ADDR, spi.DEFAULT_BUS, spi.DEFAULT_CHIP)

	// initializes pifacedigital board
	err := pfd.InitBoard()
	if err != nil {
		fmt.Printf("Error on init board: %s", err)
		return
	}
	for i := 0; i <= 10; i++ {
		progress = i * 10
		pfd.Leds[0].Toggle()
		time.Sleep(time.Second)
		pfd.Leds[0].Toggle()
		time.Sleep(time.Second)
	}
}
<commit_msg>testMotor<commit_after>package main

import (
	"fmt"
	"github.com/codegangsta/negroni"
	"github.com/luismesas/goPi/piface"
	"github.com/luismesas/goPi/spi"
	"github.com/unrolled/render"
	"net/http"
	"time"
)

var progress int

func main() {
	// Render engine
	r := render.New(render.Options{
		Layout: "layout",
	})

	// Handlers
	mux := http.NewServeMux()
	mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		r.HTML(w, http.StatusOK, "home", progress)
	})
	mux.HandleFunc("/about", func(w http.ResponseWriter, req *http.Request) {
		r.HTML(w, http.StatusOK, "about", nil)
	})
	mux.HandleFunc("/blink", func(w http.ResponseWriter, req *http.Request) {
		go testLed()
		r.Text(w, http.StatusOK, "OK")
	})

	// HTTP Server
	n := negroni.Classic()
	n.UseHandler(mux)
	n.Run(":3000")
}

func testLed() {
	// creates a new pifacedigital instance
	pfd := piface.NewPiFaceDigital(spi.DEFAULT_HARDWARE_ADDR, spi.DEFAULT_BUS, spi.DEFAULT_CHIP)

	// initializes pifacedigital board
	err := pfd.InitBoard()
	if err != nil {
		fmt.Printf("Error on init board: %s", err)
		return
	}
	for i := 0; i <= 10; i++ {
		progress = i * 10
		pfd.Leds[0].Toggle()
		time.Sleep(time.Second)
		pfd.Leds[0].Toggle()
		time.Sleep(time.Second)
	}
}
func testMotor() {
	// creates a new pifacedigital instance
	pfd := piface.NewPiFaceDigital(spi.DEFAULT_HARDWARE_ADDR, spi.DEFAULT_BUS, spi.DEFAULT_CHIP)

	// initializes pifacedigital board
	err := pfd.InitBoard()
	if err != nil {
		fmt.Printf("Error on init board: %s", err)
		return
	}
	pfd.OutputPins[0].Toggle()
	time.Sleep(time.Second * 5)
	pfd.OutputPins[0].Toggle()
}
<|endoftext|>