from watchdog.utils.dirsnapshot import DirectorySnapshot
import os
from saveFileHandler.filehandler import *
from saveFileHandler.features import Terrains, Features
import time
import multiprocessing as mp
import pyqtgraph as pg
import copy
from collections import defaultdict
import numpy as np
from saveFileHandler.civColors import CIV_LEADER_COLORS, COLORS_PRISM, CIV_OVERFLOW_COLORS,\
    CS_COLOR_MAP, CS_TYPES, CS_UNICODE_MAP  # CIV_COLORS
try:
    from saveFileHandler.civLocalization import CIV_LEADER_NAMES, CIV_NAMES, CITY_NAMES
    civLocalizationImportSuccess = True
except:
    civLocalizationImportSuccess = False
    pass
import traceback
# Terrain types:
# grassland: 				48 198 231 131:		2213004848     30 C6 E7 83    b'\x30\xC6\xE7\x83'
# grassland (hills):		112 12 157 110: 	1855786096     70 0C 9D 6E    b'\x70\x0C\x9D\x6E'
# grassland (mountains):	51 180 131 95:		1602466867     33 B4 83 5F    b'\x33\xB4\x83\x5F'
# plains: 				    94 134 230 251: 	4226188894     5E 86 E6 FB    b'\x94\x86\xE6\xFB'
# plains (hills): 		    158 100 206 230:	3872285854     9E 64 CE E6    b'\x9E\x64\xCE\xE6'
# plains (mountains):		240 168 185 163:	2746853616     F0 A8 B9 A3    b'\xF0\xA8\xB9\xA3'
# desert: 				    44 10 168 229:		3852995116     2C 0A A8 E5    b'\x2C\x0A\xA8\xE5'
# desert (hills): 		    179 52 65 185:		3108058291     B3 34 41 B9    b'\xB3\x34\x41\xB9'
# desert (mountains):   	249 190 144 84:		1418772217     F9 BE 90 54    b'\xF9\xBE\x90\x54
# tundra: 				    171 158 242 72:		1223859883     AB 9E F2 48    b'\xAB\x9E\xF2\x48'
# tundra (hills):			246 176 98 235:		3949113590     F6 B0 62 EB    b'\xF6\xB0\x62\xEB'
# tundra (mountains):		189 221 73 223:		3746160061     BD DD 49 DF    b'\xBD\xDD\x49\xDF'
# snow: 					15 132 234 103:		1743422479     0F 84 EA 67    b'\x0F\x84\xEA\x67'
# snow (hills): 			128 18 3 229:		3842183808     80 12 03 E5    b'\x80\x12\x03\xE5'
# snow (mountains):		    244 70 177 41:		699483892      F4 46 B1 29    b'\xF4\x46\xB1\x29'
# Ocean: 					221 9 201 71:		1204357597     DD 09 C9 47    b'\xDD\x09\xC9\x47'
# Coast: 					17 122 112 74:		1248885265     11 7A 70 4A    b'\x11\x7A\x70\x4A'

# Features:
# No feature: 			    255 255 255 255:	FF FF FF FF
# Rainforest:				57 84 17 233:		39 54 11 E9
# Woods:					68 9 34 10: 		44 09 22 0A
# March (grassland only):	195 89 85 35:		C3 59 55 23
# Oasis (desert only):	    163 83 31 98:		A3 53 1F 62
# Floodlands(desert only):  38 119 181 82:		26 77 B5 52
# Ice (ocean/coast):		148 3 236 91:		94 03 EC 5B
#
# Goody hut: 									EA CD 58 15		1434118760
# Barb: 										BC 0A 2B DE		DE 2B 0A BC		3727362748
#
# Terrain2?:
# Sea:					    255 255 255 255: 	FF FF FF FF
# Land:					    197 141 5 214:		C5 8D 05 D6
# Snow:					    234 205 88 21:		EA CD 58 15

cityColors = np.array([
[255, 255, 255],
[146, 221,   9],
[255, 168,  12],
[255, 255, 255],
[200, 248, 255],
[ 41,  83,  44],
[ 60,   0, 108],
[ 80,   0, 136],
[255, 255, 255],
[239, 231, 179],
[ 82,   0, 208],
[255, 255, 255],
[255,  45,  45],
[235, 235, 138],
[ 36,  43,  32],
[ 65, 141, 253],
[255, 153,  49],
[158,  46,  28],
[184,   0,   0],
[255,   0,   0],
[255, 120,   0],
[ 39, 178,  79],
[245, 230,  55],
[ 56,   0,   0],
[255, 255,  74],
[  3,  20, 124],
[239, 198,   0],
[  0,   0,   0],
[176,   7,   3],
[ 90,   0,   9],
[244, 168, 168],
[248, 246,   2],
[255, 254, 215],
[136, 238, 212],
[147, 169, 255],
[ 69,   0,   3],
[  6, 159, 119],
[251, 201, 129],
[ 23,  62,  65],
[255, 255, 255],
[ 18,  82,  30],
[ 24, 239, 206],
[106,  49,  24],
])

civColors = np.array([
[ 31,  51, 120],
[ 43,  87,  45],
[255, 243, 173],
[234,   0,   0],
[ 43,  81,  97],
[149, 221,  10],
[113, 161, 232],
[204, 204, 204],
[  0, 148,  82],
[108,  42,  20],
[255, 251,   3],
[108,   2,   0],
[  1,  39,  14],
[ 65, 141, 253],
[179, 177, 184],
[255, 255, 255],
[ 18, 135,   6],
[110, 210, 217],
[255, 255, 255],
[ 26,  32,  96],
[ 81,   0,   8],
[144,   2,   0],
[176,   7,   3],
[244,   5,   0],
[217,  88,   0],
[255, 255, 255],
[ 70,   0, 118],
[238, 238, 238],
[245, 230,  55],
[213, 145,  19],
[ 83,  26,  26],
[  7,   7, 165],
[102,  33, 161],
[161,  57,  34],
[ 21,  91,  62],
[179, 177, 163],
[255, 184,  33],
[ 65,  86,  86],
[197, 140,  98],
[255, 143,   0],
[247, 248, 199],
[ 73,  58,  45],
[255, 231, 213]]
)
civColors = np.concatenate((civColors, np.random.rand(213, 3) * 255))
# Usually Free city civ index is 62, so in theory there can't be more than 62 civs major/minor
civColors[62, :] = np.array([25, 25, 25])
civColors[255, :] = np.array([0, 0, 255])  # sea level rise when no other owner
civColorsInner = np.copy(civColors)
civColorsMinor = np.copy(civColors)

civColorsPen = []
civColorsBrush = []
civColorsPenInner = []
civColorsBrushInner = []
civColorsBrushMinor = []
for color in civColors:
    qcolor = pg.mkColor(color)
    civColorsPen.append(pg.mkPen(qcolor, width=3))
    civColorsPenInner.append(pg.mkPen(qcolor, width=3))
    civColorsBrush.append(pg.mkBrush(qcolor))
    civColorsBrushInner.append(pg.mkBrush(qcolor))
    civColorsBrushMinor.append(pg.mkBrush(qcolor))

riverPen = pg.mkPen(pg.mkColor(np.array((45, 89, 120, 255))), width=4)

emptyBrush = pg.mkBrush(pg.mkColor(np.zeros(4, )))
blackBrush = pg.mkBrush(pg.mkColor(np.zeros(3,)))
emptyPen = pg.mkPen(pg.mkColor(np.zeros(4, )))
blackPen = pg.mkPen(pg.mkColor(np.array((24, 24, 24))), width=3)

FREE_CITY_IDX = 62

def parseLeader(leaderIn):
    cityState = False
    leader = leaderIn
    if leader[:10] == "MINOR_CIV_":
        cityState = True
        leader = "City State"
    else:
        leader = " ".join(x.capitalize() for x in leader.split("_"))
    return leader, cityState

def map_civ_colors(civdata):
    added_colors = []
    print(f"Civilization colors are determined by first-come-first-serve")
    print(f"according to the jerseys used in Prismatic - Color and Jersey Overhaul mod (version 28.01.2021)")
    print(f"https://steamcommunity.com/sharedfiles/filedetails/?id=1661785509")
    for i, civ in enumerate(civdata):
        try:
            colorset = False
            for ii in range(4):
                color = COLORS_PRISM[CIV_LEADER_COLORS[civ][ii*2]]
                colorInner = COLORS_PRISM[CIV_LEADER_COLORS[civ][ii*2+1]]
                if color not in added_colors:
                    colorset = True
                    added_colors.append(color)
                    print(f"{civ} border color set to {color}/{colorInner} (option #{ii})")
                    break
            if not colorset:
                for jj in range(int(len(CIV_OVERFLOW_COLORS)/2)):
                    color = COLORS_PRISM[CIV_OVERFLOW_COLORS[jj * 2]]
                    colorInner = COLORS_PRISM[CIV_OVERFLOW_COLORS[jj * 2 + 1]]
                    if color not in added_colors:
                        added_colors.append(color)
                        print(f"{civ} border color set to {color}/{colorInner} (overflow option #{jj})")
                        break
        except:
            for jj in range(int(len(CIV_OVERFLOW_COLORS) / 2)):
                color = COLORS_PRISM[CIV_OVERFLOW_COLORS[jj * 2]]
                colorInner = COLORS_PRISM[CIV_OVERFLOW_COLORS[jj * 2 + 1]]
                if color not in added_colors:
                    added_colors.append(color)
                    print(f"{civ} border color set to {color}/{colorInner} (overflow option #{jj})")
                    break
            # continue
        qcolor = pg.mkColor(color)
        qcolorInner = pg.mkColor(colorInner)
        civColors[i] = color
        civColorsInner[i] = colorInner
        civColorsPen[i] = pg.mkPen(qcolor, width=3)
        civColorsBrush[i] = pg.mkBrush(qcolor)
        civColorsPenInner[i] = pg.mkPen(colorInner, width=4)
        civColorsBrushInner[i] = pg.mkBrush(qcolorInner)

        if len(civ) > 10:
            if civ[:10] == "MINOR_CIV_":
                try:
                    city_state = " ".join(x.capitalize() for x in civ[10:].split("_"))
                    colorMinor = COLORS_PRISM[CS_COLOR_MAP[CS_TYPES[city_state]]]
                    qcolorMinor = pg.mkColor(colorMinor)
                except:
                    print("City state not found from mapping: {}".format(civ))
                    qcolorMinor = blackBrush
                    colorMinor = np.zeros(3,)
                civColorsBrushMinor[i] = pg.mkBrush(qcolorMinor)
                civColorsMinor[i] = colorMinor

def fileWorker(idx, filePath, fileCount):
    f = open(filePath, "rb")
    data = f.read()
    f.close()
    mainDecompressedData = decompress(data)
    # writeBinDataToFile(mainDecompressedData)
    civData = []
    leaderData = []
    notifications = []
    diploStates = []
    wars = []
    try:
        if idx == 0:
            civData, leaderData = get_civ_data(data)
        # if idx == fileCount - 1:  # All grievance data stored in save file
        wars = getWars(mainDecompressedData, idx)
        # map_civ_colors(civdata)
        tileData = save_to_map_json(mainDecompressedData, idx)
        cityData = getCityData(mainDecompressedData, idx)
        diploStates = getDiploStates(mainDecompressedData, idx)

        checkCityTileOwner(cityData, tileData)
        cityNameData = getCityNameData(mainDecompressedData, idx)
        reCityData = reorderedCityData(cityData)
        # notifications = getNotifications(mainDecompressedData)
        combineNames(cityData, reCityData, cityNameData, idx)
        getCityLocNames(cityData)
    except Exception as e:
        traceback.print_exc()
        print(e)
    return idx, tileData, cityData, civData, leaderData, notifications, diploStates, wars

def getPlayerID(tile):
    if tile["OwnershipBuffer"] >= 64:
        tileBufferData = tile["buffer"]
        return tileBufferData[-5]
    else:
        return -1

def checkCityTileOwner(cityData, tileData, updateWithTileOwner=True):
    for city in cityData["cities"]:
        pId = getPlayerID(tileData["tiles"][city["LocationIdx"]])
        if pId == 255:  # Submerged land bug, No man's land -> city has been wiped out? TODO: Actually terrain changes
            pId = -1
        city["tileOwner"] = pId
        if updateWithTileOwner:
            if pId != FREE_CITY_IDX and pId != city["CivIndex"]:  # and pId > 0:
                CivCityOrderIdx, CivCityOrderIdx1, oldIdx = cityHasExiststedAlreadyInCiv(city["CityName"], pId,
                                                                                cityData)
                if CivCityOrderIdx >= 0:
                    city["CivCityOrderIdx"] = CivCityOrderIdx
                    city["CivCityOrderIdx1"] = CivCityOrderIdx1
                    city["CivIndex"] = pId
                else:
                    maxCivCityOrderIdx, maxCivCityOrderIdx1 = findLastCivCityIdx(cityData, pId)
                    if countCityName(city["CityName"], cityData) == 1:  # City is actually wiped on other player's land?
                        #print(f"A city wiped out on other player's land at location hex {city['LocationIdx']}?")
                        city["CivIndex"] = -1
                        continue
                    city["CivCityOrderIdx"] = maxCivCityOrderIdx + 1
                    city["CivCityOrderIdx1"] = maxCivCityOrderIdx1 + 1
                    city["CivIndex"] = pId

def countCityName(cityName, cityData):
    count = 0
    for city in cityData["cities"]:
        if city["CityName"] == cityName:
            count += 1
    return count

def combineNames(cityData, reCityData, cityNameData, fileNum):
    count = 0
    cityNameIdxLast = len(cityNameData["cityNames"]) - 1
    # Free cities first from the end of the list
    freeCitys = []
    for city in reversed(reCityData):
        if city["tileOwner"] == FREE_CITY_IDX:
            cityData["cities"][city["OldIdx"]]["cityNameData"] = cityNameData["cityNames"][cityNameIdxLast - count]
            count += 1
            freeCitys.append(city)
        else:
            break

    skip = 0
    stop = False
    for idx, city in enumerate(reCityData[:len(reCityData)-count]):
        total = idx + skip
        if total > cityNameIdxLast:
            break
        candidate = cityNameData["cityNames"][total]
        while True:
            if candidate["CivCityOrderIdx"] == city["CivCityOrderIdx"]:
                cityData["cities"][city["OldIdx"]]["cityNameData"] = candidate
                if candidate["Orig"]:
                    if cityData["cities"][city["OldIdx"]]["CityName"] != candidate["CityName"]:
                        print(f"File #{fileNum} Warning: a bug?, city names should be same when using originals")
                break
            elif candidate["CivCityOrderIdx"] < city["CivCityOrderIdx"]:  # Free city?
                print(f"Warning unconfirmed condition for city name mapping, a bug?")
                break
                # if city in freeCitys:
                #     pass
            else:  # Captured city state?
                skip += 1
                total = idx + skip
                if total > cityNameIdxLast:
                    stop = True
                    break
                candidate = cityNameData["cityNames"][total]
        if stop:
            break

def getCityLocNames(cityData):
    for city in cityData["cities"]:
        if "cityNameData" in city:
            if city["cityNameData"]["Orig"]:
                cityKey = "_".join(x.upper() for x in city["CityName"].split(" "))
                if cityKey in CITY_NAMES:
                    city["cityLocData"] = CITY_NAMES[cityKey]

def reorderedCityData(cityData, removeGaps=True):
    reorderedCityData = []
    usedCities = []
    replacedCities = []
    for idx, city in enumerate(cityData["cities"]):
        if city["CivIndex"] < 0:
            continue
        CityName = city["CityName"]
        cityCopy = city.copy()
        cityCopy["OldIdx"] = idx
        # If city exists already remove it from list
        if CityName in usedCities:
            for idx2, insertedCity in enumerate(reorderedCityData):
                if insertedCity["CityName"] == CityName:
                    reorderedCityData.pop(idx2)
                    break
        else:
            usedCities.append(CityName)
        # Find a slot where to insert
        insertCity(reorderedCityData, cityCopy, replacedCities)

    replacedCities2 = []
    for cityName in replacedCities:
        buildOver = False
        for city in reorderedCityData:
            if city["CityName"] == cityName:
                break
        else:
            for idx, missingCity in enumerate(reversed(cityData["cities"])):
                if missingCity["CityName"] == cityName:
                    cityLoc = missingCity["LocationIdx"]
                    for cityForLocCheck in reversed(cityData["cities"]):
                        if cityName != cityForLocCheck["CityName"]:
                            if cityForLocCheck["LocationIdx"] == cityLoc:
                                buildOver = True
                    break
            else:
                print("Failed to find a missing city candidate at reorderedCityData")
                continue
            if buildOver:
                # print("New city on same tile")
                continue
            newCity = missingCity.copy()
            maxCivCityOrderIdx, maxCivCityOrderIdx1 = findLastCivCityIdx(cityData, missingCity["tileOwner"])
            newCity["OldIdx"] = len(cityData["cities"]) - idx - 1
            newCity["CivIndex"] = missingCity["tileOwner"]
            newCity["CivCityOrderIdx"] = maxCivCityOrderIdx + 1
            newCity["CivCityOrderIdx1"] = maxCivCityOrderIdx1 + 1

            insertCity(reorderedCityData, newCity, replacedCities2)

    # Remove gaps from CivCityOrderIdx
    if removeGaps:
        first = True
        CurrentCivIndex = 0
        CurrentCivCityOrderIdx = 0
        for city in reorderedCityData:
            if first:
                first = False
                CurrentCivIndex = city["CivIndex"]
                CurrentCivCityOrderIdx = city["CivCityOrderIdx"]
                # Start from 0
                if CurrentCivCityOrderIdx != 0:
                    city["CivCityOrderIdx"] = 0
                    city["CivCityOrderIdxOld"] = CurrentCivCityOrderIdx
                    CurrentCivCityOrderIdx = 0
                continue
            if city["tileOwner"] == FREE_CITY_IDX:
                continue
            if CurrentCivIndex == city["CivIndex"]:
                CurrentCivCityOrderIdx += 1
                if CurrentCivCityOrderIdx != city["CivCityOrderIdx"]:
                    city["CivCityOrderIdxOld"] = city["CivCityOrderIdx"]
                    city["CivCityOrderIdx"] = CurrentCivCityOrderIdx
            else:  # Next Civ
                CurrentCivIndex = city["CivIndex"]
                CurrentCivCityOrderIdx = city["CivCityOrderIdx"]
                # Start from 0
                if CurrentCivCityOrderIdx != 0:
                    city["CivCityOrderIdx"] = 0
                    city["CivCityOrderIdxOld"] = CurrentCivCityOrderIdx
                    CurrentCivCityOrderIdx = 0

    return reorderedCityData

def insertCity(reorderedCityData, city, replacedCities):
    for idx2, insertedCity in enumerate(reorderedCityData):
        insertOrReplace = compareCity(city, insertedCity)
        if insertOrReplace > 0:
            reorderedCityData.insert(idx2, city)
            break
        elif insertOrReplace == 0:
            replacedCities.append(reorderedCityData[idx2]["CityName"])
            reorderedCityData[idx2] = city
            break
    else:
        reorderedCityData.append(city)

def findLastCivCityIdx(cityData, civIdx):
    maxCivCityOrderIdx = -1
    maxCivCityOrderIdx1 = -1
    for city in cityData["cities"]:
        if city["CivIndex"] == civIdx:
            if maxCivCityOrderIdx < city["CivCityOrderIdx"]:
                maxCivCityOrderIdx = city["CivCityOrderIdx"]
            if maxCivCityOrderIdx1 < city["CivCityOrderIdx1"]:
                maxCivCityOrderIdx1 = city["CivCityOrderIdx1"]
    return maxCivCityOrderIdx, maxCivCityOrderIdx1

def cityHasExiststedAlreadyInCiv(CityName, CivIndex, cityData):
    CivCityOrderIdx = -1
    CivCityOrderIdx1 = -1
    oldIdx = -1
    for idx, city in enumerate(cityData["cities"]):
        if city["CivIndex"] == CivIndex:
            if city["CityName"] == CityName:
                CivCityOrderIdx = city["CivCityOrderIdx"]
                oldIdx = idx
            if CivCityOrderIdx1 < city["CivCityOrderIdx1"]:
                CivCityOrderIdx1 = city["CivCityOrderIdx1"]
    CivCityOrderIdx1 += 1
    return CivCityOrderIdx, CivCityOrderIdx1, oldIdx

def compareCity(city1, city2):
    CivIndex = city1["CivIndex"]
    CivCityOrderIdx = city1["CivCityOrderIdx"]
    tileOwner = city1["tileOwner"]
    CivIndex2 = city2["CivIndex"]
    CivCityOrderIdx2 = city2["CivCityOrderIdx"]
    tileOwner2 = city2["tileOwner"]

    # City 1 is not free city and city 2 is
    if tileOwner != FREE_CITY_IDX and tileOwner2 == FREE_CITY_IDX:
        return 1
    # City 1 is free city and city 2 is not
    elif tileOwner == FREE_CITY_IDX and tileOwner2 != FREE_CITY_IDX:
        return -1
    # Both are either free city or not #TODO: Check that the order is not time related with Free Cities
    else:
        # Civ index priority # 1
        if CivIndex < CivIndex2:
            return 1
        elif CivIndex > CivIndex2:
            return -1
        else:
            # City index priority # 1
            if CivCityOrderIdx < CivCityOrderIdx2:
                return 1
            elif CivCityOrderIdx > CivCityOrderIdx2:
                return -1
            # Both are same replace old index (player has lost an existing city)
            else:
                return 0

class GameDataHandler:
    def __init__(self, dataFolder, fileExt=".Civ6Save"):
        self.dataFolder = dataFolder
        self.recursive = False
        self.tileData = []
        self.cityData = []
        self.civData = []
        self.leaderData = []
        self.notifications = []
        self.diploStates = []
        self.wars = []
        self.incWars = []
        self.events = []
        self.events_orig = []
        self.borderColors = []
        self.borderColorsInner = []
        self.borderColorsSC = []
        self.cityColors = []
        self.envColors = []
        self.riverColors = []
        self.goodyHuts = []
        self.fileExt = fileExt
        self.pColors = civColors
        self.X = -1
        self.Y = -1
        self.neighbours_list = []
        self.majorCivs = 0
        self.minorCivs = 0
        self.cityCounts = []
        self.razedCityLocs = []
        self.civ_text = []
        self.civHexaCounts = []
        self.playersAlive = []
        self.minorOrigos = {}
        self.minorCivTypes = {}
        self.calculatingCivNames = False

    def parseData(self):
        snapshot = DirectorySnapshot(self.dataFolder, self.recursive)
        count = 0
        filePaths = []
        for filePath in sorted(snapshot.paths):
            if self.fileExt == os.path.splitext(filePath)[1]:
                count += 1
                filePaths.append(filePath)
        self.tileData = [None] * count
        self.cityData = [None] * count
        self.civData = [None] * count
        self.leaderData = [None] * count
        self.notifications = [None] * count
        self.diploStates = [None] * count
        self.wars = [None] * count

        # self.saveResult(fileWorker(0, filePaths[0]))
        # self.calcMajorCivs()

        t0 = time.time()
        pool = mp.Pool()
        fileCount = len(filePaths)
        for ii, filePath in enumerate(filePaths):
            pool.apply_async(fileWorker, args=(ii, filePath, fileCount), callback=self.saveResult)
            # self.saveResult(fileWorker(ii, filePath, fileCount))  # debugging single thread
        pool.close()
        pool.join()
        # unique_notifications = self.checkUniqueNotifications()

        self.X, self.Y = self.getMapSize()
        self.neighbours_list = []
        for ii in range(self.X*self.Y):
            self.neighbours_list.append(self.getNeighbourIndexes(ii))

        self.calcMajorCivs()
        self.calcDiploStateWarPeaceDiff()
        self.calcCityCounts()
        self.calculateCivHexas()
        self.calcPlayersAlive()
        self.calculateCityStateOrigos()
        self.calcRazedCitys()
        self.calcIncrementalWars()
        print("Total time {} s for data parsing from {} files".format(time.time() - t0, count))

    def createEvents(self):
        self.events = []
        self.createWarEvents()
        self.createPeaceEvents()
        self.createWinningConditionEvents()
        self.createCityEvents()
        self.createWonderEvents()
        self.sortEvents()
        self.events_orig = self.events.copy()

    def filterEvents(self, filter_rules):
        self.events = self.events_orig.copy()
        for event in reversed(self.events):
            if event["Type"] in filter_rules:
                if filter_rules[event["Type"]]:
                    self.events.remove(event)

    def createPeaceEvents(self):
        colorhex = ''.join([format(int(c), '02x') for c in COLORS_PRISM["COLOR_STANDARD_WHITE_LT"]])
        peaceIcon = "<font color=#" + colorhex + "> \U0001F54A </font>"  # Dove, as white if no colored icon support

        pCount = self.majorCivs + self.minorCivs
        for idx in range(self.warPeaceDiffTable.shape[-1]):
            for p1 in range(pCount):
                for p2 in range(p1 + 1, pCount):
                    if self.warPeaceDiffTable[p1][p2][idx] == -1:
                        peaceType = "Peace Major"
                        if p1 >= self.majorCivs and p2 >= self.majorCivs:
                            peaceType = "Peace Minor Minor"
                        elif p1 >= self.majorCivs or p2 >= self.majorCivs:
                            peaceType = "Peace Minor"
                        event_txt = "[" + str(idx + 1) + "]: " + self.civ_text[p1].replace("<br>", "") + \
                                    peaceIcon + self.civ_text[p2].replace("<br>", "")
                        event = {"TurnIdx": idx, "Type": peaceType, "Event": event_txt}
                        self.events.append(event)

    def createWinningConditionEvents(self):
        pass

    def createCityEvents(self):
        # Founded, captured, razed, revolt, flipped, freed etc...
        pass

    def createWonderEvents(self):
        pass

    def createWarEvents(self):
        colorhex = ''.join([format(int(c), '02x') for c in COLORS_PRISM["COLOR_STANDARD_RED_DK"]])
        warIcon = "<font color=#" + colorhex + "> \u2694 </font>"  # Crossed swords, as red if no colored icon support
        for ii, turn in enumerate(self.incWars):
            for war in turn:
                attIdx = war["Att"]
                defIdx = war["Def"]
                warType = "War Major"
                if attIdx >= self.majorCivs and defIdx >= self.majorCivs:
                    warType = "War Minor Minor"
                elif attIdx >= self.majorCivs or defIdx >= self.majorCivs:
                    warType = "War Minor"
                event_txt = "[" + str(ii + 1) + "]: " + self.civ_text[attIdx].replace("<br>", "") +\
                            warIcon + self.civ_text[defIdx].replace("<br>", "")
                event = {"TurnIdx": ii, "Type": warType, "Event": event_txt}
                self.events.append(event)

    def sortEvents(self):
        def sortEventFunc(e):
            return e["TurnIdx"]
        self.events.sort(key=sortEventFunc)

    def checkUniqueNotifications(self):
        unique_notifications = []
        for turn in self.notifications:
            for notification in turn:
                if notification["NotiName"] not in unique_notifications:
                    unique_notifications.append(notification["NotiName"])
        return unique_notifications

    def saveResult(self, result):
        self.tileData[result[0]] = result[1]
        self.cityData[result[0]] = result[2]
        self.civData[result[0]] = result[3]
        self.leaderData[result[0]] = result[4]
        self.notifications[result[0]] = result[5]
        self.diploStates[result[0]] = result[6]
        self.wars[result[0]] = result[7]

    def calcCityCounts(self):
        self.cityCounts = []
        for i, turn in enumerate(self.cityData):
            cityCounts = [0] * self.majorCivs
            usedCities = {}
            for city in turn["cities"]:
                civIndex = city["CivIndex"]
                if civIndex >= self.majorCivs:  # Skip if city state has been liberated
                    continue
                cityCounts[civIndex] += 1
                if city["CityName"] not in usedCities:
                    usedCities[city["CityName"]] = city["CivIndex"]
                else:
                    cityCounts[usedCities[city["CityName"]]] -= 1
            self.cityCounts.append(cityCounts)

    def calcRazedCitys(self):
        self.razedCityLocs = []
        for i, turn in enumerate(self.cityData):
            razedCitysAtTurn = []
            # For minor player ruins
            for minor in self.minorOrigos:
                if not self.playersAlive[i][minor]:
                    loc = self.minorOrigos[minor]
                    for cityNewer in turn["cities"]:
                        if loc == cityNewer["LocationIdx"] and cityNewer["CivIndex"] >= 0:
                            break
                    else:
                        # No newer city exists -> razed
                        razedCitysAtTurn.append(loc)
            # For major player ruins
            for idx, city in enumerate(turn["cities"]):
                if city["CivIndex"] < 0:
                    loc = city["LocationIdx"]
                    if idx + 1 >= len(turn["cities"]):
                        razedCitysAtTurn.append(loc)
                        continue
                    for cityNewer in turn["cities"][idx+1:]:
                        if loc == cityNewer["LocationIdx"] and cityNewer["CivIndex"] >= 0:
                            break
                    else:
                        # No newer city exists -> razed
                        razedCitysAtTurn.append(loc)
            self.razedCityLocs.append(razedCitysAtTurn)

    def calculateOtherStuff(self):
        t0 = time.time()
        for turnIdx, turn in enumerate(self.tileData):
            goodyHutsAtTurn = []
            for ii, tile in enumerate(turn["tiles"]):
                terrainType = tile["TerrainType"]
                featureType = tile["FeatureType"]
                GoodyHut = tile["GoodyHut"]
                try:
                    if Features[GoodyHut]["FeatureType"] == "GoodyHut" or \
                            Features[GoodyHut]["FeatureType"] == "BarbCamp":
                        goodyHutsAtTurn.append(Features[GoodyHut]["color"])
                    else:
                        goodyHutsAtTurn.append(emptyBrush)
                except:
                    print("Unknown feature: turnIdx: {}, x: {}, y: {}, goodyHut: {}".format(turnIdx, tile["x"], int(tile["y"]), GoodyHut))
                    goodyHutsAtTurn.append(emptyBrush)
            self.goodyHuts.append(copy.copy(goodyHutsAtTurn))
        print("Total time for goody huts / barb camps: {}".format(time.time() - t0))

    def calculateEnvColors(self):
        t0 = time.time()
        if len(self.tileData) != 0:
            turn = self.tileData[0]
            count = 0
            for ii, tile in enumerate(turn["tiles"]):
                terrainType = tile["TerrainType"]
                featureType = tile["FeatureType"]
                try:
                    if Features[featureType]["FeatureType"] == "Pamukkale":
                        self.envColors.append(Features[featureType]["color"])
                    elif (Terrains[terrainType]["TerrainType"] == "Ocean" or
                            Terrains[terrainType]["TerrainType"] == "Coast"):
                        if Features[featureType]["FeatureType"] == "Ice":
                            self.envColors.append(Features[featureType]["color"])
                        else:
                            self.envColors.append(Terrains[terrainType]["color"])
                    else:
                        self.envColors.append(Terrains[terrainType]["color"])
                except:
                    count += 1
                    print("errorCount: {}, x: {}, y: {}, terrainType: {}, featureType: {}".format(count, tile["x"], tile["y"], terrainType, featureType))
                    self.envColors.append(pg.mkBrush(pg.mkColor(np.zeros(3,))))
        print("Total time for environment colors: {}".format(time.time() - t0))

    def calculateRiverColors(self, lw=4):
        t0 = time.time()
        if len(self.tileData) != 0:
            turn = self.tileData[0]
            for ii, tile in enumerate(turn["tiles"]):
                RiverBorders = tile["RiverBorders"]
                if RiverBorders > 0:
                    RiverBitMap = tile["RiverBitMap"]
                    for ii in range(6):
                        if RiverBitMap >> ii & 1:
                            self.riverColors.append(riverPen)
                        else:
                            self.riverColors.append(emptyPen)
                else:
                    for jj in range(6):
                        self.riverColors.append(emptyPen)
        print("Total time for river colors: {}".format(time.time() - t0))

    def calculateCivHexas(self):
        civNum = self.majorCivs + self.minorCivs
        self.civHexaCounts = []
        for turn in self.tileData:
            civHexaCountsAtTurn = [0] * civNum
            for ii, tile in enumerate(turn["tiles"]):
                playerID = getPlayerID(tile)
                if 0 <= playerID < civNum:
                    civHexaCountsAtTurn[playerID] += 1
            self.civHexaCounts.append(civHexaCountsAtTurn)

    def calcPlayersAlive(self):
        playersNum = self.majorCivs + self.minorCivs
        self.playersAlive = []
        for hexaCounts, cityCounts in zip(self.civHexaCounts, self.cityCounts):
            temp = [x > 0 for x in hexaCounts]
            for i in range(self.majorCivs):
                if temp[i]:
                    continue
                elif cityCounts[i] > 0:
                    temp[i] = True
            self.playersAlive.append(temp)
        turns = len(self.playersAlive)
        for i in range(playersNum):  # For each player
            alive_from_start = False
            for j in range(turns):  # Check when first alive
                if self.playersAlive[j][i]:
                    alive_from_start = True
                if alive_from_start:  # Fill from beginning until first "alive"
                    for k in range(j):
                        self.playersAlive[k][i] = True
                    break

    def calculateBorderColors(self, lw=3, outsideBordersOnly=False, use_civ_colors=True, drawWaterBorders=True):
        if use_civ_colors:
            map_civ_colors(self.leaderData[0])
        t0 = time.time()
        self.borderColors = []
        self.borderColorsInner = []
        self.borderColorsSC = []
        for turn in self.tileData:
            borderColorsAtTurn = []
            borderInnerColorsAtTurn = []
            borderSCColorsAtTurn = []
            for ii, tile in enumerate(turn["tiles"]):
                if not drawWaterBorders:
                    terrainType = tile["TerrainType"]
                    try:
                        if (Terrains[terrainType]["TerrainType"] == "Ocean" or
                                Terrains[terrainType]["TerrainType"] == "Coast"):
                            if outsideBordersOnly:
                                for jj in range(6):
                                    borderColorsAtTurn.append(emptyPen)
                                    borderInnerColorsAtTurn.append(emptyPen)
                                    borderSCColorsAtTurn.append(emptyPen)
                            else:
                                borderColorsAtTurn.append(emptyBrush)
                                borderInnerColorsAtTurn.append(emptyBrush)
                                borderSCColorsAtTurn.append(emptyBrush)
                            continue
                    except:
                        print("drawWaterBorders failure ...")
                        pass
                playerID = getPlayerID(tile)
                if playerID >= 0:
                    if outsideBordersOnly:
                        for neighbour in self.neighbours_list[ii]:
                            if neighbour < self.X*self.Y:
                                neighbourID = getPlayerID(turn["tiles"][neighbour])
                                if neighbourID == playerID:
                                    if not drawWaterBorders:
                                        terrainType = turn["tiles"][neighbour]["TerrainType"]
                                        if (Terrains[terrainType]["TerrainType"] == "Ocean" or
                                                Terrains[terrainType]["TerrainType"] == "Coast"):
                                            borderColorsAtTurn.append(civColorsPen[playerID])
                                            borderInnerColorsAtTurn.append(civColorsPenInner[playerID])
                                            if 255 > playerID >= self.majorCivs:
                                                borderSCColorsAtTurn.append(blackPen)
                                            else:
                                                borderSCColorsAtTurn.append(emptyPen)
                                        else:
                                            borderColorsAtTurn.append(emptyPen)
                                            borderInnerColorsAtTurn.append(emptyPen)
                                            borderSCColorsAtTurn.append(emptyPen)
                                    else:
                                        borderColorsAtTurn.append(emptyPen)
                                        borderInnerColorsAtTurn.append(emptyPen)
                                        borderSCColorsAtTurn.append(emptyPen)
                                else:
                                    borderColorsAtTurn.append(civColorsPen[playerID])
                                    borderInnerColorsAtTurn.append(civColorsPenInner[playerID])
                                    if 255 > playerID >= self.majorCivs:
                                        borderSCColorsAtTurn.append(blackPen)
                                    else:
                                        borderSCColorsAtTurn.append(emptyPen)
                            else:
                                borderColorsAtTurn.append(civColorsPen[playerID])
                                borderInnerColorsAtTurn.append(civColorsPenInner[playerID])
                                if 255 > playerID >= self.majorCivs:
                                    borderSCColorsAtTurn.append(blackPen)
                                else:
                                    borderSCColorsAtTurn.append(emptyPen)
                    else:
                        borderColorsAtTurn.append(civColorsBrush[playerID])
                        borderInnerColorsAtTurn.append(emptyBrush)  # no inner
                        borderSCColorsAtTurn.append(emptyBrush)  # no inner
                else:
                    if outsideBordersOnly:
                        for jj in range(6):
                            borderColorsAtTurn.append(emptyPen)
                            borderInnerColorsAtTurn.append(emptyPen)
                            borderSCColorsAtTurn.append(emptyPen)
                    else:
                        borderColorsAtTurn.append(emptyBrush)
                        borderInnerColorsAtTurn.append(emptyBrush)
                        borderSCColorsAtTurn.append(emptyBrush)
            self.borderColors.append(borderColorsAtTurn)
            self.borderColorsInner.append(borderInnerColorsAtTurn)
            self.borderColorsSC.append(borderSCColorsAtTurn)
        print("Total time for border colors: {}".format(time.time() - t0))

    def calculateCityStateOrigos(self):
        whined = False
        t0 = time.time()
        # self.borderColorsInner = []
        self.minorOrigos = {}
        for turn in self.tileData:
            for ii, tile in enumerate(turn["tiles"]):
                playerID = getPlayerID(tile)
                if playerID >= self.majorCivs and playerID not in self.minorOrigos:  # Minor only and not found yet
                    neighbour_count_inv = 6
                    found_orig = True
                    for neighbour in self.neighbours_list[ii]:  # If more than 4 are owned (or all actually)
                        if neighbour < self.X*self.Y:
                            neighbourID = getPlayerID(turn["tiles"][neighbour])
                            if neighbourID != playerID:
                                neighbour_count_inv -= 1
                        if neighbour_count_inv <= 3:
                            found_orig = False
                            break
                    if found_orig:
                        if playerID not in self.minorOrigos and playerID != FREE_CITY_IDX:
                            self.minorOrigos[playerID] = ii
                        else:
                            if not whined:
                                whined = True
                                print(f"Warning: CityState(s) location calculation possibly failed! Affects visually only!\n"
                                      f"This happens if not starting from turn #1 (or maybe some mod)!")
        print("Total time for city state origos: {}".format(time.time() - t0))

    def calcIncrementalWars(self):
        used_wars = []
        self.incWars = []
        for turn in self.wars:
            new_wars = []
            for war in turn:
                war_copy = {"Att": war["Att"], "Def": war["Def"], "Turn": war["Turn"]}
                war_inverse = {"Att": war["Def"], "Def": war["Att"], "Turn": war["Turn"]}
                if war_copy not in used_wars and war_inverse not in used_wars:
                    new_wars.append(war_copy)
                    used_wars.append(war_copy)
            self.incWars.append(new_wars)

    def findMinorAllies(self, pIdx, turnIdx):
        diploAtTurnIdx = self.diploStates[turnIdx]
        pCount = self.majorCivs + self.minorCivs
        allies = []
        for minor in range(self.majorCivs, pCount):
            if diploAtTurnIdx[pIdx][minor]["state"][:3] == "MAX":
                allies.append(minor)
        return allies

    def calcDiploStateWarPeaceDiff(self):
        pCount = self.majorCivs + self.minorCivs
        # Fixing invisible minorCivs with diploStates, TODO: might be better to move this to somewhere else
        M = len(max(self.diploStates, key=len))
        if 62 in self.diploStates[0]:
            M -= 1
        if M < pCount:
            self.minorCivs = M - self.majorCivs
            pCount = M
        diploDiffsWars = np.zeros((pCount, pCount, len(self.diploStates)), dtype=np.int8)
        for idx, diploAtTurnIdx in enumerate(self.diploStates):
            pCount = len(diploAtTurnIdx)  # TODO: Remove FREE_CITIES from diploStates in first place?
            if 62 in diploAtTurnIdx:  # Remove FREE_CITIES from turn player count at turn also
                pCount -= 1
            for p1 in range(pCount):
                for p2 in range(pCount):
                    if diploAtTurnIdx[p1][p2]["state"][:3] == "WAR" or diploAtTurnIdx[p1][p2]["state"][-3:] == "WAR":
                        diploDiffsWars[p1][p2][idx] = 1
                        # MAX_INFLUENCE PATRON
        self.warPeaceDiffTable = np.diff(diploDiffsWars)  # starting from turn "2" (idx - 1)

    def getOwner(self, turnIdx, x, y, language=None):
        civ_text = ""
        civs = self.civData[0]
        leaders = self.leaderData[0]
        turn = self.tileData[turnIdx]
        if 0 < x <= self.X and 0 < y <= self.Y:
            tile = turn["tiles"][y * self.X + x]
            playerID = getPlayerID(tile)
            if playerID >= 0:
                if playerID < len(leaders):
                    leader = leaders[playerID]
                    leader_name, cityState = parseLeader(leader)
                    colorhex = ''.join([format(int(c), '02x') for c in civColors[playerID]])
                    colorhexInner = ''.join([format(int(c), '02x') for c in civColorsInner[playerID]])
                    civ = civs[playerID]
                    civ_name = " ".join(x.capitalize() for x in civ.split("_"))

                    civ_name, leader_name = self.languageChanger(language, civ, leader, cityState, civ_name,
                                                                 leader_name)

                    civ_text += "<font color=#" + colorhex + ">" + civ_name + "</font><br>"
                    civ_text += "<font color=#" + colorhexInner + ">" + leader_name + "</font>"
                elif playerID == 62:  # Free City
                    civ_text += "Free City"
                elif playerID == 255:
                    civ_text += "Coastal Flood"
        return civ_text

    def calcMajorCivs(self):
        leaders = self.leaderData[0]
        count = 0
        for leader in leaders:
            if leader[:10] == "MINOR_CIV_":
                break
            count += 1
        self.majorCivs = count
        self.minorCivs = len(leaders) - count
        self.minorCivTypes = {}
        for ii in range(count, len(leaders)):
            civ_name = " ".join(x.capitalize() for x in leaders[ii][10:].split("_"))
            try:
                self.minorCivTypes[ii] = CS_TYPES[civ_name]
            except:
                self.minorCivTypes[ii] = "Unknown"

    def languageChanger(self, language, civ, leader, cityState, civ_name, leader_name):
        if language != "en_EN" and language is not None:
            if civ in CIV_NAMES:
                if language in CIV_NAMES[civ]:
                    civ_name = CIV_NAMES[civ][language]
            if leader in CIV_LEADER_NAMES and not cityState:
                if language in CIV_LEADER_NAMES[leader]:
                    leader_name = CIV_LEADER_NAMES[leader][language]
        return civ_name, leader_name

    def parseCivNames(self, language=None):
        self.calculatingCivNames = True
        civs = self.civData[0]
        leaders = self.leaderData[0]
        self.civ_text = []
        for i, civ in enumerate(civs):
            colorhex = ''.join([format(int(c), '02x') for c in civColors[i]])
            colorhexInner = ''.join([format(int(c), '02x') for c in civColorsInner[i]])
            leader = leaders[i]
            leader_name, cityState = parseLeader(leader)
            civ_name = " ".join(x.capitalize() for x in civ.split("_"))

            civ_name, leader_name = self.languageChanger(language, civ, leader, cityState, civ_name, leader_name)

            # \u2b22 hexa
            if cityState:
                colorhexMinor = ''.join([format(int(c), '02x') for c in civColorsMinor[i]])
                try:
                    symbol = CS_UNICODE_MAP[self.minorCivTypes[i]]
                except:
                    symbol = " \u2b22  "
                self.civ_text.append("<font color=#" + colorhexMinor + ">" + symbol + "</font>" +
                                     "<font color=#" + colorhex + ">" + civ_name + "</font> " +
                                     "<font color=#" + colorhexInner + ">" + "CS " + "</font><br>")
            else:
                self.civ_text.append("<font color=#" + colorhex + ">" + civ_name + "</font> - " +
                                     "<font color=#" + colorhexInner + ">" + leader_name + "</font><br>")
        self.calculatingCivNames = False

    def getCivNames(self, turnIdx):
        playersRemaining = self.playersAlive[turnIdx]
        civ_text = ""
        if not self.calculatingCivNames:
            for i, alive in enumerate(playersRemaining):
                # First minor civ
                if i == self.majorCivs:
                    civ_text += "<br>"
                if alive:
                    civ_text += self.civ_text[i]
        return civ_text

    def calculateCityColors(self, useInnerAsCityColor=True, useMinorType=False):
        t0 = time.time()
        cityColorsAtTurnEmpty = [emptyBrush] * self.X*self.Y
        self.cityColors = []
        for ii, turn in enumerate(self.cityData):
            cityColorsAtTurn = cityColorsAtTurnEmpty.copy()
            for minor in self.minorOrigos:
                if self.playersAlive[ii][minor] and ii != 0:
                    if useMinorType:
                        cityColorsAtTurn[self.minorOrigos[minor]] = civColorsBrushMinor[minor]
                    else:
                        cityColorsAtTurn[self.minorOrigos[minor]] = civColorsBrushInner[minor]
            for city in turn["cities"]:
                if city["CivIndex"] >= 0:
                    if useInnerAsCityColor:
                        cityColorsAtTurn[city["LocationIdx"]] = civColorsBrushInner[city["CivIndex"]]
                    else:
                        cityColorsAtTurn[city["LocationIdx"]] = civColorsBrush[city["CivIndex"]]
            self.cityColors.append(cityColorsAtTurn)
        print("Total time for city colors: {}".format(time.time() - t0))

    def calculateMinorCityColors(self):
        t0 = time.time()
        cityColorsAtTurnEmpty = [emptyBrush] * self.X*self.Y
        self.minorCityColors = []
        for ii, turn in enumerate(self.cityData):
            cityColorsAtTurn = cityColorsAtTurnEmpty.copy()
            for minor in self.minorOrigos:
                if self.playersAlive[ii][minor]:
                    cityColorsAtTurn[self.minorOrigos[minor]] = civColorsBrushMinor[minor]
            self.minorCityColors.append(cityColorsAtTurn)
        print("Total time for minor city colors: {}".format(time.time() - t0))

    def getMapSize(self):
        if len(self.tileData) != 0:
            return self.tileData[0]["mapSize"][0], self.tileData[0]["mapSize"][1]
        else:
            return None

    def getTurnCount(self):
        return len(self.tileData)

    def index2XY(self, index):
        y = int(np.floor(index / self.X))
        x = int(index % self.X)
        if 0 <= y < self.Y:
            return x, y
        else:
            return self.X, self.Y

    #   5   0
    # 4   x   1
    #   3   2
    def getNeighbourIndexes(self, index):
        neighbours = np.array([index]*6)
        nanvalue = self.X*self.Y
        x, y = self.index2XY(index)

        if y % 2 == 0:
            offsets = np.array([self.X, 1, -self.X, -self.X-1, -1, self.X-1])
        else:
            offsets = np.array([self.X+1, 1, -self.X+1, -self.X, -1, self.X])

        neighbours += offsets
        if y == self.Y - 1:
            # Top row, -> no 0, 5 neighbours
            neighbours[0] = nanvalue
            neighbours[5] = nanvalue
        elif y == 0:
            # Bottom row, -> no 2, 3 neighbours
            neighbours[2] = nanvalue
            neighbours[3] = nanvalue
        if x % self.X == 0:
            # First column, [4] += self.X
            neighbours[4] += self.X
            if (0 < y < self.Y - 1) and (y % 2 == 0):
                # If not top/bottom row, and even row -> [3/5] += self.X
                neighbours[3] += self.X
                neighbours[5] += self.X
        elif x % self.X == self.X - 1:
            # Last column, [1] -= self.X
            neighbours[1] -= self.X
            if (0 < y < self.Y - 1) and (y % 2 == 1):
                # If not top/bottom row, and uneven row -> [0/2] -= self.X
                neighbours[0] -= self.X
                neighbours[2] -= self.X
        return neighbours

    def randomCivColors(self, N):
        oldColors = self.pColors
        self.pColors = np.random.rand(N, 3)
        for borderColorsAtTurn in self.borderColors:
            for ii, color in enumerate(borderColorsAtTurn):
                idx = np.where((oldColors[:, 0] == color[0]) & (oldColors[:, 1] == color[1]) & (oldColors[:, 2] == color[2]))[0]
                if len(idx) > 0:
                    borderColorsAtTurn[ii] = np.append(self.pColors[idx], 0.9)

# Python program to detect cycle
# in a graph
class Graph:
    def __init__(self, vertices):
        self.graph = defaultdict(list)
        self.V = vertices

    def addEdge(self, u, v):
        self.graph[u].append(v)

    def isCyclicUtil(self, v, visited, recStack):

        # Mark current node as visited and
        # adds to recursion stack
        visited[v] = True
        recStack[v] = True

        # Recur for all neighbours
        # if any neighbour is visited and in
        # recStack then graph is cyclic
        for neighbour in self.graph[v]:
            if visited[neighbour] == False:
                if self.isCyclicUtil(neighbour, visited, recStack) == True:
                    return True
            elif recStack[neighbour] == True:
                return True

        # The node needs to be poped from
        # recursion stack before function ends
        recStack[v] = False
        return False

    # Returns true if graph is cyclic else false
    def isCyclic(self):
        visited = [False] * self.V
        recStack = [False] * self.V
        for node in range(self.V):
            if visited[node] == False:
                if self.isCyclicUtil(node, visited, recStack) == True:
                    return True
        return False


#!/usr/bin/env python
# coding: utf-8

# In the [last notebook](https://github.com/caiomiyashiro/RecommenderSystemsNotebooks/blob/master/Month%202%20Part%20I%20-%20User%20User%20Collaborative%20Filtering.ipynb), we took a look at the Nearest Neighboor User User CF, a form of recommender system that looked at the similarity between users to define which item should be suggested to a new customer.
#
# We saw the benefits from User User CF when comparing it to non personalised and content based recommendations, but we also saw that it comes with one difficulty.
#
# - It doesn't scale well. Even in a big e-commerce dataset, the amount of intersecting items between 2 users is not as big as it could be when User User CF was created. Because of this, when a user bought an item that intersected now with another different customer, the new similarity could drastically change, causing the system's owners to recalculate the new similarities very often.
#
# - If they don't update the matrix often, they can also lose profits over it because it wouldn't map the user's short term interests which, on the internet, can vary quite a lot.
#
# All in all, short term interest and sparse mutual interest space make the User User CF inapt for high scale companies.
#
# # Item Item Collaborative Filtering (CF)
#
# [Item Item CF](https://en.wikipedia.org/wiki/Item-item_collaborative_filtering) was created by ([Sarwar et all, 1998](https://patentimages.storage.googleapis.com/41/80/fb/07d4d9e61e7431/US6266649.pdf)) in partneship with Amazon in order to fix the problems with the User User CF. In Item Item perspective, as the name suggests, changes the perspective from User centered to a Item centered view, *i.e.*, instead of having a User User similarity matrix, they started to use a item item similarity matrix. Then, when a user $u$ bought and liked an item $i_{1}$ and $i_{1}$ was similar to item $i_{2}$, then we predicted that $u$ would also like $i_{2}$. Take a look at the image below:
#
#
#
# Why this simple change in perspective helped to solve the inneficiency problems present in the User User CF?
#
# By considering an enviroment of a big e-commerce company, we end with the number of users >> number of items.
# In this case, even if a single user hasn't given many reviews, the chances are that many users have given a review to a specific item.
# By having a big number of reviews, an item relationship to other items doesn't change too much by receiving a few more reviews, *i.e.*, item item relationship are more stable. Therefore, by being more stable, the similarity matrix doesn't have to be recalculated often, as in the User User CF.
#
# An extra perfomance improvement comes also from the prediction calculation. In the Item Item CF, a new prediction for a user $u$ for a product $p$ is made by retrieving the items similarities and calculating a weighted average. The number of neighboors for this calculation is only the item that $u$ has liked or bought in the past and this number is often small enought. Therefore, we don't need to search the big user user similarity matrix to find the best $k$ neighboors.
#
# # Item Item Steps
#
# As always, we're going to work with one of the datasets from the [Coursera's Specialization on Recommender Systems](https://www.coursera.org/specializations/recommender-systems). This dataset is from the last week in the course of [Nearest Neighboors CF](https://www.coursera.org/learn/collaborative-filtering) for Item Item CF. Dataset is [here](https://d396qusza40orc.cloudfront.net/umntestsite/on-demand_files/A5/Assignment%205.xls) (Coursera's page) and [here](https://github.com/caiomiyashiro/RecommenderSystemsNotebooks/blob/master/data/Item%20Item%20Collaborative%20Filtering%20-%20Ratings.csv) (personal Github account).
#
# The steps taken to evaluate and recommend are similar to User User CF, with some different calculations in the prediction step, as we've said.
#
# - Load traditional input - User Item Review dataset
# - Create similarity matrix
# - Make predictions
#
# Lets go!
#
# ## Example Dataset
#
# The dataset is a matrix with size 25 users x 25 movies and each cell $c_{u,m}$ contains the rating user $u$ gave to movie $m$. If user $u$ didn't rate movie $m$, the cell is empty. As the float values were stored with commas and consequently were being casted as strings, I had to process it a little bit to replace the commas for dots and then convert the column to floats

# In[1]:

import pandas as pd
import numpy as np

# In[2]:

df = pd.read_csv('data/Item Item Collaborative Filtering - Ratings.csv', index_col=0, nrows=20)

df.drop('Mean', axis=1, inplace=True) # remove mean column that comes at the end

# replace commas for dots and convert previous string column into float
def processCol(col):
    return col.astype(str).apply(lambda val: val.replace(',','.')).astype(float)
df = df.apply(processCol)

print('Dataset shape: ' + str(df.shape))
df.head()

# ## Create Similarity Matrix
#
#
# ### Similarity Function
#
# As for the User User CF, we have a few possibilities to choose from when deciding how we're going to define if an item is similar to another item. Again, ([Herlocker et all, 2002](https://grouplens.org/site-content/uploads/evaluating-TOIS-20041.pdf)) did an analysis on the performance of these metrics on Item Item CF and realised that, for this case, the [cosine similarity](https://en.wikipedia.org/wiki/Cosine_similarity) was the best performant metric. So we're going with them this time. On the **next notebook**, we try to analyse these metrics and see why it performs better in certain cases and others not.
#
#
# ## Calculating User User Similarity with Cosine Similarity:
#
# One important point here is on the calculation of the denominator of the cosine similarity. Even though we make the dot product only with values existing in both arrays, the norm of the individual vectors are considering all values, and not the intersections between array1 and array2.

# In[3]:

def cos_similarity(item1, item2):
    item1Values = ~np.isnan(item1)
    item2Values = ~np.isnan(item2)
    allValues = np.logical_and(item1Values,item2Values) # get only existent elements of both vectors
    return np.dot(item1[allValues], item2[allValues])/(np.linalg.norm(item1[item1Values]) * np.linalg.norm(item2[item2Values]))

def pre_cos_similarity(item1, df):
    return df.apply(lambda item2: cos_similarity(item1, item2))

df_corr = df.apply(lambda item1: pre_cos_similarity(item1, df))
df_corr.head()

# ## Predictions Calculation
#
# By now we already know which items are more similar to each other. This will help us when predicting a new rating, by giving higher weights for more similar items than other the user has bough.
#
# The way we're going to calculate the new predictions is the same we used for User User CF, *i.e.*, a weighted average:
#
# $$\frac{\sum_{n=1}^{k} r_{n}w_{n}}{\sum_{n=1}^{k} w_{n}}$$
#
# The difference is that we don't have the neighboors anymore, so the $n$ in the summation is considering **all** the items user $u$ has rated and $w$ is still the similarities, but now *item similarity*.

# In[4]:

def predictRating(userRatings, itemSimilarity):
    userHasRating = ~np.isnan(userRatings)
    return np.dot(userRatings[userHasRating], itemSimilarity[userHasRating])/np.sum(itemSimilarity[userHasRating])

def pre_predictRating(userRatings, df_corr):
    return df_corr.apply(lambda itemSimilarity: predictRating(userRatings, itemSimilarity))

predictions = df.apply(lambda userRatings: pre_predictRating(userRatings, df_corr), axis=1)
predictions.head()

# ## Mean Normalised Weighted Average
#
# As in the same way of the User User CF, we can calculate predictions using the absolute value of the reviews or from the mean centralised values of it. The advantages are the same: consider the scale variability of reviewers when attributing a final score for a item of interest:
#
# $$\bar{r_{u}} + \frac{\sum_{n=1}^{k} (r_{n} - \bar{r_{n}})w_{n}}{\sum_{n=1}^{k} w_{n}}$$
#
# We took the same function as above, but added two extra parameters:
# - $userMeanRating$: mean average ratings for a specific user
# - $neighboorsMeanRating$: mean average rating for all the nearest neighboors for a specific user

# In[5]:

# mean normalise
def subtractFromMean(col, meanCol):
    result = np.array([np.nan] * col.shape[0])
    isValidValue = ~np.isnan(col)
    result[isValidValue] = col.values[isValidValue] - meanCol.values[isValidValue]
    return result
userMeanRatings = df.apply(np.mean, axis=1)
df_ratings_norm = df.apply(lambda col: subtractFromMean(col, userMeanRatings))

# similarity matrix
df_corr_norm = df_ratings_norm.apply(lambda item1: pre_cos_similarity(item1, df_ratings_norm))

# ### Remove negative correlations
#
# In this example, we are replacing the negative correlations by 0, as we can interpret as a maximum weight for unwanted items:
#

# In[6]:

def replaceNegative(col):
    col[col < 0] = 0
    return col
df_corr_norm2 = df_corr_norm.apply(replaceNegative)

# ### Predict!

# In[7]:

predictions_norm = df.apply(lambda userRatings: pre_predictRating(userRatings, df_corr_norm2), axis=1)
predictions_norm.head()

# * I didn't quite understand why we didn't use the not normalised item's ratings in this calculation. Ideally, we would use the mean centered user ratings as well..

# # Comparison Between Approaches
#
# The comparison follows the same guidelines we used when evaluating the User User CF.
#
#
# # Final Considerations on User User CF
#
# When we mean centered the user's rating for the User User CF, the objective was clear, we wanted to take into account that users rate in different parts of the scale. But what about mean centering for the Item Item CF? This evaluation I'll leave it to the next notebook, where we evaluate and compare the main metrics used for similarity calculation in a CF system.
#
# --
#
# Item Item CF brings efficiencies steps forward from the User User CF schema. With it, we bring personalised recommendations and in a way that is computationally efficient to scale for giant e-commerce companies, such as Amazon or Netflix. But Item Item CF isn't a gold system, where we can implement it and always get good results. It has a few premisses:
#
# - First, it has the premisse that number of users >> number of items. This is a prerequisite to have stable entities, items in this case, and doesn't need to recalculate the similarity matrix often, as in the User User CF.
#
#
# - Secondly, and this is an interesting feature, Item Item CF is better when the item ratings are stable, *i.e.*, they have lots of evaluations. This means that the user's items are probably going to have a lot of influence from these popular items and, at the end, receiving popular items recommendations. This is good when you want to be safe about your recommendations, such as expensive services or products or rarely bought, such as houses or cars. However, this lack of '*serendipity*' is missed when we want to enable users to find that particular rare item and amazingly matched with your tastes. As an example, If we take Spotify, we don't want to receive recommendations such as 'Hey, as you listened to Mozart, here is what we think you'd like: Bach'. Spotify greatness works on the premisse of finding the bands and songs that can surprise you, so they wouldn't be effective by working on the Item Item CF schema. Of course, we are going to see more advance techniques in the future where these companies apply modern algorithms to have good recommendations and still be performatic, but the idea now was to show how we can't rely on one algorithm as the best of them all.
#
#
#
#
# In the [next notebook](https://github.com/caiomiyashiro/RecommenderSystemsNotebooks/blob/master/Month%202%20Part%20III%20-%20Notes%20on%20Similarity%20Metrics%20for%20CF.ipynb), we finalise the discussion over Collaborative Filtering by investigating a little more on how the similarity metrics work and try to find out some of its features such as:
#
# * Why pearson end up being better for User User CF and Cosine Similarity better for Item Item CF?
# * What are the strenghts and weakness when thinking on using one of the evaluated metrics?
# * Some filosophies on what they represent and how we can think about them geometrically
#
# Stay tuned :)


#!/usr/bin/env python
# coding: utf-8

#
#
# # Earth Analytics Education - EA  Python Course Spring 2021

# ## Important  - Assignment Guidelines
#
# 1. Before you submit your assignment to GitHub, make sure to run the entire notebook with a fresh kernel. To do this first, **restart the kernel** (in the menubar, select Kernel$\rightarrow$Restart & Run All)
# 2. Always replace the `raise NotImplementedError()` code with your code that addresses the activity challenge. If you don't replace that code, your notebook will not run.
#
# ```
# # YOUR CODE HERE
# raise NotImplementedError()
# ```
#
# 3. Any open ended questions will have a "YOUR ANSWER HERE" within a markdown cell. Replace that text with your answer also formatted using Markdown.
# 4. **DO NOT RENAME THIS NOTEBOOK File!** If the file name changes, the autograder will not grade your assignment properly.
# 6. When you create a figure, comment out `plt.show()` to ensure the autograder can grade your plots. For figure cells, DO NOT DELETE the code that says `DO NOT REMOVE LINE BELOW`.
#
# ```
# ### DO NOT REMOVE LINE BELOW ###
# student_plot1_ax = nb.convert_axes(plt)
# ```
#
# * Only include the package imports, code, and outputs that are required to run your homework assignment.
# * Be sure that your code can be run on any operating system. This means that:
#    1. the data should be downloaded in the notebook to ensure it's reproducible
#    2. all paths should be created dynamically using the `os.path.join`
#
# ## Follow to PEP 8 Syntax Guidelines & Documentation
#
# * Run the `autopep8` tool on all cells prior to submitting (HINT: hit shift + the tool to run it on all cells at once!
# * Use clear and expressive names for variables.
# * Organize your code to support readability.
# * Check for code line length
# * Use comments and white space sparingly where it is needed
# * Make sure all python imports are at the top of your notebook and follow PEP 8 order conventions
# * Spell check your Notebook before submitting it.
#
# For all of the plots below, be sure to do the following:
#
# * Make sure each plot has a clear TITLE and, where appropriate, label the x and y axes. Be sure to include UNITS in your labels.
#

# ### Add Your Name Below
# **Your Name:** Brian Pinke

#

# ---

# # Week 04 and 05 Homework - Automate NDVI Workflow
#
# For this assignment, you will write code to generate a plot of the mean normalized difference vegetation index (NDVI) for two different sites in the United States across one year of data:
#
# * San Joaquin Experimental Range (SJER) in Southern California, United States
# * Harvard Forest (HARV) in the Northeastern United States
#
# The data that you will use for this week is available from **earthpy** using the following download:
#
# `et.data.get_data('ndvi-automation')`
#
# ## Assignment Goals
#
# Your goal in this assignment is to create the most efficient and concise workflow that you can that allows for:
#
# 1. The code to scale if you added new sites or more time periods to the analysis.
# 2. Someone else to understand your workflow.
# 3. The LEAST and most efficient (i.e. runs fast, minimize repetition) amount of code that completes the task.
#
# ### HINTS
#
# * Remove values outside of the landsat valid range of values as specified in the metadata, as needed.
# * Keep any output files SEPARATE FROM input files. Outputs should be created in an outputs directory that is created in the code (if needed) and/or tested for.
# * Use the functions that we demonstrated during class to make your workflow more efficient.
# * BONUS - if you  chose - you can export your data as a csv file. You will get bonus points for doing this.
#
#
# ## Assignment Requirements
#
# Your submission to the GitHub repository should include:
# * This Jupyter Notebook file (.ipynb) with:
#     * The code to create a plot of mean NDVI across a year for  2 NEON Field Sites:
#         * NDVI on the x axis and formatted dates on the y for both NEON sites on one figure/axis object
#     * The **data should be cleaned to remove the influence of clouds**. See the [earthdatascience website for an example of what your plot might look like with and without removal of clouds](https://www.earthdatascience.org/courses/earth-analytics-python/create-efficient-data-workflows/).
# * BONUS: Create one output `.csv` file that has 3 columns - NDVI, Date and Site Name - with values for SJER and HARV.
#
# Your notebook should:
# * Have *at least* 2 well documented and well named functions with docstrings.
# * Include a Markdown cell at the top of the notebook that outlines the overall workflow using pseudocode (i.e. plain language, not code)
# * Include additional Markdown cells throughout the notebook to describe:
#     * the data that you used - and where it is from
#     * how data are being processing
#     * how the code is optimized to run fast and be more concise

# # Replace this cell with your pseudocode  for this workflow
#
# If you happen to be a diagram person a diagram is ok too
#
#

# **Unable to edit/delete cell above.**
# # Pseudocode of Workflow
# ###### Import packages
# * Import packages that are needed in PEP 8 order
# * Only includeds packages that are used
# * Gets the ndvi-automation data using earthpy
# * Changes the working directory to earth-analytics/data
#
# ###### Functions
# * Created 4 functions, each defined in functions
#     * open_clean_bands
#     * calc_mndvi
#     * temp_list
#     * df_create
#
# ###### Task 1
# * Use os to set the parent path of the data
# * Use glob to get a list of all sites
# * Use os to get the name of the desired site
# * Open the clipping shapefile
# * Open the crop boundary using geopandas for the desired site
# * Create a sorted list of all tif bands
# * Select the specific landsat scene desired
# * Open the bands at the scene
# * Open and clip the cloud mask layer using rasterio and clip
# * Loop through bands for desired scene using open bands function
# * Store bands in a list and calculate mean ndvi from mndvi function
# * Use temp_list function to create list of desired attributes
# * Create pandas df of landsat scene from list
#
# ###### Task 2
# * In this case we desire all scenes from all sites defined in the parent path
# * Initial path steps are completed in prior task.
# * Loop through all sites in the sites list (2 sites). For each loop:
#     * Get site name
#     * Open shapefile of site for clipping using os and gpd
#     * Get subdirectories of the site and create path to all scenes
#     * Loop through all scenes at each site. For each loop:
#         * Create sorted list of all bands in scene
#         * Get the cloud mask layer for scene and clip with rasterio
#         * Loop through the two bands per scene. For each loop:
#             * Open and clean the two bands using open_clean_bands
#             * Append opened bands to outer list: all_bands
#         * Calculate mean ndvi of single scene using calc_mndvi
#         * Create list of site name, scene date, scene mean ndvi
#         * Append this list to outermost list: all_list
# * Use all_list, a list of lists containing site name, date and mean ndvi, to create a pandas dataframe
# * Set dataframe date column to datetime format and set date to index
# * Return completed ndvi_df dataframe.
#
# ###### Task 3
# * Create a new df without nan values
# * Create plot
# * Group df by site name and loop through:
#     * plot x-axis as date
#     * plot y-axis as mean ndvi
#     * set label as site name
# * Set axis labels and x-axis boundaries
# * Set x-axis format to month and create plot grid
#
# ###### Bonus Task
# * Define desired directory location for .csv file
# * Check if location exists
#     * If it exists, continue to end of cell
#     * If doesn't exist, make the directory location and continue to end of cell
# * use .to_csv to create csv file at desired directory location
#

# In[1]:

# Autograding imports - do not modify this cell
import matplotcheck.autograde as ag
import matplotcheck.notebook as nb
import matplotcheck.timeseries as ts
from datetime import datetime

# In[2]:

# Import needed packages in PEP 8 order
# and no unused imports listed (10 points total)

# YOUR CODE HERE
# Import necessary packages
import os
from glob import glob

import matplotlib.pyplot as plt
import numpy as np
import geopandas as gpd
import rioxarray as rxr
import xarray as xr
import earthpy as et
import pandas as pd

import matplotlib.dates as mdates
from matplotlib.dates import DateFormatter

# Get data and set working directory
data = et.data.get_data('ndvi-automation')
os.chdir(os.path.join(et.io.HOME,
                      "earth-analytics",
                      "data"))

# In[3]:

# DO NOT MODIFY THIS CELL
# Tests that the working directory is set to earth-analytics/data

path = os.path.normpath(os.getcwd())
student_wd_parts = path.split(os.sep)

if student_wd_parts[-2:] == ['earth-analytics', 'data']:
    print("\u2705 Great - it looks like your working directory is set correctly to ~/earth-analytics/data")
else:
    print("\u274C Oops, the autograder will not run unless your working directory is set to earth-analytics/data")

# # Figure 1: Plot 1 - Mean NDVI For Each Site Across the Year (50 points)
#
# Create a plot of the mean normalized difference vegetation index (NDVI) for the two different sites in the United States across the year:
#
# * NDVI on the x axis and formatted dates on the y for both NEON sites on one figure/axis object.
# * Each site should be identified with a different color in the plot and legend.
# * The final plot **data should be cleaned to remove the influence of clouds**.
# * Be sure to include appropriate title and axes labels.
#
# Add additional cells as needed for processing data (e.g. defining functions, etc), but be sure to:
# * follow the instructions in the code cells that have been provided to ensure that you are able to use the sanity check tests that are provided.
# * include only the plot code in the cell identified for the final plot code below

# ## Task 1:
#
# In the cell below, create a single dataframe containing MEAN NDVI, the site name,
# and the date of the data for the HARV site
# scene `HARV/landsat-crop/LC080130302017031701T1-SC20181023151837`.  The column names for the  final
# DataFrame should be`mean_ndvi`, and `site`, and the data should be **indexed on the date**.
#
# Use the functions that we reviewed in class (or create your own versions of them) to implement your code
#
# ### In the Cell below Place  All Functions Needed to Run this Notebook (20 points)

# In[4]:

### DO NOT REMOVE THIS LINE OR EDIT / MOVE THIS CELL ###
start_time = datetime.now()

# # Functions to process and optimize code
# * These four functions are created to reduce copypasta code throughout the notebook
# * The use of functions also allows for easier modification of code and flow paths
# * Open_clean_bands function utilizes 'from_disk' to slice where it's only grabbing the data that is needed, saving time and memory resources

# In[5]:

# In this cell place all of the functions needed to run your notebook
# You will be graded here on function application, docstrings, efficiency so ensure
# All functions are placed here!

# YOUR CODE HERE
def open_clean_bands(band_path,
                     crop_bound,
                     valid_range=None,
                     a_mask=None,
                     vals=None):
    """Open and mask a single landsat band using a pixel_qa layer.

    Parameters
    -----------
    band_path : string
        A path to the array to be opened
    crop_bound : geopandas GeoDataFrame
        A geopandas dataframe to be used to crop the raster data using
            rasterio mask().
    valid_range : tuple (optional)
        A tuple of min and max range of values for the data. Default = None
    a_mask : xarray DataArray
        An xarray DataArray with values that have not yet been set to 1
    vals : list
        A list of values needed to create the cloud mask

    Returns
    -----------
    arr : xarray DataArray
        An xarray DataArray with values that should be masked set to 1 for
            True (Boolean)
    """
    # Open and crop band
    band_crop = rxr.open_rasterio(band_path, masked=True).rio.clip(crop_bound.geometry,
                                                                   from_disk=True).squeeze()
    # Mask band
    # Only run this step if a valid range tuple is provided
    if valid_range:
        mask = ((band_crop < valid_range[0]) | (band_crop > valid_range[1]))
        band_crop = band_crop.where(~xr.where(mask, True, False))

    if len(a_mask.shape) == 3 & a_mask.shape[0] == 1:
        a_mask = a_mask.squeeze()

    band_crop = band_crop.where(~a_mask.isin(vals))

    return band_crop

def calc_mndvi(band_list):
    """Calculate mean ndvi from a list of two bands from a single scene.

    Parameters
    -----------
    band_list : list
        A list of two opened bands of xarray DataArray.

    Returns
    -----------
    mndvi : xarray DataArray
        An xarray DataArray of mean normalized difference vegetation index for
            a single scene.
    """
    # Calculate mean ndvi
    # I expect to see RuntimeWarnings when calculated on nan values
    mndvi = np.nanmean((band_list[1]-band_list[0])/(band_list[1]+band_list[0]))

    return mndvi

def temp_list(site_name, adir, ndvi):
    """Take the site name, scene date, and scene ndvi, and create a list.

    Parameters
    -----------
    site_name : string
        String of the site's name.
    adir : string
        String of path to scene file.
    ndvi : ndvi of scene

    Returns
    -----------
    temp_list : list
        A list of the site name, scene date, and scene ndvi.
    """

    # Append site name, site date, mean ndvi to temp_list
    temp_list = []
    temp_list.append(site_name)
    temp_list.append(adir.split(os.sep)[4][10:18])
    temp_list.append(ndvi)

    return temp_list

def df_create(all_list):
    """Take a list of lists containing site name, site date, and mean ndvi and
            create pandas dataframe indexed at date with renamed columns.

    Parameters
    -----------
    all_list : list
        List of all lists containing site name, site date, and mean ndvi.

    Return
    -----------
    ndvi_df : pandas dataframe
        Pandas dataframe indexed at date with renamed columns.
    """
    # Create pandas dataframe from list
    ndvi_df = pd.DataFrame(all_list,
                           columns=["site", "date", "mean_ndvi"])
    ndvi_df["date"] = pd.to_datetime(ndvi_df["date"])
    ndvi_df = ndvi_df.reset_index().set_index(["date"])

    return ndvi_df

# # The Data Used
# * The same data used in task 1, task 2, and the figure
# * The data is imported using et.get_data(ndvi_automation)
# * This data is Landsat data from two sites:
#     * San Joaquin Experimental Range / SJER
#     * Harvard Forest / HARV
# * SJER is a terrestrial NEON field site located approximately 40 km (25 mi.) north of Fresno, CA.
# * HARV is a terrestrial NEON field site located approximately 65 miles west of Boston, Massachusetts in the county of Worcester.

# In[6]:

# Create dataframe of mean NDVI in this cell using the functions created above
# Important: to use the ungraded tests below as a sanity check,
# name your columns: mean_ndvi and site
# Call the dataframe at the end of the cell so the tests run on it!
# Be sure that the date column is an index of type date
# HINT: the time series lessons may help you remember how to do this!

# YOUR CODE HERE
# Get a list of each directory
path = os.path.join("ndvi-automation", "sites")

# Get a list of both site directories
sites = glob(path + "/*/")

# Get the site name
site_name = os.path.basename(os.path.normpath(sites[0]))

# Open up the shapefile for clipping your landsat data to the study area
vector_dir = os.path.join(sites[0],
                          "vector")

# Open crop boundary
site_boundary_path = os.path.join(vector_dir,  site_name + "-crop.shp")
crop_bound = gpd.read_file(site_boundary_path)

landsat_dir = os.path.join(sites[0],
                           "landsat-crop")

# Path to all desired .tif files
landsat_dirs = sorted(glob(os.path.join(landsat_dir, "LC08*")))

# Select just a single directory and grab bands 4-5 from the directory
adir = landsat_dirs[4]

# Open bands
band_paths = sorted(glob(os.path.join(adir, "*band*[4-5].tif")))

# Open and clip the cloud mask layer
# Cloud no data vals for Landsat 8 -
vals = [328, 392, 840, 904, 1350, 352, 368, 416,
        432, 480, 864, 880, 928, 944, 992, 480, 992]

# Get cloud mask layer path
qa_r = glob(os.path.join(adir, "*qa*"))

# Clip the cloud mask layer
cl_mask = rxr.open_rasterio(qa_r[0], masked=True).rio.clip(
    crop_bound.geometry, from_disk=True).squeeze()

# Open and clean bands
all_bands = []
for aband in band_paths:
    cleaned_band = open_clean_bands(band_path=aband,
                                    crop_bound=crop_bound,
                                    valid_range=(0, 10000),
                                    a_mask=cl_mask,
                                    vals=vals)
    all_bands.append(cleaned_band)

# Calculate mean ndvi
final_ndvi = calc_mndvi(all_bands)

# Append site name, site date, ndvi mean to dir_list
dir_list = temp_list(site_name=site_name, adir=adir, ndvi=final_ndvi)

# Append dir_list to all_list
all_list_scene = []
all_list_scene.append(dir_list)

# Create pandas dataframe from scene list
ndvi_df = df_create(all_list=all_list_scene)

ndvi_df

# In[7]:

# This cell  is testing your data output above

student_ndvi_ts_single_site = _

single_scene_points = 0

# Ensure the data is stored in a dataframe.
if isinstance(student_ndvi_ts_single_site, pd.DataFrame):
    print('\u2705 Your data is stored in a DataFrame!')
    single_scene_points += 1
else:
    print('\u274C It appears your data is not stored in a DataFrame. ',
          'To see what type of object your data is stored in, check its type with type(object)')

# Ensure that the date column is the index
if isinstance(student_ndvi_ts_single_site.index, pd.core.indexes.datetimes.DatetimeIndex):
    print('\u2705 You have the index set to the date column!')
    single_scene_points += 2
else:
    print('\u274C You do not have the index set to the date column.')

# Ensure that the date column is datetime
if isinstance(student_ndvi_ts_single_site.index[0], pd._libs.tslibs.timestamps.Timestamp):
    print('\u2705 The data in your date column is datetime!')
    single_scene_points += 2
else:
    print('\u274C The data in your date column is not datetime.')

# Ensure the site name is correct
if student_ndvi_ts_single_site.site.values[0] == 'HARV':
    print('\u2705 You have the correct site name!')
    single_scene_points += 5
else:
    print('\u274C You do not have the correct site name.')

if np.allclose(0.281131628228094, student_ndvi_ts_single_site.mean_ndvi.values[0]):
    print('\u2705 You have the correct mean NDVI value!')
    single_scene_points += 5
else:
    print('\u274C You do not have the correct mean ndvi value.')

print("\n \u27A1 You received {} out of 15 points for creating a dataframe.".format(
    single_scene_points))
single_scene_points

# ## Task 2:
#
# In the cell below, process all of the landsat scenes. Create a DataFrame that contains the following
# information for each scene
#
#
# |   | index  | site  | mean_ndvi  |
# |---|---|---|---|
# | Date  |   |   |   |
# |  2017-01-07  | 0  | SJER  | .4  |
#
# Be sure to call your dataframe at the end of the cell to ensure autograding works.
# HINT: FOR THIS STEP, leave any rows containing missing values (`NAN`).

# # Code Processing
# * The code below utilizes some objects from Task 1 to reduce processing
# * Nested for loops are utilized to reduce copypasta of code
# * Several lists are used to capture information from loops
# * Four functions are utilized throughout, helping make the code more concise and readable

# In[8]:

# Create dataframe of NDVI including the cleaning data to deal with clouds

# Important: to use the ungraded tests below as a sanity check,
# name your columns: mean_ndvi and site
# Don't forget to set date as the index and make the values of type datetime

# YOUR CODE HERE

all_list = []
for asite in sites:

    # Get the site name
    site_name = os.path.basename(os.path.normpath(asite))

    # Open site shapefile for clipping landsat data to study area
    vector_dir = os.path.join(asite, "vector")

    # Open site crop boundary
    site_boundary_path = os.path.join(vector_dir, site_name + "-crop.shp")
    crop_bound = gpd.read_file(site_boundary_path)

    # Get subdirectories of site
    site_dir = os.path.join(asite, "landsat-crop")

    # Path to all desired .tif files
    landsat_dirs = sorted(glob(os.path.join(site_dir, "*")))

    # Create df
    site_list = []
    for adir in landsat_dirs:

        # Open bands
        band_paths = sorted(glob(os.path.join(adir, "*band*[4,5].tif")))

        # Get cloud mask layer path
        qa_r = glob(os.path.join(adir, "*qa*"))

        # Clip the cloud mask layer
        cl_mask = rxr.open_rasterio(qa_r[0], masked=True).rio.clip(
            crop_bound.geometry, from_disk=True).squeeze()

        # Open and clean bands
        all_bands = []
        for aband in band_paths:
            cleaned_band = open_clean_bands(band_path=aband,
                                            crop_bound=crop_bound,
                                            valid_range=(0, 10000),
                                            a_mask=cl_mask,
                                            vals=vals)
            all_bands.append(cleaned_band)

        # Calculate mean ndvi
        final_ndvi = calc_mndvi(all_bands)

        # Append site name, site date, ndvi mean to dir_list
        site_list = temp_list(site_name=site_name, adir=adir, ndvi=final_ndvi)

        # Append site_list to all_list
        all_list.append(site_list)

# Create final df from returned list of lists with mean ndvi, site, and date
ndvi_df = pd.DataFrame(all_list,
                       columns=["site", "date", "mean_ndvi"])
ndvi_df["date"] = pd.to_datetime(ndvi_df["date"])
ndvi_df = ndvi_df.reset_index().set_index(["date"])

ndvi_df

# In[9]:

# Last sanity check before creating your plot (10 points)

# Ensure that you call your dataframe at the bottom of the cell above
# and that it has columns called: mean_ndvi and site

# Ensure the data is stored in a dataframe.
student_ndvi_df = _

df_points = 0

if isinstance(student_ndvi_df, pd.DataFrame):
    print('\u2705 Your data is stored in a DataFrame!')
    df_points += 2
else:
    print('\u274C It appears your data is not stored in a DataFrame. ',
          'To see what type of object your data is stored in, check its type with type(object)')

# Check that dataframe contains the appropriate number of NAN values
if student_ndvi_df.mean_ndvi.isna().sum() == 15:
    print('\u2705 Correct number of masked data values!')
    df_points += 2
else:
    print('\u274C The amount of null data in your dataframe is incorrect.')

# Ensure that the date column is the index
if isinstance(student_ndvi_df.index, pd.core.indexes.datetimes.DatetimeIndex):
    print('\u2705 You have the index set to the date column!')
    df_points += 3
else:
    print('\u274C You do not have the index set to the date column.')

# Ensure that the date column is datetime
if isinstance(student_ndvi_df.index[0], pd._libs.tslibs.timestamps.Timestamp):
    print('\u2705 The data in your date column is datetime!')
    df_points += 3
else:
    print('\u274C The data in your date column is not datetime.')

# Output for timer, # DO NOT MODIFY
end_time = datetime.now()
total_time = end_time - start_time
print(
    "Your total run time for processing the data was {0}.".format(total_time))

print("\n \u27A1 You received {} out of 10 points for creating a dataframe.".format(
    df_points))

df_points

# In[10]:

# Add only the plot code to this cell

# This is the final figure of mean NDVI
# for both sites across the year
# with data cleaned to deal with clouds

# YOUR CODE HERE
# Create new df with nan values removed for plotting
plot_df = ndvi_df[ndvi_df['mean_ndvi'].notna()]

date_form_month = DateFormatter("%b")

f, ax = plt.subplots(figsize=(10, 5))
for title, group in plot_df.reset_index().groupby('site'):
    group.groupby('site').plot(x='date',
                               y='mean_ndvi',
                               title="Mean Normalized Difference Vegetation Index (NDVI) \n Jan 2017 - Dec 2017 \n Landsat 8 with Clouds Removed",
                               label=title,
                               style='.-',
                               ax=ax,
                               alpha=.8)

ax.set(xlabel="Month", ylabel="Mean NDVI", xlim=['2017-01-01', '2017-12-31'])

ax.xaxis.set_major_formatter(date_form_month)
ax.xaxis.set_major_locator(mdates.MonthLocator(interval=1))

plt.grid(b=None, which='major', axis='both')

# plt.show()

### DO NOT REMOVE LINES BELOW ###
final_masked_solution = nb.convert_axes(plt, which_axes="current")

# In[11]:

# Ignore this cell for the autograding tests

# In[12]:

# Ignore this cell for the autograding tests

# # Question 1 (10 points)
#
# Imagine that you are planning NEON’s upcoming flight season to capture remote sensing data in these locations and want to ensure that you fly the area when the vegetation is the most green.
#
# When would you recommend the flights take place for each site?
#
# Answer the question in 2-3 sentences in the Markdown cell below.

# A higher NDVI value generally indicates more vegetation, and a lower value generally indicates less vegetation. In the NDVI plot, HARV has a higher mean NDVI, above 0.8, from mid May to late September. This correlates well with the warmer and rainy season of MA and would be the good period of time to fly. SJER has a higher NDVI, above 0.65, between mid February and April, staying above 0.6 through the end of April. This correlates well with the peak growing season of the area in early March, and would be a good period of time to fly.

# # Question 2 (10 points)
#
# How could you modify your workflow to look at vegetation changes over time in each site?
#
# Answer the question in 2-3 sentences in the Markdown cell below.

# The workflow here captures mean ndvi over the course of a single year. If data were provided for multiple years, this workflow would loop through the additional directory paths and capture the NDVI for all dates provided. The plot could then be adapted to show vegetation changes over time by plotting different line colors for each year, with constant symbols for each location.
#
# Another way to capture vegetation change over time would be to groupby mean NDVI for the full year, and plot Mean NDVI per scene by year, rather than by month.
#

# # Do not edit this cell! (10 points)
#
# The notebook includes:
# * additional Markdown cells throughout the notebook to describe:
#     * the data that you used - and where it is from
#     * how data are being processing
#     * how the code is optimized to run fast and be more concise

# # Do not edit this cell! (20 points)
#
# The notebook will also be checked for overall clean code requirements as specified at the **top** of this notebook. Some of these requirements include (review the top cells for more specifics):
#
# * Notebook begins at cell [1] and runs on any machine in its entirety.
# * PEP 8 format is applied throughout (including lengths of comment and code lines).
# * No additional code or imports in the notebook that is not needed for the workflow.
# * Notebook is fully reproducible. This means:
#    * reproducible paths using the os module.
#    * data downloaded using code in the notebook.
#    * all imports at top of notebook.

# ## BONUS - Export a  .CSV File to Share (10 points possible)
#
# This is optional - if you export a **.csv** file with the columns specified above: Site, Date and NDVI Value you can get an additional 10 points.
#
# * FULL CREDIT: File exists in csv format and contains the columns specified.
# We will check your github repo for this file!
#

# In[13]:

csv_dir = os.path.join(et.io.HOME, 'earth-analytics',
                       'spring', 'hw', 'ea-2021-04-ndvi-automation-brianpinke')

# Check path exists before trying to change directory, else make the directory
if os.path.exists(csv_dir):
    print("Directory exists! Creating CSV.")
else:
    print("Directory did not exist. Making data directory and creating CSV.")
    os.makedirs(csv_dir)

# Drop column of index to leave just three desired columns. send csv to path
ndvi_df.drop(columns='index').to_csv(os.path.join(csv_dir, 'ndvi_df.csv'))

# In[ ]: