# Get a list of all year Id's in the year dropdown: 
# find the container where the year dropdown data is stored: 
container = driver.find_element(By.ID, 'dnn_ctr453_TTPSCrimeMonthTotals_ASPxRoundPanel1_ASPxComboBox3_DDD_L_LBT')

# get all the td elements in the container: 
td_elements = container.find_elements(By.TAG_NAME, 'td')

# extract the id's from the td tag:
year_ids = [td.get_attribute('id') for td in td_elements if td.get_attribute('id')]

# print all the id's collected: 
for year_id in year_ids: 
    print(year_id)
        

# create an empty list to store the data in: 
table_info = []


# loop through each year in the dropdown year, generate the report then web scrap the table info.
# store the table info in the empty list created above

for report_year in year_ids: 
    
    # find the year dropdown: 
    year_dropdown = driver.find_element(By.ID, 'dnn_ctr453_TTPSCrimeMonthTotals_ASPxRoundPanel1_ASPxComboBox3_I')
    # select dropdown: 
    ActionChains(driver).move_to_element(year_dropdown).click(year_dropdown).perform()
    # Wait for the page to load
    time.sleep(2)
    
    # Get the year in the dropdown: 
    year_select = driver.find_element(By.ID, report_year)
    # Extract the year text: 
    year = year_select.text
    #select the year: 
    ActionChains(driver).move_to_element(year_select).click(year_select).perform()
    # Wait for the page to load
    time.sleep(2)
    
    # find the generate report button: 
    generate = driver.find_element(By.ID, 'dnn_ctr453_TTPSCrimeMonthTotals_ASPxRoundPanel1_ASPxButton1_CD')
    # select generate report: 
    ActionChains(driver).move_to_element(generate).click(generate).perform()
    time.sleep(3)
    
    
    
    
    # Here we are swtitching to web scraping with Beautiful Soup: 
    
    
    
    # get the page source: 
    page_source = driver.page_source
    # parse with bs4
    soup = BeautifulSoup(page_source, 'html.parser')
    
    # get the table location: 
    page_table = soup.find('table', id='dnn_ctr453_TTPSCrimeMonthTotals_ASPxPageControl1_ASPxGridView1_DXMainTable')
    # there is no tagged table headers so we will skip this step and create our own headers later.
    
    # get the table rows: 
    rows = page_table.find_all('tr')#[3:-1]  Here you can skip rows if needed, but we will keep all rows and clean later in the data cleaning section.
    
    # loop through all the rows and extract the info then store them in the empty list we created
    for row in rows: 
        cells = row.find_all('td')
        #strip the text in each cell of any whitespace.
        cell_data = [cell.text.strip() for cell in cells]
        #add the year in which this info was taken
        cell_data.append(year)
        # Add the data to the empty list: 
        table_info.append(cell_data)
    
    
    # print the year when the data is finished extracting and stored: 
    print(f'All data in {year}, has been extracted and stored in the list: table_info')
            
        

# transfer the collected table info to a dataframe: 
crime_df = pd.DataFrame(table_info)

crime_df.info()
        

crime_df.head()
        

crime_df.tail()
        

# Having the year and the Month in separate columns and in differemt formats isnt helpful.
        
# create a mapping for months: 
month_map = {
    'Jan': '01', 'Feb': '02', 'Mar': '03', 'Apr': '04',
    'May': '05', 'Jun': '06', 'Jul': '07', 'Aug': '08',
    'Sep': '09', 'Oct': '10', 'Nov': '11', 'Dec': '12'
}

# Add month numbers to df    
crime_df['Month_Num'] = crime_df['Month'].map(month_map)  
# combine month number and year to df as a pandas datetime object.
crime_df['date'] = pd.to_datetime(crime_df['Month_Num'].astype(str) + crime_df['Year'])
#format the month_year column
crime_df['Month_Year'] = crime_df['date'].dt.strftime('%m-%Y')

# Create a copy of the df for the Prophet Forecasting (we will use 2 type of models)
crime_df_fb = crime_df.copy()


#apply the month_year as the index 
crime_df.index = crime_df['Month_Year']

# drop all other columns except totals: 
crime_df = crime_df[[ 'Total']]

# change the index to a period type with a monthly frequency: 
crime_df.index = pd.PeriodIndex(crime_df.index, freq='M')

# change the total to int type.
crime_df['Total'] = crime_df['Total'].astype(int)

crime_df.head()
        

# plot the trained data against the test data:
plot_series(crime_df_cleaned, crime_pred)
        

#get the Mean percentage error: 
mape = mean_absolute_percentage_error(crime_df_cleaned_test, crime_pred)

print(f'Mean Absolute Percentage Error: {round(mape*100, 2)}%')
        

# Forecast into the future: 
forecaster = ThetaForecaster(sp=12)
fh = ForecastingHorizon([i for i in range(1,12)])

# Train the model on all of the data
forecaster.fit(crime_df_cleaned)

# Predict the future data
crime_pred = forecaster.predict(fh)

# plot the trained data against the test data:
plot_series(crime_df_cleaned, crime_pred)
        

# Remove unwanted columns: 
crime_df_fb =  crime_df_fb[['Month_Year', 'Total']]

# change Month_Year column to datetime
crime_df_fb['Month_Year'] = pd.to_datetime(crime_df_fb['Month_Year'])
# change Total column to integer
crime_df_fb['Total'] = crime_df_fb['Total'].astype(int)

# Rename columns: 
crime_df_fb.rename(columns= {'Month_Year' : 'ds', 'Total': 'y'}, inplace=True)

crime_df_fb['ds'] = crime_df_fb['ds'] + pd.offsets.MonthEnd(0)
crime_df_fb.head()
        

lockdowns = pd.DataFrame([
    {'holiday': 'lockdown_1', 'ds': '2020-03-31', 'lower_window': 0, 'ds_upper': '2021-09-30'},

])
for t_col in ['ds', 'ds_upper']:
    lockdowns[t_col] = pd.to_datetime(lockdowns[t_col])
lockdowns['upper_window'] = (lockdowns['ds_upper'] - lockdowns['ds']).dt.days
lockdowns
        

train_df.reset_index(drop=True, inplace=True)
train_df
        

crime_df_fb
        

# Post Covid test
post_covid = crime_df_fb[crime_df_fb['ds'] >= '2021-10-01']
post_covid.reset_index(inplace=True, drop = True)

post_covid = post_covid.head(33)
post_covid.tail()
        

# set the model seasonality
m = Prophet( holidays=lockdowns, seasonality_mode='multiplicative', changepoint_prior_scale=0.05, changepoint_range=0.95)
#m2 = Prophet(seasonality_mode='multiplicative')

# train the model on the data
m.fit(train_df)
#m2.fit(post_covid)

# set the predicted dates amount
future = m.make_future_dataframe( periods=12, freq='M')
#future2 = m2.make_future_dataframe( periods=12, freq='M')

future.tail()
        

# predict the next 12 months: 
        
forecast = m.predict(future)
#forecast2 = m2.predict(future2)

forecast[['ds', 'yhat', 'yhat_lower', 'yhat_upper']].tail()
        

# add residuals columns
forecast['residuals'] = crime_df_fb['y'] - forecast['yhat']
#forecast2['residuals'] = post_covid['y'] - forecast2['yhat']

# calculate mape
forecast['absolute_percentage_error'] = abs(forecast['residuals'] / crime_df_fb['y']) * 100
#forecast2['absolute_percentage_error'] = abs(forecast2['residuals'] / post_covid['y']) * 100

# calcualte mape
mape = forecast['absolute_percentage_error'].mean()
#mape2 = forecast2['absolute_percentage_error'].mean()

round(mape,2)
        

# plot forecast
        
fig1 = m.plot(forecast)
        

# plot components
        
fig1 = m.plot_components(forecast)
        

# Python
        
from prophet.plot import add_changepoints_to_plot
        
fig = m.plot(forecast)
        
a = add_changepoints_to_plot(fig.gca(), m, forecast)
        

# Interactive Plot
        
HTML(plot_plotly(m, forecast).to_html())
        

print(f'Mean Absolute Percentage Error: {round(mape, 2)}%')