Predictions_backup.py

import time
from collections import deque, namedtuple
from keras.models import Sequential 
from keras.layers import Dense, LSTM, Dropout
import numpy as np
#import PIL.Image
import tensorflow as tf
import utils

from pyvirtualdisplay import Display
from keras import Sequential
from keras.losses import MSE
from keras.optimizers import Adam

####################

import numpy as np
import pandas as pd
from pandas_datareader import data
import matplotlib.pyplot as plt
import datetime as dt
import os
import urllib.request, json
import tensorflow as keras  # Use the official 'tensorflow' package instead of 'keras'
from keras.models import Sequential 
from keras.layers import Dense, LSTM, Dropout
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split
from keras.activations import sigmoid
from keras.callbacks import ModelCheckpoint, EarlyStopping
from keras.models import load_model
from keras import layers
from keras.layers import LSTM, Dropout, Dense, Input
from keras.models import Model
import pandas_ta as ta
from sklearn.model_selection import TimeSeriesSplit
from sklearn.metrics import mean_squared_error
from math import sqrt
import pandas_market_calendars as mcal 
from sklearn.model_selection import KFold
from sklearn.preprocessing import StandardScaler
from keras.regularizers import l2, l1

#data_source = 'kaggle'  # alphavantage or kaggle
data_source = 'alphavantage'

if data_source == 'alphavantage':
    # ====================== Loading Data from Alpha Vantage ==================================

    api_key = '<RR02P5TH2UNCD10X>'

    # American Airlines stock market prices
    ticker = "F"

    # JSON file with all the stock market data for AAL from the last 20 years
    url_string = "https://www.alphavantage.co/query?function=TIME_SERIES_DAILY&symbol=%s&outputsize=full&apikey=%s"%(ticker,api_key)

    # Save data to this file
    file_to_save = 'stock_market_data-%s.csv'%ticker

    # If you haven't already saved data,
    # Go ahead and grab the data from the url
    # And store date, low, high, volume, close, open values to a Pandas DataFrame
    if not os.path.exists(file_to_save):
        with urllib.request.urlopen(url_string) as url:
            data = json.loads(url.read().decode())
            # extract stock market data
            data = data['Time Series (Daily)']
            df = pd.DataFrame(columns=['Date','Low','High','Close','Open','Volume'])
            for k,v in data.items():
                date = dt.datetime.strptime(k, '%Y-%m-%d')
                data_row = [date.date(),float(v['3. low']),float(v['2. high']),
                            float(v['4. close']),float(v['1. open']), float(v['5. volume'])]
                df.loc[-1,:] = data_row
                df.index = df.index + 1
        print('Data saved to : %s'%file_to_save)        
        df.to_csv(file_to_save)

    # If the data is already there, just load it from the CSV
    else:
        print('File already exists. Loading data from CSV')
        df = pd.read_csv(file_to_save)
        df = df.drop('Unnamed: 0', axis=1)

else:

    # ====================== Loading Data from Kaggle ==================================
    # You will be using HP's data. Feel free to experiment with other data.
    # But while doing so, be careful to have a large enough dataset and also pay attention to the data normalization
    df = pd.read_csv(os.path.join('Stocks','hpq.us.txt'),delimiter=',',usecols=['Date','Open','High','Low','Close'])
    print('Loaded data from the Kaggle repository')

#Data saved to : stock_market_data-AAL.csv


# Sort DataFrame by date
df = df.sort_values('Date')

#print(df)




df['Date'] = pd.to_datetime(df['Date'])  # Convert 'Date' column to datetime
df.set_index('Date', inplace=True)  # Set 'Date' column as index

# Calculate moving averages
df['MA20'] = df['Close'].rolling(window=20).mean()
df['MA50'] = df['Close'].rolling(window=50).mean()
df['MA200'] = df['Close'].rolling(window=200).mean()

# Calculate RSI and ATR
df['RSI'] = ta.rsi(df['Close'])
df['ATR'] = ta.atr(df['High'], df['Low'], df['Close'])

df.dropna(inplace=True)  # Drop rows with missing moving averages

# Calculate SMA (Simple Moving Average)
def calculate_sma(df, window):
    return df['Close'].rolling(window=window).mean()

# Calculate EMA (Exponential Moving Average)
def calculate_ema(df, window):
    return df['Close'].ewm(span=window, adjust=False).mean()

# Calculate MACD (Moving Average Convergence Divergence)
def calculate_macd(df, short_window, long_window, signal_window):
    short_ema = calculate_ema(df, short_window)
    long_ema = calculate_ema(df, long_window)
    macd = short_ema - long_ema
    signal_line = macd.ewm(span=signal_window, adjust=False).mean()
    return macd, signal_line

# Calculate Bollinger Bands
def calculate_bollinger_bands(df, window, num_std_dev):
    rolling_mean = df['Close'].rolling(window=window).mean()
    rolling_std = df['Close'].rolling(window=window).std()
    upper_band = rolling_mean + (rolling_std * num_std_dev)
    lower_band = rolling_mean - (rolling_std * num_std_dev)
    return upper_band, lower_band

df.dropna(inplace=True)  # Drop rows with missing moving averages

# Calculate and add SMA_20
df['SMA_20'] = calculate_sma(df, 20)

# Calculate and add EMA_20
df['EMA_20'] = calculate_ema(df, 20)

# Calculate MACD and Signal Line
macd, signal_line = calculate_macd(df, 12, 26, 9)
df['MACD'] = macd
df['Signal_Line'] = signal_line

# Calculate and add Bollinger Bands
upper_band, lower_band = calculate_bollinger_bands(df, 20, 2)
df['Bollinger_Upper'] = upper_band
df['Bollinger_Lower'] = lower_band

df.dropna(inplace=True)  # Drop rows with missing moving averages

# Define functions to identify candlestick patterns

def is_doji(row):
    return abs(row['Open'] - row['Close']) < 0.1 * (row['High'] - row['Low'])

def is_hammer(row):
    return (row['Close'] > row['Open']) and ((row['Close'] - row['Low']) / (0.001 + row['High'] - row['Low']) > 0.6)

def is_bullish_engulfing(df, index):
    if index < 1:
        return False
    yesterday = df.iloc[index - 1]
    today = df.iloc[index]
    return (yesterday['Open'] > yesterday['Close']) and (today['Open'] < today['Close']) and (today['Close'] > yesterday['Open']) and (today['Open'] < yesterday['Close'])

def is_bearish_engulfing(df, index):
    if index < 1:
        return False
    yesterday = df.iloc[index - 1]
    today = df.iloc[index]
    return (yesterday['Open'] < yesterday['Close']) and (today['Open'] > today['Close']) and (today['Close'] < yesterday['Open']) and (today['Open'] > yesterday['Close'])

def is_morning_star(df, index):
    if index < 2:
        return False
    yesterday = df.iloc[index - 2]
    today = df.iloc[index - 1]
    tomorrow = df.iloc[index]
    return (yesterday['Open'] > yesterday['Close']) and (today['Close'] > yesterday['Open']) and (tomorrow['Open'] < today['Close'])

def is_evening_star(df, index):
    if index < 2:
        return False
    yesterday = df.iloc[index - 2]
    today = df.iloc[index - 1]
    tomorrow = df.iloc[index]
    return (yesterday['Open'] < yesterday['Close']) and (today['Close'] < yesterday['Open']) and (tomorrow['Open'] > today['Close'])

def is_three_white_soldiers(df, index):
    if index < 2:
        return False
    prev_3_candles = df.iloc[index - 2 : index + 1]
    return all(prev_3_candles['Close'] > prev_3_candles['Open'])

def is_three_black_crows(df, index):
    if index < 2:
        return False
    prev_3_candles = df.iloc[index - 2 : index + 1]
    return all(prev_3_candles['Open'] > prev_3_candles['Close'])

def is_shooting_star(row):
    return (row['Open'] > row['Close']) and ((row['High'] - row['Close']) / (0.001 + row['High'] - row['Low']) > 0.6)

def is_inverted_hammer(row):
    return (row['Open'] < row['Close']) and ((row['Close'] - row['Low']) / (0.001 + row['High'] - row['Low']) > 0.6)

# Identify patterns in the data

df['Doji'] = df.apply(is_doji, axis=1)
df['Hammer'] = df.apply(is_hammer, axis=1)
df['Bullish Engulfing'] = [is_bullish_engulfing(df, i) for i in range(len(df))]
df['Bearish Engulfing'] = [is_bearish_engulfing(df, i) for i in range(len(df))]
df['Morning Star'] = [is_morning_star(df, i) for i in range(len(df))]
df['Evening Star'] = [is_evening_star(df, i) for i in range(len(df))]
df['Three White Soldiers'] = [is_three_white_soldiers(df, i) for i in range(len(df))]
df['Three Black Crows'] = [is_three_black_crows(df, i) for i in range(len(df))]
df['Shooting Star'] = df.apply(is_shooting_star, axis=1)
df['Inverted Hammer'] = df.apply(is_inverted_hammer, axis=1)

# Aroon   
from sklearn.preprocessing import LabelEncoder
import ta
def aroon(df, periods):
    aroon_up = []
    aroon_down = []
    for i in range(len(df)):
        if i < periods:
            aroon_up.append(np.nan)
            aroon_down.append(np.nan)
        else:
            highs = list(df['High'].iloc[i-periods:i])
            lows = list(df['Low'].iloc[i-periods:i])
            aroon_up.append(((highs.index(max(highs)) / periods) * 100))
            aroon_down.append(((lows.index(min(lows)) / periods) * 100))
    df['Aroon_up'] = aroon_up
    df['Aroon_down'] = aroon_down

    # Fill NaN values with the last observed non-null value
    df['Aroon_up'].fillna(method='ffill', inplace=True)
    df['Aroon_down'].fillna(method='ffill', inplace=True)

aroon(df, 25)

# Create 'Trend' column
df['Trend'] = 'Undefined'
df.loc[(df['Aroon_up'] < 50) & (df['Aroon_down'] < 50), 'Trend'] = 'Consolidation'

# Apply LabelEncoder
le = LabelEncoder()
df['Trend'] = le.fit_transform(df['Trend'])
    
    # Calculate ADX #### TBA

df.dropna(inplace=True)  # Drop rows with missing moving averages

##### Lag Features #####

# Define the window size
window_size_lag = 5

# Create rolling windows for SMA_20
df['SMA_20_Rolling_Mean'] = df['SMA_20'].rolling(window=window_size_lag).mean()

# Create rolling windows for EMA_20
df['EMA_20_Rolling_Mean'] = df['EMA_20'].rolling(window=window_size_lag).mean()

# Create rolling windows for MACD
df['MACD_Rolling_Mean'] = df['MACD'].rolling(window=window_size_lag).mean()

# Create rolling windows for Signal Line
df['Signal_Line_Rolling_Mean'] = df['Signal_Line'].rolling(window=window_size_lag).mean()

# Create rolling windows for Bollinger Bands
df['Bollinger_Upper_Rolling_Mean'] = df['Bollinger_Upper'].rolling(window=window_size_lag).mean()
df['Bollinger_Lower_Rolling_Mean'] = df['Bollinger_Lower'].rolling(window=window_size_lag).mean()

# Create rolling windows for the candlestick patterns
df['Doji_Rolling_Mean'] = df['Doji'].rolling(window=window_size_lag).mean()
df['Hammer_Rolling_Mean'] = df['Hammer'].rolling(window=window_size_lag).mean()
df['Bullish Engulfing_Rolling_Mean'] = df['Bullish Engulfing'].rolling(window=window_size_lag).mean()
df['Bearish Engulfing_Rolling_Mean'] = df['Bearish Engulfing'].rolling(window=window_size_lag).mean()
df['Morning Star_Rolling_Mean'] = df['Morning Star'].rolling(window=window_size_lag).mean()
df['Evening Star_Rolling_Mean'] = df['Evening Star'].rolling(window=window_size_lag).mean()
df['Three White Soldiers_Rolling_Mean'] = df['Three White Soldiers'].rolling(window=window_size_lag).mean()
df['Three Black Crows_Rolling_Mean'] = df['Three Black Crows'].rolling(window=window_size_lag).mean()
df['Shooting Star_Rolling_Mean'] = df['Shooting Star'].rolling(window=window_size_lag).mean()
df['Inverted Hammer_Rolling_Mean'] = df['Inverted Hammer'].rolling(window=window_size_lag).mean()

df.replace(np.nan, 0, inplace=True)

# Merge Data

# Load the merged data
merged_data = pd.read_csv('merged_data.csv', index_col=0, parse_dates=True)

# Fill missing values
merged_data.fillna(method='ffill', inplace=True)

# Fill missing values
merged_data.fillna(method='bfill', inplace=True)

# Merge df with merged_data
df = pd.merge(df, merged_data, left_index=True, right_index=True, how='inner')

# Print the merged DataFrame
print(df)

#columns_to_drop = [ "Low", "High", "Open"]

#df = df.drop(columns=columns_to_drop)

P = 20

df_new = df.tail(P)

#df = df.drop(df.index[-P:]) # put a # to do actual predictions

window_size = P

print(df)

############## FEATURE ENGINEERING ################

from sklearn.model_selection import TimeSeriesSplit
from keras.callbacks import ModelCheckpoint
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score

############## FEATURE ENGINEERING ################

from sklearn.model_selection import TimeSeriesSplit

# Select features and target
features = df.copy()
target = 'Close' 
num_feats = len(features.columns)

# Scale the features
scaler = MinMaxScaler()
df_scaled = pd.DataFrame(scaler.fit_transform(df), columns=df.columns)

print(len(features))

# Prepare the sequences for LSTM (we're using a window size of 30 days)

X = []
y = []

for i in range(window_size, len(df_scaled)):
    X.append(df_scaled.iloc[i-window_size:i, :].values)
    y.append(df_scaled.iloc[i, df_scaled.columns.get_loc(target)])

# Split the data into training, validation and testing sets
train_size = int(len(X) * 0.7)
val_size = int(len(X) * 0.2)
X_train, X_val, X_test = X[:train_size], X[train_size:train_size+val_size], X[train_size+val_size:]
y_train, y_val, y_test = y[:train_size], y[train_size:train_size+val_size], y[train_size+val_size:]

# Convert lists to numpy arrays
X_train = np.asarray(X_train)
y_train = np.asarray(y_train)
X_val = np.asarray(X_val)
y_val = np.asarray(y_val)
X_test = np.asarray(X_test)
y_test = np.asarray(y_test)


#df_scaled = df_scaled.drop('Unnamed: 0', axis=1)
#df = df.drop('Unnamed: 0', axis=1)
print(df_scaled)

########## Hyperp Tuning ############# TBA

import seaborn as sns
import matplotlib.pyplot as plt

#plt.figure(figsize=(14, 10))
#sns.heatmap(df.corr(), annot=True)
#plt.show()
########### Training The Model #######################

# Define the number of folds
num_folds = 7

# Define the TimeSeriesSplit object
tscv = TimeSeriesSplit(n_splits=num_folds)

# Define the EarlyStopping callback
early_stopping = EarlyStopping(monitor='val_loss', patience=2, restore_best_weights=False)

# Store the performance for each fold
loss_per_fold = []

# K-fold cross validation
fold_no = 1
for train, validation in tscv.split(X_train, y_train):
    # Define the model architecture
    model = Sequential([
    layers.LSTM(40, return_sequences= False, input_shape=(window_size, num_feats)), # Set to False for low volatity stocks
    layers.Dropout(0.2),
    layers.Dense(20, activation='tanh'),  # Added a Dense layer before the output: softmax, sigmoid, tanh, relu, selu, softsign
    layers.Dense(1)  # Output Close Prices
])


    # Compile the model
    model.compile(optimizer='adam', loss='mean_squared_error')

    # Fit the model
    history = model.fit(X_train[train], y_train[train], epochs=100, batch_size=32, validation_data=(X_train[validation], y_train[validation]), callbacks=[early_stopping])
# Evaluate the model
    scores = model.evaluate(X_train[validation], y_train[validation], verbose=0)
    print(f'Score for fold {fold_no}: {model.metrics_names[0]} of {scores}')
    loss_per_fold.append(scores)

    # Increase fold number
    fold_no = fold_no + 1

# Provide average scores
print('Average scores for all folds:')
#print(f'> Accuracy: {np.mean(acc_per_fold)} (+- {np.std(acc_per_fold)})')
print(f'> Loss: {np.mean(loss_per_fold)}')

# Plot the loss
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('Model loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Train', 'Validation'], loc='upper right')
plt.show(block=False)
plt.close()
#plt.show 
####### Accuracy #######

# Get the last 300 days of data
y_test_last_300_days = y_test[-1000:] # change to see how it would do for x amount of days divided by P
X_test_last_300_days = X_test[-1000:]

# Initialize counters
correct_predictions = 0
total_predictions = 0

# Walk-forward validation
for i in range(0, len(y_test_last_300_days) - window_size + 1, window_size):
 # Create a rolling window of size window_size
 X_test_window = X_test_last_300_days[i:i+window_size]
 y_test_window = y_test_last_300_days[i:i+window_size]
 
 # Only use the closing prices in the window
 #X_test_window = X_test_window[:, 2]

 # Predict the values
 predictions = model.predict(X_test_window)
# Increment the total_predictions counter
 total_predictions += 1
 # Create a temporary array filled with zeros
 temp_array = np.zeros((predictions.shape[0], num_feats))

 # Replace the third column of temp_array with your predictions
 temp_array[:, 2] = predictions.ravel()

 # Now you can use inverse_transform on temp_array
 predicted_prices = scaler.inverse_transform(temp_array)

 # The inverse transformed predictions are in the third column of predicted_prices
 predicted_prices = predicted_prices[:, 2]

 # Create a temporary array filled with zeros
 temp_array = np.zeros((y_test_window.shape[0], num_feats))

 # Replace the third column of temp_array with your y_test data
 temp_array[:, 2] = y_test_window.ravel()

 # Now you can use inverse_transform on temp_array
 y_test_prices = scaler.inverse_transform(temp_array)

 # The inverse transformed y_test data are in the third column of y_test_prices
 y_test_prices = y_test_prices[:, 2]
 
# Check if the price at the end of each window is greater than the price at the start of the same window for both the predicted and actual prices
 if ((predicted_prices[0] >= predicted_prices[-1]) and (y_test_prices[0] >= y_test_prices[-1])) or \
  ((predicted_prices[0] <= predicted_prices[-1]) and (y_test_prices[0] <= y_test_prices[-1])):
  correct_predictions += 1
  

# Calculate the percentage of correct predictions
percentage_correct = (correct_predictions / total_predictions) * 100
print(f'Fraction of correct predictions: {correct_predictions}/{total_predictions}')
print(f'Percentage of correct predictions: {percentage_correct}%')

###Metric Scors###
from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score

# Predict on the validation set
y_val_pred = model.predict(X_val)

# Calculate metrics
mse = mean_squared_error(y_val, y_val_pred)
rmse = np.sqrt(mse)
mae = mean_absolute_error(y_val, y_val_pred)
r2 = r2_score(y_val, y_val_pred)

# Print metrics
print('MSE:', mse)
print('RMSE:', rmse)
print('MAE:', mae)
print('R2:', r2)

####Test Results####
#Part 1
# Predict the values
predictions = model.predict(X_test)

# Create a temporary array filled with zeros
temp_array = np.zeros((predictions.shape[0], num_feats))

# Replace the first column of temp_array with your predictions
temp_array[:, 2] = predictions.ravel()

# Now you can use inverse_transform on temp_array
predicted_prices = scaler.inverse_transform(temp_array)

# The inverse transformed predictions are in the first column of predicted_prices
predicted_prices = predicted_prices[:, 2]

# Create a temporary array filled with zeros
temp_array = np.zeros((y_test.shape[0], num_feats))

# Replace the first column of temp_array with your y_test data
temp_array[:, 2] = y_test.ravel()

# Now you can use inverse_transform on temp_array
y_test_prices = scaler.inverse_transform(temp_array)

# The inverse transformed y_test data are in the first column of y_test_prices
y_test_prices = y_test_prices[:, 2]

# Plot the actual test data values
plt.plot(y_test_prices, label='Actual')

# Plot the predicted values
plt.plot(predicted_prices, label='Predicted')

# Set the labels and title
plt.xlabel('Time')
plt.ylabel('Stock Price')
plt.title('Actual vs Predicted Stock Prices')

# Show the legend
plt.legend()

# Show the plot
plt.show(block=False)
plt.close()
#plt.show 
####################################################
future_days = P  # Number of future days to predict

###### Predictions ######

last_window = X_test[-1]

future_predictions = []

col_names = df.columns.tolist()
target_index = col_names.index('Close')

for i in range(future_days):
    
    next_day_prediction = model.predict(np.array([last_window]))[0][0]
    
    future_predictions.append(next_day_prediction)
    
    last_window = np.roll(last_window, shift=-1, axis=0)
    last_window[-1][target_index] = next_day_prediction

temp_array = np.zeros((future_days, num_feats))
temp_array[:, 2] = future_predictions

predicted_prices = scaler.inverse_transform(temp_array)

last_date_actual = df.index.max()
future_dates = pd.date_range(start=last_date_actual + pd.DateOffset(days=1), periods=future_days, freq='B')

predicted_df = pd.DataFrame({'Date': future_dates, 'Predicted_Close': predicted_prices[:, 2]})

print(predicted_df)

plt.figure(figsize=(12, 6))
plt.plot(df_new.index, df_new['Close'], label='Actual Close Price', color='red')
plt.plot(predicted_df['Date'], predicted_df['Predicted_Close'], label='Predicted Close Price', color='blue')
plt.xlabel('Date')
plt.ylabel('Close Price')
plt.title('Predicted Close Price for the Next Business Days ' + ticker)
plt.grid(True)
plt.legend()
plt.show()
#Saving the plot as an image
#plt.savefig('line plot.jpg', bbox_inches='tight', dpi=150)