Stock_Predictor_Winter_2024_Predictor.py

import time
from collections import deque, namedtuple
from keras.models import Sequential 
from keras.layers import Dense, LSTM, Dropout
import numpy as np
#import PIL.Image
import tensorflow as tf
import utils

from pyvirtualdisplay import Display
from keras import Sequential
from keras.losses import MSE
from keras.optimizers import Adam

####################

import numpy as np
import pandas as pd
from pandas_datareader import data
import matplotlib.pyplot as plt
import datetime as dt
import os
import urllib.request, json
import tensorflow as keras  # Use the official 'tensorflow' package instead of 'keras'
from keras.models import Sequential 
from keras.layers import Dense, LSTM, Dropout
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split
from keras.activations import sigmoid
from keras.callbacks import ModelCheckpoint, EarlyStopping
from keras.models import load_model
from keras import layers
from keras.layers import LSTM, Dropout, Dense, Input
from keras.models import Model
import pandas_ta as ta
from sklearn.model_selection import TimeSeriesSplit
from sklearn.metrics import mean_squared_error
from math import sqrt
import pandas_market_calendars as mcal 
from sklearn.model_selection import KFold
from sklearn.preprocessing import StandardScaler
from keras.regularizers import l2, l1

#data_source = 'kaggle'  # alphavantage or kaggle
data_source = 'alphavantage'

if data_source == 'alphavantage':
    # ====================== Loading Data from Alpha Vantage ==================================

    api_key = '<RR02P5TH2UNCD10X>'

    # American Airlines stock market prices
    ticker = "AAPL"

    # JSON file with all the stock market data for AAL from the last 20 years
    url_string = "https://www.alphavantage.co/query?function=TIME_SERIES_DAILY&symbol=%s&outputsize=full&apikey=%s"%(ticker,api_key)

    # Save data to this file
    file_to_save = 'stock_market_data-%s.csv'%ticker

    # If you haven't already saved data,
    # Go ahead and grab the data from the url
    # And store date, low, high, volume, close, open values to a Pandas DataFrame
    if not os.path.exists(file_to_save):
        with urllib.request.urlopen(url_string) as url:
            data = json.loads(url.read().decode())
            # extract stock market data
            data = data['Time Series (Daily)']
            df = pd.DataFrame(columns=['Date','Low','High','Close','Open','Volume'])
            for k,v in data.items():
                date = dt.datetime.strptime(k, '%Y-%m-%d')
                data_row = [date.date(),float(v['3. low']),float(v['2. high']),
                            float(v['4. close']),float(v['1. open']), float(v['5. volume'])]
                df.loc[-1,:] = data_row
                df.index = df.index + 1
        print('Data saved to : %s'%file_to_save)        
        df.to_csv(file_to_save)

    # If the data is already there, just load it from the CSV
    else:
        print('File already exists. Loading data from CSV')
        df = pd.read_csv(file_to_save)
        df = df.drop('Unnamed: 0', axis=1)

else:

    # ====================== Loading Data from Kaggle ==================================
    # You will be using HP's data. Feel free to experiment with other data.
    # But while doing so, be careful to have a large enough dataset and also pay attention to the data normalization
    df = pd.read_csv(os.path.join('Stocks','hpq.us.txt'),delimiter=',',usecols=['Date','Open','High','Low','Close'])
    print('Loaded data from the Kaggle repository')

#Data saved to : stock_market_data-AAL.csv


# Sort DataFrame by date
df = df.sort_values('Date')

#print(df)

df['Date'] = pd.to_datetime(df['Date'])  # Convert 'Date' column to datetime
df.set_index('Date', inplace=True)  # Set 'Date' column as index

# Calculate moving averages
df['MA20'] = df['Close'].rolling(window=20).mean()
df['MA50'] = df['Close'].rolling(window=50).mean()
df['MA200'] = df['Close'].rolling(window=200).mean()

# Calculate RSI and ATR
df['RSI'] = ta.rsi(df['Close'])
df['ATR'] = ta.atr(df['High'], df['Low'], df['Close'])

df.dropna(inplace=True)  # Drop rows with missing moving averages

# Calculate SMA (Simple Moving Average)
def calculate_sma(df, window):
    return df['Close'].rolling(window=window).mean()

# Calculate EMA (Exponential Moving Average)
def calculate_ema(df, window):
    return df['Close'].ewm(span=window, adjust=False).mean()

# Calculate MACD (Moving Average Convergence Divergence)
def calculate_macd(df, short_window, long_window, signal_window):
    short_ema = calculate_ema(df, short_window)
    long_ema = calculate_ema(df, long_window)
    macd = short_ema - long_ema
    signal_line = macd.ewm(span=signal_window, adjust=False).mean()
    return macd, signal_line

# Calculate Bollinger Bands
def calculate_bollinger_bands(df, window, num_std_dev):
    rolling_mean = df['Close'].rolling(window=window).mean()
    rolling_std = df['Close'].rolling(window=window).std()
    upper_band = rolling_mean + (rolling_std * num_std_dev)
    lower_band = rolling_mean - (rolling_std * num_std_dev)
    return upper_band, lower_band

df.dropna(inplace=True)  # Drop rows with missing moving averages

# Calculate and add SMA_20
df['SMA_20'] = calculate_sma(df, 20)

# Calculate and add SMA_20
df['SMA_50'] = calculate_sma(df, 50)

# Calculate and add SMA_20
df['SMA_200'] = calculate_sma(df, 200)

# Calculate and add EMA_20
df['EMA_20'] = calculate_ema(df, 20)

# Calculate and add SMA_20
df['EMA_50'] = calculate_ema(df, 50)

# Calculate and add SMA_20
df['EMA_200'] = calculate_ema(df, 200)

# Calculate MACD and Signal Line
macd, signal_line = calculate_macd(df, 12, 26, 9)
df['MACD'] = macd
df['Signal_Line'] = signal_line

# Calculate and add Bollinger Bands
upper_band, lower_band = calculate_bollinger_bands(df, 20, 2)
df['Bollinger_Upper'] = upper_band
df['Bollinger_Lower'] = lower_band

df.dropna(inplace=True)  # Drop rows with missing moving averages

# Define functions to identify candlestick patterns

def is_doji(row):
    return abs(row['Open'] - row['Close']) < 0.1 * (row['High'] - row['Low'])

def is_hammer(row):
    return (row['Close'] > row['Open']) and ((row['Close'] - row['Low']) / (0.001 + row['High'] - row['Low']) > 0.6)

def is_bullish_engulfing(df, index):
    if index < 1:
        return False
    yesterday = df.iloc[index - 1]
    today = df.iloc[index]
    return (yesterday['Open'] > yesterday['Close']) and (today['Open'] < today['Close']) and (today['Close'] > yesterday['Open']) and (today['Open'] < yesterday['Close'])

def is_bearish_engulfing(df, index):
    if index < 1:
        return False
    yesterday = df.iloc[index - 1]
    today = df.iloc[index]
    return (yesterday['Open'] < yesterday['Close']) and (today['Open'] > today['Close']) and (today['Close'] < yesterday['Open']) and (today['Open'] > yesterday['Close'])

def is_morning_star(df, index):
    if index < 2:
        return False
    yesterday = df.iloc[index - 2]
    today = df.iloc[index - 1]
    tomorrow = df.iloc[index]
    return (yesterday['Open'] > yesterday['Close']) and (today['Close'] > yesterday['Open']) and (tomorrow['Open'] < today['Close'])

def is_evening_star(df, index):
    if index < 2:
        return False
    yesterday = df.iloc[index - 2]
    today = df.iloc[index - 1]
    tomorrow = df.iloc[index]
    return (yesterday['Open'] < yesterday['Close']) and (today['Close'] < yesterday['Open']) and (tomorrow['Open'] > today['Close'])

def is_three_white_soldiers(df, index):
    if index < 2:
        return False
    prev_3_candles = df.iloc[index - 2 : index + 1]
    return all(prev_3_candles['Close'] > prev_3_candles['Open'])

def is_three_black_crows(df, index):
    if index < 2:
        return False
    prev_3_candles = df.iloc[index - 2 : index + 1]
    return all(prev_3_candles['Open'] > prev_3_candles['Close'])

def is_shooting_star(row):
    return (row['Open'] > row['Close']) and ((row['High'] - row['Close']) / (0.001 + row['High'] - row['Low']) > 0.6)

def is_inverted_hammer(row):
    return (row['Open'] < row['Close']) and ((row['Close'] - row['Low']) / (0.001 + row['High'] - row['Low']) > 0.6)

# Identify patterns in the data

df['Doji'] = df.apply(is_doji, axis=1)
df['Hammer'] = df.apply(is_hammer, axis=1)
df['Bullish Engulfing'] = [is_bullish_engulfing(df, i) for i in range(len(df))]
df['Bearish Engulfing'] = [is_bearish_engulfing(df, i) for i in range(len(df))]
df['Morning Star'] = [is_morning_star(df, i) for i in range(len(df))]
df['Evening Star'] = [is_evening_star(df, i) for i in range(len(df))]
df['Three White Soldiers'] = [is_three_white_soldiers(df, i) for i in range(len(df))]
df['Three Black Crows'] = [is_three_black_crows(df, i) for i in range(len(df))]
df['Shooting Star'] = df.apply(is_shooting_star, axis=1)
df['Inverted Hammer'] = df.apply(is_inverted_hammer, axis=1)

# Aroon   
from sklearn.preprocessing import LabelEncoder
import ta
def aroon(df, periods):
    aroon_up = []
    aroon_down = []
    for i in range(len(df)):
        if i < periods:
            aroon_up.append(np.nan)
            aroon_down.append(np.nan)
        else:
            highs = list(df['High'].iloc[i-periods:i])
            lows = list(df['Low'].iloc[i-periods:i])
            aroon_up.append(((highs.index(max(highs)) / periods) * 100))
            aroon_down.append(((lows.index(min(lows)) / periods) * 100))
    df['Aroon_up'] = aroon_up
    df['Aroon_down'] = aroon_down

    # Fill NaN values with the last observed non-null value
    df['Aroon_up'].fillna(method='ffill', inplace=True)
    df['Aroon_down'].fillna(method='ffill', inplace=True)

aroon(df, 25)

# Create 'Trend' column
df['Trend'] = 'Undefined'
df.loc[(df['Aroon_up'] < 50) & (df['Aroon_down'] < 50), 'Trend'] = 'Consolidation'

# Apply LabelEncoder
le = LabelEncoder()
df['Trend'] = le.fit_transform(df['Trend'])
    
    # Calculate ADX #### TBA

df.dropna(inplace=True)  # Drop rows with missing moving averages
##### Lag Features #####

# Define the window size
window_size_lag = 5

# Create rolling windows for SMA_20
df['SMA_20_Rolling_Mean'] = df['SMA_20'].rolling(window=window_size_lag).mean()

df['SMA_50_Rolling_Mean'] = df['SMA_50'].rolling(window=window_size_lag).mean()

df['SMA_200_Rolling_Mean'] = df['SMA_200'].rolling(window=window_size_lag).mean()

# Create rolling windows for EMA_20
df['EMA_20_Rolling_Mean'] = df['EMA_20'].rolling(window=window_size_lag).mean()

df['EMA_50_Rolling_Mean'] = df['EMA_50'].rolling(window=window_size_lag).mean()

df['EMA_200_Rolling_Mean'] = df['EMA_200'].rolling(window=window_size_lag).mean()

# Create rolling windows for MACD
df['MACD_Rolling_Mean'] = df['MACD'].rolling(window=window_size_lag).mean()

# Create rolling windows for Signal Line
df['Signal_Line_Rolling_Mean'] = df['Signal_Line'].rolling(window=window_size_lag).mean()

# Create rolling windows for Bollinger Bands
df['Bollinger_Upper_Rolling_Mean'] = df['Bollinger_Upper'].rolling(window=window_size_lag).mean()
df['Bollinger_Lower_Rolling_Mean'] = df['Bollinger_Lower'].rolling(window=window_size_lag).mean()

# Create rolling windows for the candlestick patterns
df['Doji_Rolling_Mean'] = df['Doji'].rolling(window=window_size_lag).mean()
df['Hammer_Rolling_Mean'] = df['Hammer'].rolling(window=window_size_lag).mean()
df['Bullish Engulfing_Rolling_Mean'] = df['Bullish Engulfing'].rolling(window=window_size_lag).mean()
df['Bearish Engulfing_Rolling_Mean'] = df['Bearish Engulfing'].rolling(window=window_size_lag).mean()
df['Morning Star_Rolling_Mean'] = df['Morning Star'].rolling(window=window_size_lag).mean()
df['Evening Star_Rolling_Mean'] = df['Evening Star'].rolling(window=window_size_lag).mean()
df['Three White Soldiers_Rolling_Mean'] = df['Three White Soldiers'].rolling(window=window_size_lag).mean()
df['Three Black Crows_Rolling_Mean'] = df['Three Black Crows'].rolling(window=window_size_lag).mean()
df['Shooting Star_Rolling_Mean'] = df['Shooting Star'].rolling(window=window_size_lag).mean()
df['Inverted Hammer_Rolling_Mean'] = df['Inverted Hammer'].rolling(window=window_size_lag).mean()

df.replace(np.nan, 0, inplace=True)

# Merge Data

# Load the merged data
merged_data = pd.read_csv('merged_data.csv', index_col=0, parse_dates=True)

# Fill missing values
merged_data.fillna(method='ffill', inplace=True)

# Fill missing values
merged_data.fillna(method='bfill', inplace=True)

# Merge df with merged_data
df = pd.merge(df, merged_data, left_index=True, right_index=True, how='inner')

# Print the merged DataFrame
print(df)

#columns_to_drop = ["Low", "High", "Open"]

#df = df.drop(columns=columns_to_drop)

num_columns = len(df.columns)
print(f"Number of variables (columns) in the DataFrame: {num_columns}")

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.preprocessing import MinMaxScaler
from keras.models import Sequential
from keras.layers import LSTM, Dense, Dropout
from keras.callbacks import EarlyStopping
import tensorflow as tf  # For dropout during inference

# Load and preprocess data (assuming `df` is already loaded as a DataFrame with a 'Close' column)
scaler = MinMaxScaler()
target = 'Close'

# Scale data
df_scaled = pd.DataFrame(scaler.fit_transform(df), columns=df.columns, index=df.index)

# Prepare sequences
window_size = 5  # Change based on your preference
P = 5  # Number of future days to predict
X, y = [], []

# Use all data for training (no exclusion of last P rows)
df_train = df  # Use entire dataset for training

for i in range(window_size, len(df_train) - P + 1):
    X.append(df_scaled.iloc[i - window_size:i].values)
    y.append(df_scaled[target].values[i:i + P])  # Collect P future values

X = np.array(X)
y = np.array(y)  # y is 2D: P target values per input sequence

# Define the model once outside the loop
def create_model():
    model = Sequential([
        LSTM(70, return_sequences=False, input_shape=(window_size, len(df.columns))),
        Dropout(0.1),
        Dense(35, activation='swish'),
        Dense(P)  # Predict P steps ahead
    ])
    model.compile(optimizer='RMSprop', loss='huber_loss', metrics=['mae', 'mse'])
    return model

# Create the base model
base_model = create_model()

# Define early stopping callback
early_stopping = EarlyStopping(monitor='loss', patience=2, restore_best_weights=True)

# Train the model on the entire dataset
base_model.fit(X, y, epochs=100, batch_size=32, verbose=1, callbacks=[early_stopping])

# Predict the next P days of prices (using bootstrapping) for all data
last_window = df_scaled.iloc[-window_size:].values

# Generate bootstrap samples of predictions
num_bootstrap_samples = 20  # Number of bootstrap samples
bootstrap_predictions = np.zeros((num_bootstrap_samples, P))

# Generate predictions for each bootstrap sample
for i in range(num_bootstrap_samples):
    # Sample with replacement from the training data (valid X indices)
    sampled_indices = np.random.choice(len(X), size=len(X), replace=True)  # Sample from valid range
    
    # Get bootstrapped X and y
    bootstrap_X = X[sampled_indices]
    bootstrap_y = y[sampled_indices]
    
    # Reinitialize the weights of the base model for each bootstrap iteration
    bootstrap_model = create_model()
    bootstrap_model.set_weights(base_model.get_weights())  # Load weights from the trained model
    
    # Train the model on the bootstrap sample
    bootstrap_model.fit(bootstrap_X, bootstrap_y, epochs=10, batch_size=32, verbose=0)
    
    # Predict the next P days for the bootstrapped model
    bootstrap_predictions[i] = bootstrap_model.predict(np.array([last_window]))[0]

# Calculate percentiles for the predictions
percentiles = np.percentile(bootstrap_predictions, [5, 50, 95], axis=0)

# Inverse transform predictions to original price scale
predicted_prices_5 = scaler.inverse_transform(np.tile(percentiles[0, :], (len(df.columns), 1)).T)[:, df.columns.get_loc(target)]
predicted_prices_50 = scaler.inverse_transform(np.tile(percentiles[1, :], (len(df.columns), 1)).T)[:, df.columns.get_loc(target)]
predicted_prices_95 = scaler.inverse_transform(np.tile(percentiles[2, :], (len(df.columns), 1)).T)[:, df.columns.get_loc(target)]

# Extract the dates for the predictions (just the last P days, as actual prices do not exist)
prediction_dates = pd.date_range(df.index[-1] + pd.Timedelta(days=1), periods=P, freq='D')

# Create a DataFrame for comparison (no actual prices since we're predicting)
prediction_df = pd.DataFrame({
    'Date': prediction_dates,
    'Predicted_Close_5': predicted_prices_5,
    'Predicted_Close_50': predicted_prices_50,
    'Predicted_Close_95': predicted_prices_95
})

print("Predicted Prices with Percentiles:")
print(prediction_df)

# Extract the actual last P days' prices for comparison
actual_prices = df.iloc[-5:][target].values

# Prepare the dates for the comparison
last_dates = df.index[-5:]

# Plotting the results
plt.figure(figsize=(12, 6))

# Plot predicted 50th percentile (median) prices as green line
plt.scatter(last_dates, actual_prices, label='Current Data', color='orange', s=20)

# Plot predicted 5th percentile prices as red line
plt.plot(prediction_dates, predicted_prices_5, label='5th Percentile', color='red', linestyle='--')

# Plot predicted 50th percentile (median) prices as green line
plt.scatter(prediction_dates, predicted_prices_50, label='50th Percentile (Median)', color='green', s=20)

# Plot predicted 95th percentile prices as red line
plt.plot(prediction_dates, predicted_prices_95, label='95th Percentile', color='red', linestyle='--')

# Add title and labels
plt.title('Predicted Stock Prices with Percentiles ' + ticker)
plt.xlabel('Date')
plt.ylabel('Price')

# Add legend
plt.legend(loc='upper left')

# Add grid
plt.grid(True)

# Display the plot
plt.show()