diff_api_predictor.py

################## TEST_UNLIMITED ###################
import requests
import json
import pandas as pd
import os
import urllib

############################

import time
from collections import deque, namedtuple
from keras.models import Sequential 
from keras.layers import Dense, LSTM, Dropout
import numpy as np
import PIL.Image
import tensorflow as tf
import utils

from pyvirtualdisplay import Display
from keras import Sequential
from keras.losses import MSE
from keras.optimizers import Adam

####################

import numpy as np
import pandas as pd
from pandas_datareader import data
import matplotlib.pyplot as plt
import datetime as dt
import os
import urllib.request, json
import tensorflow as keras  # Use the official 'tensorflow' package instead of 'keras'
from keras.models import Sequential 
from keras.layers import Dense, LSTM, Dropout
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split
from keras.activations import sigmoid
from keras.callbacks import ModelCheckpoint, EarlyStopping
from keras.models import load_model
from keras import layers
from keras.layers import LSTM, Dropout, Dense, Input
from keras.models import Model
import pandas_ta as ta
from sklearn.model_selection import TimeSeriesSplit
from sklearn.metrics import mean_squared_error
from math import sqrt
import pandas_market_calendars as mcal 
from sklearn.model_selection import KFold
from sklearn.preprocessing import StandardScaler

import os
import requests
import pandas as pd
from pandas import json_normalize

# Your API Key
api_key = '<RR02P5TH2UNCD10X>'

# Stock ticker symbol
ticker = "TSLA"

# CSV file path
csv_file_path = 'test_unlimited_stock_market_data-%s.csv'%ticker

# Check if the CSV file already exists
if os.path.isfile(csv_file_path):
    print("File already exists. Reading the file...")
    try:
        df = pd.read_csv(csv_file_path, parse_dates=['Date'])
        print(df.head())  # print the first 5 rows of the DataFrame
    except Exception as e:
        print(f"Error while reading file: {e}")
else:
        # Define the API endpoints
    daily_time_series_url = f"https://www.alphavantage.co/query?function=TIME_SERIES_DAILY&symbol={ticker}&outputsize=full&apikey={api_key}"
    overview_url = f"https://www.alphavantage.co/query?function=OVERVIEW&symbol={ticker}&apikey={api_key}"

    # Fetch the data from each endpoint
    daily_time_series_response = requests.get(daily_time_series_url)
    overview_response = requests.get(overview_url)

# Convert the responses to JSON
    daily_time_series_data = daily_time_series_response.json()
    overview_data = overview_response.json()

# Extract the time series data into a DataFrame
    time_series_data = daily_time_series_data['Time Series (Daily)']
    time_series_df = pd.DataFrame.from_dict(time_series_data, orient='index')
    time_series_df.index.name = 'Date'
    time_series_df.reset_index(drop=False, inplace=True)

# Convert the overview data into a DataFrame
    overview_df = pd.DataFrame([overview_data])

# Repeat the overview data to match the length of the time series data
    overview_df = pd.concat([overview_df]*len(time_series_df), ignore_index=True)

# Concatenate the DataFrames horizontally
    result_df = pd.concat([time_series_df, overview_df], axis=1)

# Ensure 'Date' column is of datetime type
    result_df['Date'] = pd.to_datetime(result_df['Date'])

    # Save the DataFrame to a CSV file
    result_df.to_csv('test_unlimited_stock_market_data-%s.csv'%ticker, index=False)

    df = (result_df)

    # List of columns to convert
    columns_to_convert = ['1. open', '2. high', '3. low', '4. close', '5. volume',
                          'MarketCapitalization', 'EBITDA', 'PERatio', 'PEGRatio', 
                          'BookValue', 'DividendPerShare', 'DividendYield', 'EPS',
                          'RevenuePerShareTTM', 'ProfitMargin', 'OperatingMarginTTM',
                          'ReturnOnAssetsTTM', 'ReturnOnEquityTTM', 'RevenueTTM',
                          'GrossProfitTTM', 'DilutedEPSTTM', 'QuarterlyEarningsGrowthYOY',
                          'QuarterlyRevenueGrowthYOY', 'AnalystTargetPrice', 'TrailingPE',
                          'ForwardPE', 'PriceToSalesRatioTTM', 'PriceToBookRatio', 
                          'EVToRevenue', 'EVToEBITDA', 'Beta', '52WeekHigh', '52WeekLow', 
                          '50DayMovingAverage', '200DayMovingAverage', 'SharesOutstanding']

    # Convert columns to numeric
    for column in columns_to_convert:
        df[column] = pd.to_numeric(df[column], errors='coerce')

    columns_to_drop = ['Description', 'Symbol', 'AssetType', 'Name', 'CIK', 'Exchange', 'Currency', 
                       'Country', 'Sector', 'Industry', 'Address', 'FiscalYearEnd', 'LatestQuarter', 'DividendDate' , 'ExDividendDate', 'PERatio',
                       'PEGRatio', 'MarketCapitalization', 'DividendPerShare', 'DividendYield', 'BookValue', 'EBITDA', 'RevenuePerShareTTM', 'RevenuePerShareTTM', 
                       'EPS', 'ProfitMargin', 'OperatingMarginTTM', 'ReturnOnAssetsTTM', 'ReturnOnEquityTTM', 'RevenueTTM', 'GrossProfitTTM', 
                       'DilutedEPSTTM', 'QuarterlyEarningsGrowthYOY', 'QuarterlyRevenueGrowthYOY', 'AnalystTargetPrice','TrailingPE', 'ForwardPE',
                       'PriceToSalesRatioTTM', 'PriceToBookRatio', 'EVToRevenue',  'EVToEBITDA', 'Beta', '52WeekHigh', '52WeekLow',
                       '50DayMovingAverage', '200DayMovingAverage',  'SharesOutstanding']
    df = df.drop(columns=columns_to_drop)
    df = df.rename(columns={'1. open': 'open', '2. high': 'high', '3. low': 'low', '4. close': 'close', '5. volume': 'volume'})

    # Sort DataFrame by date
    df = df.sort_values('Date')

# Calculate RSI and ATR
    df['RSI'] = ta.rsi(df['close'])
    df['ATR'] = ta.atr(df['high'], df['low'], df['close'])

# Calculate SMA (Simple Moving Average)
    def calculate_sma(df, window):
        return df['close'].rolling(window=window).mean()

# Calculate EMA (Exponential Moving Average)
    def calculate_ema(df, window):
        return df['close'].ewm(span=window, adjust=False).mean()

# Calculate MACD (Moving Average Convergence Divergence)
    def calculate_macd(df, short_window, long_window, signal_window):
        short_ema = calculate_ema(df, short_window)
        long_ema = calculate_ema(df, long_window)
        macd = short_ema - long_ema
        signal_line = macd.ewm(span=signal_window, adjust=False).mean()
        return macd, signal_line

# Calculate Bollinger Bands
    def calculate_bollinger_bands(df, window, num_std_dev):
        rolling_mean = df['close'].rolling(window=window).mean()
        rolling_std = df['close'].rolling(window=window).std()
        upper_band = rolling_mean + (rolling_std * num_std_dev)
        lower_band = rolling_mean - (rolling_std * num_std_dev)
        return upper_band, lower_band

# Calculate and add SMA_20
    df['SMA_20'] = calculate_sma(df, 20)

# Calculate and add EMA_20
    df['EMA_20'] = calculate_ema(df, 20)

# Calculate MACD and Signal Line
    macd, signal_line = calculate_macd(df, 12, 26, 9)
    df['MACD'] = macd
    df['Signal_Line'] = signal_line

# Calculate and add Bollinger Bands
    upper_band, lower_band = calculate_bollinger_bands(df, 20, 2)
    df['Bollinger_Upper'] = upper_band
    df['Bollinger_Lower'] = lower_band
    
# Aroon   
    from sklearn.preprocessing import LabelEncoder
    import ta
    def aroon(df, periods):
        aroon_up = []
        aroon_down = []
        for i in range(len(df)):
            if i < periods:
                aroon_up.append(np.nan)
                aroon_down.append(np.nan)
            else:
                highs = list(df['high'].iloc[i-periods:i])
                lows = list(df['low'].iloc[i-periods:i])
                aroon_up.append(((highs.index(max(highs)) / periods) * 100))
                aroon_down.append(((lows.index(min(lows)) / periods) * 100))
        df['Aroon_up'] = aroon_up
        df['Aroon_down'] = aroon_down

    # Fill NaN values with the last observed non-null value
        df['Aroon_up'].fillna(method='ffill', inplace=True)
        df['Aroon_down'].fillna(method='ffill', inplace=True)

    aroon(df, 25)

# Create 'Trend' column
    df['Trend'] = 'Undefined'
    df.loc[(df['Aroon_up'] < 50) & (df['Aroon_down'] < 50), 'Trend'] = 'Consolidation'

# Apply LabelEncoder
    le = LabelEncoder()
    df['Trend'] = le.fit_transform(df['Trend'])
    
    # Calculate ADX #### TBA

    df.dropna(inplace=True)  # Drop rows with missing moving averages

    try:
        df.to_csv('test_unlimited_stock_market_data-%s.csv'%ticker, index=False)
        print("File saved successfully")
    except Exception as e:
        print(f"Error while saving file: {e}")
    
# Sort DataFrame by date
df = df.sort_values('Date')

##### Lag Features #####

# Define functions to identify candlestick patterns

def is_doji(row):
    return abs(row['open'] - row['close']) < 0.1 * (row['high'] - row['low'])

def is_hammer(row):
    return (row['close'] > row['open']) and ((row['close'] - row['low']) / (0.001 + row['high'] - row['low']) > 0.6)

def is_bullish_engulfing(df, index):
    if index < 1:
        return False
    yesterday = df.iloc[index - 1]
    today = df.iloc[index]
    return (yesterday['open'] > yesterday['close']) and (today['open'] < today['close']) and (today['close'] > yesterday['open']) and (today['open'] < yesterday['close'])

def is_bearish_engulfing(df, index):
    if index < 1:
        return False
    yesterday = df.iloc[index - 1]
    today = df.iloc[index]
    return (yesterday['open'] < yesterday['close']) and (today['open'] > today['close']) and (today['close'] < yesterday['open']) and (today['open'] > yesterday['close'])

def is_morning_star(df, index):
    if index < 2:
        return False
    yesterday = df.iloc[index - 2]
    today = df.iloc[index - 1]
    tomorrow = df.iloc[index]
    return (yesterday['open'] > yesterday['close']) and (today['close'] > yesterday['open']) and (tomorrow['open'] < today['close'])

def is_evening_star(df, index):
    if index < 2:
        return False
    yesterday = df.iloc[index - 2]
    today = df.iloc[index - 1]
    tomorrow = df.iloc[index]
    return (yesterday['open'] < yesterday['close']) and (today['close'] < yesterday['open']) and (tomorrow['open'] > today['close'])

def is_three_white_soldiers(df, index):
    if index < 2:
        return False
    prev_3_candles = df.iloc[index - 2 : index + 1]
    return all(prev_3_candles['close'] > prev_3_candles['open'])

def is_three_black_crows(df, index):
    if index < 2:
        return False
    prev_3_candles = df.iloc[index - 2 : index + 1]
    return all(prev_3_candles['open'] > prev_3_candles['close'])

def is_shooting_star(row):
    return (row['open'] > row['close']) and ((row['high'] - row['close']) / (0.001 + row['high'] - row['low']) > 0.6)

def is_inverted_hammer(row):
    return (row['open'] < row['close']) and ((row['close'] - row['low']) / (0.001 + row['high'] - row['low']) > 0.6)

# Identify patterns in the data

df['Doji'] = df.apply(is_doji, axis=1)
df['Hammer'] = df.apply(is_hammer, axis=1)
df['Bullish Engulfing'] = [is_bullish_engulfing(df, i) for i in range(len(df))]
df['Bearish Engulfing'] = [is_bearish_engulfing(df, i) for i in range(len(df))]
df['Morning Star'] = [is_morning_star(df, i) for i in range(len(df))]
df['Evening Star'] = [is_evening_star(df, i) for i in range(len(df))]
df['Three White Soldiers'] = [is_three_white_soldiers(df, i) for i in range(len(df))]
df['Three Black Crows'] = [is_three_black_crows(df, i) for i in range(len(df))]
df['Shooting Star'] = df.apply(is_shooting_star, axis=1)
df['Inverted Hammer'] = df.apply(is_inverted_hammer, axis=1)

df.dropna(inplace=True)  # Drop rows with missing moving averages

# Define the window size
window_size_lag = 7

# Create rolling windows for SMA_20
df['SMA_20_Rolling_Mean'] = df['SMA_20'].rolling(window=window_size_lag).mean()

# Create rolling windows for EMA_20
df['EMA_20_Rolling_Mean'] = df['EMA_20'].rolling(window=window_size_lag).mean()

# Create rolling windows for MACD
df['MACD_Rolling_Mean'] = df['MACD'].rolling(window=window_size_lag).mean()

# Create rolling windows for Signal Line
df['Signal_Line_Rolling_Mean'] = df['Signal_Line'].rolling(window=window_size_lag).mean()

# Create rolling windows for Bollinger Bands
df['Bollinger_Upper_Rolling_Mean'] = df['Bollinger_Upper'].rolling(window=window_size_lag).mean()
df['Bollinger_Lower_Rolling_Mean'] = df['Bollinger_Lower'].rolling(window=window_size_lag).mean()

# Create rolling windows for the candlestick patterns
df['Doji_Rolling_Mean'] = df['Doji'].rolling(window=window_size_lag).mean()
df['Hammer_Rolling_Mean'] = df['Hammer'].rolling(window=window_size_lag).mean()
df['Bullish Engulfing_Rolling_Mean'] = df['Bullish Engulfing'].rolling(window=window_size_lag).mean()
df['Bearish Engulfing_Rolling_Mean'] = df['Bearish Engulfing'].rolling(window=window_size_lag).mean()
df['Morning Star_Rolling_Mean'] = df['Morning Star'].rolling(window=window_size_lag).mean()
df['Evening Star_Rolling_Mean'] = df['Evening Star'].rolling(window=window_size_lag).mean()
df['Three White Soldiers_Rolling_Mean'] = df['Three White Soldiers'].rolling(window=window_size_lag).mean()
df['Three Black Crows_Rolling_Mean'] = df['Three Black Crows'].rolling(window=window_size_lag).mean()
df['Shooting Star_Rolling_Mean'] = df['Shooting Star'].rolling(window=window_size_lag).mean()
df['Inverted Hammer_Rolling_Mean'] = df['Inverted Hammer'].rolling(window=window_size_lag).mean()

df.replace(np.nan, 0, inplace=True)


# Dates
import datetime


df['Date'] = pd.to_datetime(df['Date'])  # Convert 'Date' column to datetime
df.set_index('Date', inplace=True)  
print(df)

# Load the merged data
merged_data = pd.read_csv('merged_data.csv', index_col=0, parse_dates=True)

# Fill missing values
merged_data.fillna(method='ffill', inplace=True)

# Fill missing values
merged_data.fillna(method='bfill', inplace=True)

# Merge df with merged_data
df = pd.merge(df, merged_data, left_index=True, right_index=True, how='inner')

# Print the merged DataFrame
print(df)


##### Feature Engineering #####

# Create a copy of 'Date' column
#date_column = df.pop('Date')

#columns_to_drop = [ "low", "high", "open"]

#df = df.drop(columns=columns_to_drop)

## Define P and the window size

P = 3

df_new = df.tail(P)

#df = df.drop(df.index[-P:])

window_size = P

print(df)

# Select features and target
features = df.copy()
target = 'close' 
num_feats = len(features.columns)

# Scale the features
scaler = MinMaxScaler()
df_scaled = pd.DataFrame(scaler.fit_transform(df), columns=df.columns)

print(len(features))

# Prepare the sequences for LSTM (we're using a window size of 30 days)
X = []
y = []

for i in range(window_size, len(df_scaled)):
    X.append(df_scaled.iloc[i-window_size:i, :].values)
    y.append(df_scaled.iloc[i, df_scaled.columns.get_loc(target)])

# Split the data into training, validation and testing sets
train_size = int(len(X) * 0.7)
val_size = int(len(X) * 0.2)
X_train, X_val, X_test = X[:train_size], X[train_size:train_size+val_size], X[train_size+val_size:]
y_train, y_val, y_test = y[:train_size], y[train_size:train_size+val_size], y[train_size+val_size:]

# Convert lists to numpy arrays
X_train = np.asarray(X_train)
y_train = np.asarray(y_train)
X_val = np.asarray(X_val)
y_val = np.asarray(y_val)
X_test = np.asarray(X_test)
y_test = np.asarray(y_test)

########## Hyperp Tuning ############# TBA

#from scikeras.wrappers import KerasRegressor
from sklearn.model_selection import RandomizedSearchCV
from keras.models import Sequential
from keras.layers import LSTM, Dense, Dropout
from sklearn.model_selection import GridSearchCV


    
########### Training The Model #######################

# Define the number of folds
num_folds = 5

# Define the TimeSeriesSplit object
tscv = TimeSeriesSplit(n_splits=num_folds)

# Define the EarlyStopping callback
early_stopping = EarlyStopping(monitor='val_loss', patience=2, restore_best_weights=False)

# Store the performance for each fold
loss_per_fold = []

# K-fold cross validation
fold_no = 1
for train, validation in tscv.split(X_train, y_train):
    # Define the model architecture
    model = Sequential([
    #layers.LSTM(50, return_sequences= True, input_shape=(window_size, num_feats)), # Set to False for low volatity stocks
    #layers.Dropout(0.4),
    layers.LSTM(50, return_sequences= False, input_shape=(window_size, num_feats)), # Set to False for low volatity stocks
    layers.Dropout(0.4),
    #layers.LSTM(40),
    #layers.Dropout(0.4),# Third LSTM layer without return_sequences=True (typically the last layer) # Add for high volaltity stocks
    layers.Dense(25, activation='tanh'),  # Added a Dense layer before the output
    layers.Dense(1)  # Output Close Prices
])


    # Compile the model
    model.compile(optimizer='adam', loss='mean_squared_error')

    # Fit the model
    history = model.fit(X_train[train], y_train[train], epochs=100, batch_size=20, validation_data=(X_train[validation], y_train[validation]), callbacks=[early_stopping])

    # Evaluate the model
    scores = model.evaluate(X_train[validation], y_train[validation], verbose=0)
    print(f'Score for fold {fold_no}: {model.metrics_names[0]} of {scores}')
    loss_per_fold.append(scores)

    # Increase fold number
    fold_no = fold_no + 1

# Provide average scores
print('Average scores for all folds:')
print(f'> Loss: {np.mean(loss_per_fold)}')

###Metric Scors###
from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score

# Predict on the validation set
y_val_pred = model.predict(X_val)

# Calculate metrics
mse = mean_squared_error(y_val, y_val_pred)
rmse = np.sqrt(mse)
mae = mean_absolute_error(y_val, y_val_pred)
r2 = r2_score(y_val, y_val_pred)

# Print metrics
print('MSE:', mse)
print('RMSE:', rmse)
print('MAE:', mae)
print('R2:', r2)


# Plot the loss
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('Model loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Train', 'Validation'], loc='upper right')
plt.show()

# Predict the values
predictions = model.predict(X_test)

# Create a temporary array filled with zeros
temp_array = np.zeros((predictions.shape[0], num_feats))

# Replace the first column of temp_array with your predictions
temp_array[:, 3] = predictions.ravel()

# Now you can use inverse_transform on temp_array
predicted_prices = scaler.inverse_transform(temp_array)

# The inverse transformed predictions are in the first column of predicted_prices
predicted_prices = predicted_prices[:, 3]

# Create a temporary array filled with zeros
temp_array = np.zeros((y_test.shape[0], num_feats))

# Replace the first column of temp_array with your y_test data
temp_array[:, 3] = y_test.ravel()

# Now you can use inverse_transform on temp_array
y_test_prices = scaler.inverse_transform(temp_array)

# The inverse transformed y_test data are in the first column of y_test_prices
y_test_prices = y_test_prices[:, 3]

# Plot the actual test data values
plt.plot(y_test_prices, label='Actual')

# Plot the predicted values
plt.plot(predicted_prices, label='Predicted')

# Set the labels and title
plt.xlabel('Time')
plt.ylabel('Stock Price')
plt.title('Actual vs Predicted Stock Prices')

# Show the legend
plt.legend()

# Show the plot
plt.show()

###### Predictions ######

# Define the number of future days you want to predict (30 days)
future_days = window_size

# Get the most recent window of data from the test set
last_window = X_test[-1]  # Use the last window from the test set

# Create an empty list to store future predictions
future_predictions = []

# prepare target_index
col_names = df.columns.tolist()
target_index = col_names.index('close')

# Predict future prices for the next 'future_days' days
for i in range(future_days):
    
    # Predict the next day's price using the model
    next_day_prediction = model.predict(np.array([last_window]))[0][0]
    
    # Append the prediction to the list
    future_predictions.append(next_day_prediction)
    
    # Update the 'last_window' with the new prediction
    last_window = np.roll(last_window, shift=-1, axis=0)
    last_window[-1][target_index] = next_day_prediction  # Update the last element with the new prediction

# Create a temporary array with the same number of features (20)
temp_array = np.zeros((future_days, num_feats))
temp_array[:, 3] = future_predictions

# Inverse transform the temporary array
predicted_prices = scaler.inverse_transform(temp_array)

# Generate future business dates starting from the last date in the test set
last_date = df.index[-1]
future_dates = pd.date_range(start=last_date, periods=future_days + 1, freq='B')[1:]

# Create a DataFrame to store the predicted prices and their corresponding dates
predicted_df = pd.DataFrame({'Date': future_dates, 'Predicted_Close': predicted_prices[:, 3]})

# Print the predicted DataFrame
print(predicted_df)

# Plot the predicted close prices
plt.figure(figsize=(12, 6))
plt.plot(predicted_df['Date'], predicted_df['Predicted_Close'], label='Predicted Close Price', color='blue')
plt.plot(df_new.index, df_new['close'], label='Actual Close Price', color='red')  # Use DataFrame's index as x-axis
plt.xlabel('Date')
plt.ylabel('Predicted Close Price')
plt.title('Predicted Close Price for the Next 10 Business Days ' + ticker)
plt.grid(True)
plt.legend()
plt.show()