CAT/get_time_series.py at main · Boiler-Quant/CAT · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
#### idea is to create a timed I/O s.t
import sys
import ccxt
import time
import pandas as pd
import threading
# supported_exchanges = []
# exchange_names = ccxt.exchanges
# for exchange_id in exchange_names:
#     exchange_kraken = getattr(ccxt, exchange_id)()
#     if exchange_kraken.has['fetchOrderBooks']:
#         supported_exchanges.append(exchange_id)
# print(supported_exchanges)
# print(symbols)

# euth = exchange_kraken.markets['ETH/USDT']
# print(euth)

# greeks = exchange_kraken.fetchGreeks('ETH/USDT')
# print(greeks)
# print("VOLOTILIY", exchange_kraken.fetchVolatilityHistory('BTC'))
#print(supported_exchanges)

# Calculate the duration by subtracting the start time from the current time

# exchange_list = ['gateio', 'bitfinex', 'bitmart', 'digifinex', 'kraken', 'bitvavo']
# for exchagnge_id in exchange_list:
#     exchange_obj = getattr(ccxt, exchange_id)()
#     start_time = time.time()

#     exchange_obj.loadMarkets()

#     symbols = exchange_obj.symbols
#     for symbol in symbols:
#         order_book = exchange_obj.fetchOrderBook(symbol, limit = 5)
#         print(order_book)
#     duration = time.time() - start_time

#     print(duration)
# exchanges_list = ['bequant', 'bitcoincom', 'hitbtc', 'hitbtc3', 'hollaex', 'oceanex', 'upbit']
# # exchanges_list = ['bequant', 'bitcoincom', 'hitbtc', 'hitbtc3', 'hollaex', 'oceanex', 'upbit']
# # taken out = yobit, and fmfwio
# row_num = 500
# interval = 5 ### number of secinds intervals
# def queryTimeSeries(exchange_name):
#     exchange_obj = getattr(ccxt, exchange_name)()


#     df = pd.DataFrame(columns=['Timestamp', 'OrderBook'])

# # Target interval between starts of data collection iterations (5 seconds)
#     index = 0
#     #new_list = []
#     while True:
#         start_time = time.time()
#         timestamp = int(start_time)

#         order_book =  exchange_obj.fetchOrderBooks()
#         #new_list.append(order_book)

#         df_new = pd.DataFrame({'Timestamp': timestamp, 'OrderBook': str(order_book)}, index = [index])

#         df = pd.concat([df, df_new])
#         execution_time = time.time() - start_time
#         print("this is execution time:", execution_time)
#         time_to_sleep = max(0, interval - execution_time)
#         print("BAD IF ALWAYS 0:", time_to_sleep)
#         time.sleep(time_to_sleep)
#         index +=1

#         if len(df) >= row_num:

#            # print(str(sys.getsizeof(new_list)))
#             return df
#             break

# full_start_time = time.time()
# for exchange_name in exchanges_list:
#     df = queryTimeSeries(exchange_name)

#     file_path = './time_series/' + exchange_name + '_time_series_order_book.csv'
#     df.to_csv(file_path, index=False)

# full_end_time =  time.time()

# total_time = full_end_time - full_start_time

# print(total_time)
exchanges_list = ['bequant', 'bitcoincom', 'hitbtc', 'hitbtc3', 'hollaex', 'oceanex', 'upbit']

# Function to query timeseries data for a given exchange
def queryTimeSeries(exchange_name):
    exchange_obj = getattr(ccxt, exchange_name)()
    df = pd.DataFrame(columns=['Timestamp', 'OrderBook'])
    index = 0
    while True:
        start_time = time.time()
        timestamp = int(start_time)

        # Assuming this is a pseudo method for demonstration; replace with actual method to fetch order book
        order_book = exchange_obj.fetchOrderBooks()

        df_new = pd.DataFrame({'Timestamp': timestamp, 'OrderBook': str(order_book)}, index=[index])
        df = pd.concat([df, df_new])
        execution_time = time.time() - start_time
        print(f"Execution time for {exchange_name}: {execution_time}")
        time_to_sleep = max(0, interval - execution_time)
        time.sleep(time_to_sleep)
        index += 1

        if len(df) >= row_num:
            file_path = f'./timeseries/{exchange_name}_time_series_order_book.csv'
            df.to_csv(file_path, index=False)
            break

# Number of rows to collect and interval
row_num = 1000
interval = 5  # Number of seconds intervals

# Main function to start threads for each exchange
def main():
    threads = []
    for exchange_name in exchanges_list:
        thread = threading.Thread(target=queryTimeSeries, args=(exchange_name,))
        threads.append(thread)
        thread.start()

    # Wait for all threads to complete
    for thread in threads:
        thread.join()

if __name__ == "__main__":
    full_start_time = time.time()
    main()
    full_end_time = time.time()
    total_time = full_end_time - full_start_time
    print(f"Total execution time: {total_time}")