Like 0
Fork 0

Last active 1729656337

cyberes's Avatar cyberes revised this gist 1729656337. Go to revision

1 file changed, 3 insertions, 1 deletion

README.md

@@ -1 +1,3 @@
1 - Data is available at
1 + Data is available at
2 +
3 + Use <https://github.com/elasticsearch-dump/elasticsearch-dump> to import into Elastic.

cyberes's Avatar cyberes revised this gist 1729655750. Go to revision

2 files changed, 287 insertions

README.md(file created)

@@ -0,0 +1 @@
1 + Data is available at

tally-costs.py(file created)

@@ -0,0 +1,286 @@
1 + #!/usr/bin/env python3
2 + import argparse
3 + import csv
4 + import locale
5 + import logging
6 + from datetime import datetime
7 + from typing import Union
8 +
9 + import elastic_transport
10 + import numpy as np
11 + from elasticsearch.helpers import scan
12 + from elasticsearch_dsl import Search, Q
13 + from pydantic import BaseModel
14 + from tqdm import tqdm
15 +
16 + from lib.config import ELASTIC_HOST, ELASTIC_INDEX, ELASTIC_API_KEY
17 + from lib.elastic.client import ElasticClient
18 +
19 + logging.basicConfig()
20 + logger = logging.getLogger('MAIN')
21 + logger.setLevel(logging.INFO)
22 +
23 + _IS_SERVICE = False
24 + _IS_TEST = False
25 + _WRITE_CSVS = False
26 +
27 +
28 + class Usage(BaseModel):
29 + type: str
30 + cost: Union[float, None] = None
31 + tokens: int
32 +
33 +
34 + API_COSTS_PER_1M = {
35 + # Input, output.
36 + 'turbo': np.mean([1.50, 2.00]),
37 + 'gpt4-turbo': np.mean([10.00, 30.00]),
38 + 'gpt4': np.mean([30.00, 60.00]),
39 + 'gpt4-32k': np.mean([60.00, 120.00]),
40 + 'gpt4o': np.mean([2.5, 10.00]),
41 + 'o1-mini': np.mean([3.00, 12.00]),
42 + 'o1': np.mean([15, 60]),
43 + 'claude': np.mean([8, 24]),
44 + 'claude-sonnet': np.mean([3, 15]),
45 + 'claude-opus': np.mean([15, 75]),
46 + 'gemini': np.mean([0.50, 1.50]),
47 + 'gemini-pro': np.mean([1.25, 5.00]),
48 + 'gemini-flash': np.mean([0.075, 0.30]),
49 + 'mistral-tiny': np.mean([2.5 / 4, 7.5 / 4]),
50 + 'mistral-small': np.mean([2.5 / 3, 7.5 / 3]),
51 + 'mistral-medium': np.mean([2.5 / 2, 7.5 / 2]),
52 + 'mistral-large': np.mean([2.5, 7.5]),
53 + 'aws-claude': np.mean([8, 24]),
54 + 'aws-claude-sonnet': np.mean([3, 15]),
55 + 'aws-claude-opus': np.mean([15, 75]),
56 + 'aws-mistral-small': np.mean([0.001 * 1000, 0.003 * 1000]),
57 + 'aws-mistral-large': np.mean([0.004 * 1000, 0.012 * 1000]),
58 + 'azure-gpt4-32k': np.mean([60.00, 120.00]),
59 + 'azure-gpt4': np.mean([30.00, 60.00]),
60 + 'azure-gpt4-turbo': np.mean([10.00, 30.00]),
61 + 'azure-gpt4o': np.mean([5, 15]),
62 + 'azure-turbo': np.mean([1.5, 2]),
63 +
64 + # These are ignored.
65 + # Not doing image models because the API returns the tokens, not the number of images generated.
66 + 'dall-e': 0,
67 + 'azure-dall-e': 0,
68 + 'palm-bison': 0,
69 + 'bison': 0,
70 + }
71 +
72 +
73 + def dynamic_print(msg: str):
74 + if _IS_SERVICE:
75 + logger.info(msg)
76 + else:
77 + tqdm.write(msg)
78 +
79 +
80 + def get_unique_urls(index_name="proxy_stats"):
81 + """
82 + Retrieves all unique URLs from the specified Elasticsearch index.
83 +
84 + :param index_name: Name of the index to search (default: "proxy_stats")
85 + :return: A list of unique URLs
86 + """
87 + s = Search(using=ElasticClient().client, index=index_name)
88 + s.aggs.bucket('unique_urls', 'terms', field='url.keyword', size=10000)
89 + s = s.extra(size=0)
90 + response = s.execute()
91 + unique_urls = [bucket.key for bucket in response.aggregations.unique_urls.buckets if bucket.key != 'proxy.chub-archive.evulid.cc']
92 + return unique_urls
93 +
94 +
95 + def process_docs_by_url(index_name="proxy_stats", batch_size=1000):
96 + """
97 + Processes documents for each unique URL in the index, sorted by timestamp.
98 +
99 + :param index_name: Name of the index to search (default: "proxy_stats")
100 + :param batch_size: Number of documents to fetch in each batch (default: 1000)
101 + """
102 + es_client = ElasticClient().client
103 + unique_urls = get_unique_urls(index_name)
104 + grand_total = 0
105 +
106 + main_pbar = tqdm(unique_urls, position=0, disable=_IS_SERVICE)
107 +
108 + for url in main_pbar:
109 + s = Search(using=es_client, index=index_name)
110 + s = s.query(Q('term', url__keyword=url)).sort({"timestamp": {"order": "asc"}})
111 + doc_count = s.count()
112 +
113 + if _IS_SERVICE:
114 + logger.info(f'Processing {doc_count} documents for {url}')
115 +
116 + csv_rows = []
117 + total_api_costs = {}
118 + total_cost = 0
119 + previous_doc = None
120 +
121 + docs = scan(es_client, query=s.to_dict(), index=index_name, size=batch_size, preserve_order=True)
122 + pbar = tqdm(docs, total=doc_count, desc=url, disable=_IS_SERVICE, leave=True, position=1)
123 +
124 + i = 0
125 + for current_doc in pbar:
126 + assert current_doc['_source']['url'] == url
127 + if _IS_TEST and i == 1400:
128 + previous_doc = None
129 + break
130 + i += 1
131 +
132 + current_timestamp = current_doc['_source']['timestamp']
133 + current_timestamp_d = datetime.fromtimestamp(current_timestamp)
134 +
135 + if previous_doc:
136 + previous_timestamp = previous_doc['_source']['timestamp']
137 + previous_timestamp_d = datetime.fromtimestamp(previous_timestamp)
138 + if current_timestamp_d < previous_timestamp_d:
139 + raise Exception(f'ASSERT FAILED: {current_timestamp_d} < {previous_timestamp_d}')
140 + current_uptime = current_doc['_source']['uptime']
141 + previous_uptime = previous_doc['_source']['uptime']
142 + if previous_uptime > current_uptime:
143 + # Process and save the previous document's stats (which were before the proxy restarted).
144 + new_costs = determine_costs(previous_doc['_source'])
145 + costs = combine_model_stats(total_api_costs, new_costs)
146 + total_cost += calc_total_cost(costs)
147 + timestamp = previous_timestamp_d.strftime('%m-%d-%Y %H:%M')
148 + csv_rows.append([new_costs, timestamp, previous_uptime])
149 +
150 + previous_doc = current_doc
151 + pbar.set_postfix({
152 + 'Timestamp': current_timestamp_d.strftime('%m-%d-%Y %H:%M'),
153 + 'Total Cost': f'{locale.currency(total_cost, grouping=True)}'
154 + })
155 + main_pbar.set_postfix({
156 + 'Grand Total': f'{locale.currency(grand_total, grouping=True)}'
157 + })
158 +
159 + # Process the last document
160 + if previous_doc:
161 + new_costs = determine_costs(previous_doc['_source'])
162 + costs = combine_model_stats(total_api_costs, new_costs)
163 + total_cost += calc_total_cost(costs)
164 + timestamp = datetime.fromtimestamp(previous_doc['_source']['timestamp']).strftime('%m-%d-%Y %H:%M')
165 + csv_rows.append([new_costs, timestamp, previous_doc['_source']['uptime']])
166 +
167 + grand_total += total_cost
168 + if _WRITE_CSVS and len(csv_rows):
169 + write_to_csv(csv_rows, filename=f'{url.replace(".", "-")}.csv')
170 + if _IS_SERVICE:
171 + logger.info(f'Total cost for {url}: {locale.currency(total_cost, grouping=True)}')
172 +
173 + return grand_total
174 +
175 +
176 + def determine_costs(doc):
177 + costs = {}
178 + for k, v in doc.items():
179 + if isinstance(v, dict) and v.get('usage'):
180 + usage = Usage(type=k, cost=v['usage']['cost'] if v['usage']['cost'] > 0 else None, tokens=v['usage']['tokens'])
181 + if k not in costs:
182 + costs[k] = []
183 + costs[k].append(usage)
184 + return calculate_costs(costs)
185 +
186 +
187 + def calculate_costs(data: dict):
188 + results = {}
189 + for k, v in data.items():
190 + if k not in results:
191 + results[k] = {
192 + 'cost': 0,
193 + 'tokens': 0,
194 + }
195 + for x in v:
196 + if x.cost:
197 + results[k]['cost'] = results[k]['cost'] + x.cost
198 + results[k]['tokens'] = results[k]['tokens'] + x.tokens
199 +
200 + for api_type, v in results.items():
201 + if v['cost'] == 0:
202 + v['cost'] = (v['tokens'] / 1_000_000) * API_COSTS_PER_1M[api_type]
203 +
204 + return results
205 +
206 +
207 + def calc_total_cost(data: dict):
208 + total = 0
209 + for k, v in data.items():
210 + total += v['cost']
211 + return np.round(total, 2)
212 +
213 +
214 + def combine_model_stats(dict1: dict, dict2: dict):
215 + result = {}
216 + all_models = set(dict1.keys()) | set(dict2.keys())
217 + for model in all_models:
218 + result[model] = {
219 + 'cost': (dict1.get(model, {}).get('cost', 0) +
220 + dict2.get(model, {}).get('cost', 0)),
221 + 'tokens': (dict1.get(model, {}).get('tokens', 0) +
222 + dict2.get(model, {}).get('tokens', 0))
223 + }
224 + return result
225 +
226 +
227 + def write_to_csv(data: list, filename: str):
228 + all_models = set()
229 + for row in data:
230 + all_models.update(row[0].keys())
231 +
232 + header = ['timestamp', 'uptime_seconds'] + [f'{model}_{metric}' for model in sorted(all_models) for metric in ['cost', 'tokens']]
233 +
234 + with open(filename, 'w', newline='') as csvfile:
235 + writer = csv.writer(csvfile)
236 + writer.writerow(header)
237 +
238 + for row in data:
239 + model_data = row[0]
240 + timestamp = row[1]
241 +
242 + if isinstance(timestamp, (int, float)):
243 + timestamp = datetime.fromtimestamp(timestamp).strftime('%m-%d-%Y %H:%M')
244 +
245 + csv_row = [timestamp]
246 +
247 + # Add uptime (if present, otherwise use 0)
248 + csv_row.append(row[2] if len(row) > 2 else 0)
249 +
250 + for model in sorted(all_models):
251 + if model in model_data:
252 + csv_row.extend([model_data[model].get('cost', 0), model_data[model].get('tokens', 0)])
253 + else:
254 + csv_row.extend([0, 0]) # Add zeros for missing models
255 +
256 + writer.writerow(csv_row)
257 +
258 + dynamic_print(f"Data has been written to {filename}")
259 +
260 +
261 + def main(args):
262 + try:
263 + ElasticClient.initialise(ELASTIC_HOST, ELASTIC_INDEX, ELASTIC_API_KEY)
264 + except elastic_transport.ConnectionError as e:
265 + logger.critical(f'Failed to connect to Elastic: {e}')
266 + quit(1)
267 +
268 + locale.setlocale(locale.LC_ALL, '')
269 +
270 + logger.info('Fetching URLs from Elastic...')
271 + unique_urls = get_unique_urls()
272 + logger.info(f'Found {len(unique_urls)} unique URLs.')
273 + grand_total = process_docs_by_url()
274 + dynamic_print(f'Total wasted....... {locale.currency(grand_total, grouping=True)}')
275 +
276 +
277 + if __name__ == '__main__':
278 + parser = argparse.ArgumentParser()
279 + parser.add_argument('-s', '--service', action='store_true', help='Run as a service.')
280 + parser.add_argument('-t', '--test', action='store_true')
281 + parser.add_argument('--write-csv', action='store_true', help='Write the CSVs.')
282 + args = parser.parse_args()
283 + _IS_SERVICE = args.service
284 + _IS_TEST = args.test
285 + _WRITE_CSVS = args.write_csv
286 + main(args)
Newer Older

Powered by Opengist ⋅ Load: 53ms⋅

English