import json import random import threading import time import traceback import redis import requests from flask import Flask, jsonify, request from flask_caching import Cache redis_conn = redis.Redis(host='localhost', port=6379, db=2) redis_conn.set('last_updated', 0) redis_conn.set('fetching_new_data', 0) if not redis_conn.hget('nodes', 'nodes'): redis_conn.hset('nodes', 'nodes', json.dumps({'data': {'nodes': []}})) headers = { 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/114.0', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.5', 'Accept-Encoding': 'gzip;q=0,deflate;q=0', 'DNT': '1', 'Connection': 'keep-alive', 'Upgrade-Insecure-Requests': '1', 'Sec-Fetch-Dest': 'document', 'Sec-Fetch-Mode': 'navigate', 'Sec-Fetch-Site': 'cross-site', 'Pragma': 'no-cache', 'Cache-Control': 'no-cache', 'TE': 'trailers', } def update_node_cache(): # Semaphore if bool(int(redis_conn.get('fetching_new_data').decode())): return redis_conn.set('fetching_new_data', 1) print('Fetching new data...') start = time.time() try: r = requests.get('https://api.chub.ai/search?search=&first=5000000&min_tokens=50&nsfw=true', headers=headers, timeout=120, proxies={'http': 'http://172.0.4.7:9000', 'https': 'http://172.0.4.7:9000'}) j = r.json() j['timestamp'] = int(time.time()) redis_conn.set('node_count', len(j['data']['nodes'])) redis_conn.hset('nodes', 'nodes', json.dumps(j)) except: traceback.print_exc() finally: redis_conn.set('fetching_new_data', 0) print('Finished fetching new data in', time.time() - start) app = Flask(__name__) cache = Cache(app, config={'CACHE_TYPE': 'redis', 'CACHE_REDIS_URL': 'redis://localhost:6379/1', 'CACHE_KEY_PREFIX': 'chub_random__'}) cache.clear() @cache.memoize(timeout=900) def cached_node_json(): return json.loads(redis_conn.hget('nodes', 'nodes').decode()) def get_nodes(): last_updated = int(redis_conn.get('last_updated')) if time.time() > last_updated: threading.Thread(target=update_node_cache).start() count_raw = redis_conn.get('node_count') if not count_raw or not int(count_raw): # Return dummy data if there isn't anything loaded yet. # This avoids caching the empty data. return {'data': {'nodes': []}} else: # Cache and return data. return cached_node_json() @cache.memoize(timeout=900) def get_items_by_topics(included_topics, excluded_topics): count_raw = redis_conn.get('node_count') if not count_raw or not int(count_raw): return [] nodes = get_nodes()['data']['nodes'] result = [] for node in nodes: if set(included_topics).issubset(set(node['topics'])) and not set(excluded_topics).intersection(set(node['topics'])): result.append(node) return result n = get_nodes() redis_conn.set('node_count', len(n['data']['nodes'])) @cache.cached(timeout=900) @app.route('/api/characters', methods=['GET']) def all_characters(): c = get_nodes() if not c: r = jsonify({'error': 'Something went wrong fetching response'}) r.headers['Access-Control-Allow-Origin'] = '*' return r, 500 else: r = jsonify(c) r.headers['Cache-Control'] = 'public, max-age=900' r.headers['Access-Control-Allow-Origin'] = '*' return r @app.route('/api/characters/random', methods=['GET']) def random_character(): c = get_nodes() tags_arg = request.args.get('tags') response_data = {} if not tags_arg: if not c: r = jsonify({'error': 'Something went wrong'}) r.headers['Access-Control-Allow-Origin'] = '*' return r, 500 elif len(c['data']['nodes']): count = int(redis_conn.get('node_count').decode()) i = random.randint(0, count) response_data = c['data']['nodes'][i] response_data['timestamp'] = c['timestamp'] else: tags = tags_arg.split(',') included_tags = [x for x in tags if not x.startswith('-')] excluded_tags = [x.lstrip('-') for x in tags if x.startswith('-')] matches = get_items_by_topics(included_tags, excluded_tags) if len(matches): response_data = random.choice(matches) response_data['timestamp'] = c['timestamp'] else: response_data = [] r = jsonify(response_data) r.headers['Access-Control-Allow-Origin'] = '*' r.headers['Cache-Control'] = 'no-store' return r if __name__ == '__main__': app.run(debug=True, host='0.0.0.0')