httpd: remove memory profiling code
Remove objgraph/pympler imports, gc.get_objects() caching, and memory_samples tracking. Keep basic RSS tracking for dashboard.
This commit is contained in:
117
httpd.py
117
httpd.py
@@ -58,32 +58,14 @@ _LIB_CACHE = {}
|
|||||||
# Cache for dashboard static files (HTML, CSS, JS)
|
# Cache for dashboard static files (HTML, CSS, JS)
|
||||||
_STATIC_CACHE = {}
|
_STATIC_CACHE = {}
|
||||||
|
|
||||||
# Optional memory profiling (installed via requirements.txt)
|
|
||||||
try:
|
|
||||||
import objgraph
|
|
||||||
_has_objgraph = True
|
|
||||||
except ImportError:
|
|
||||||
_has_objgraph = False
|
|
||||||
|
|
||||||
try:
|
|
||||||
from pympler import muppy, summary
|
|
||||||
_has_pympler = True
|
|
||||||
except ImportError:
|
|
||||||
_has_pympler = False
|
|
||||||
|
|
||||||
# Memory tracking for leak detection
|
|
||||||
_memory_samples = []
|
|
||||||
_memory_sample_max = 60 # Keep last 60 samples (5 min at 5s intervals)
|
|
||||||
_peak_rss = 0
|
|
||||||
_start_rss = 0
|
|
||||||
|
|
||||||
# Cache for expensive operations
|
# Cache for expensive operations
|
||||||
_gc_objects_cache = {'value': 0, 'time': 0}
|
|
||||||
_gc_objects_ttl = 30 # seconds
|
|
||||||
|
|
||||||
_db_health_cache = {'value': {}, 'time': 0}
|
_db_health_cache = {'value': {}, 'time': 0}
|
||||||
_db_health_ttl = 10 # seconds
|
_db_health_ttl = 10 # seconds
|
||||||
|
|
||||||
|
# Simple RSS tracking for dashboard
|
||||||
|
_peak_rss = 0
|
||||||
|
_start_rss = 0
|
||||||
|
|
||||||
# Worker registry for distributed testing
|
# Worker registry for distributed testing
|
||||||
import hashlib
|
import hashlib
|
||||||
import random
|
import random
|
||||||
@@ -669,39 +651,26 @@ def get_system_stats():
|
|||||||
stats['proc_rss'] = 0
|
stats['proc_rss'] = 0
|
||||||
stats['proc_threads'] = 0
|
stats['proc_threads'] = 0
|
||||||
|
|
||||||
# Memory leak detection
|
# Simple RSS tracking for dashboard
|
||||||
global _memory_samples, _peak_rss, _start_rss
|
global _peak_rss, _start_rss
|
||||||
rss = stats.get('proc_rss', 0)
|
rss = stats.get('proc_rss', 0)
|
||||||
if rss > 0:
|
if rss > 0:
|
||||||
if _start_rss == 0:
|
if _start_rss == 0:
|
||||||
_start_rss = rss
|
_start_rss = rss
|
||||||
if rss > _peak_rss:
|
if rss > _peak_rss:
|
||||||
_peak_rss = rss
|
_peak_rss = rss
|
||||||
_memory_samples.append((time.time(), rss))
|
|
||||||
if len(_memory_samples) > _memory_sample_max:
|
|
||||||
_memory_samples.pop(0)
|
|
||||||
|
|
||||||
stats['proc_rss_peak'] = _peak_rss
|
stats['proc_rss_peak'] = _peak_rss
|
||||||
stats['proc_rss_start'] = _start_rss
|
stats['proc_rss_start'] = _start_rss
|
||||||
stats['proc_rss_growth'] = rss - _start_rss if _start_rss > 0 else 0
|
stats['proc_rss_growth'] = rss - _start_rss if _start_rss > 0 else 0
|
||||||
|
|
||||||
# GC stats for leak detection
|
# GC generation counts (fast - no object iteration)
|
||||||
try:
|
try:
|
||||||
gc_counts = gc.get_count()
|
gc_counts = gc.get_count()
|
||||||
stats['gc_count_gen0'] = gc_counts[0]
|
stats['gc_count_gen0'] = gc_counts[0]
|
||||||
stats['gc_count_gen1'] = gc_counts[1]
|
stats['gc_count_gen1'] = gc_counts[1]
|
||||||
stats['gc_count_gen2'] = gc_counts[2]
|
stats['gc_count_gen2'] = gc_counts[2]
|
||||||
# Cache gc.get_objects() - expensive call (~23ms)
|
except Exception:
|
||||||
global _gc_objects_cache
|
|
||||||
now = time.time()
|
|
||||||
if now - _gc_objects_cache['time'] > _gc_objects_ttl:
|
|
||||||
_gc_objects_cache['value'] = len(gc.get_objects())
|
|
||||||
_gc_objects_cache['time'] = now
|
|
||||||
stats['gc_objects'] = _gc_objects_cache['value']
|
|
||||||
except Exception as e:
|
|
||||||
_log('gc stats error: %s' % e, 'debug')
|
|
||||||
stats['gc_count_gen0'] = stats['gc_count_gen1'] = stats['gc_count_gen2'] = 0
|
stats['gc_count_gen0'] = stats['gc_count_gen1'] = stats['gc_count_gen2'] = 0
|
||||||
stats['gc_objects'] = 0
|
|
||||||
|
|
||||||
return stats
|
return stats
|
||||||
|
|
||||||
@@ -1438,76 +1407,6 @@ class ProxyAPIServer(threading.Thread):
|
|||||||
return json.dumps({'count': row[0] if row else 0}), 'application/json', 200
|
return json.dumps({'count': row[0] if row else 0}), 'application/json', 200
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
return json.dumps({'error': str(e)}), 'application/json', 500
|
return json.dumps({'error': str(e)}), 'application/json', 500
|
||||||
elif path == '/api/memory':
|
|
||||||
# Memory profiling endpoint (localhost only)
|
|
||||||
if not is_localhost(remote_addr):
|
|
||||||
return json.dumps({'error': 'not available'}), 'application/json', 404
|
|
||||||
try:
|
|
||||||
mem = {}
|
|
||||||
|
|
||||||
# Process memory from /proc/self/status
|
|
||||||
try:
|
|
||||||
with open('/proc/self/status', 'r') as f:
|
|
||||||
for line in f:
|
|
||||||
if line.startswith('Vm'):
|
|
||||||
parts = line.split()
|
|
||||||
key = parts[0].rstrip(':')
|
|
||||||
mem[key] = int(parts[1]) * 1024 # Convert to bytes
|
|
||||||
except IOError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
# GC stats
|
|
||||||
gc_stats = {
|
|
||||||
'collections': gc.get_count(),
|
|
||||||
'threshold': gc.get_threshold(),
|
|
||||||
'objects': len(gc.get_objects()),
|
|
||||||
}
|
|
||||||
|
|
||||||
# Object type counts (top 20)
|
|
||||||
type_counts = {}
|
|
||||||
for obj in gc.get_objects():
|
|
||||||
t = type(obj).__name__
|
|
||||||
type_counts[t] = type_counts.get(t, 0) + 1
|
|
||||||
top_types = sorted(type_counts.items(), key=lambda x: -x[1])[:20]
|
|
||||||
|
|
||||||
# Memory samples history
|
|
||||||
samples = []
|
|
||||||
for ts, rss in _memory_samples[-30:]:
|
|
||||||
samples.append({'time': int(ts), 'rss': rss})
|
|
||||||
|
|
||||||
result = {
|
|
||||||
'process': mem,
|
|
||||||
'gc': gc_stats,
|
|
||||||
'top_types': [{'type': t, 'count': c} for t, c in top_types],
|
|
||||||
'samples': samples,
|
|
||||||
'peak_rss': _peak_rss,
|
|
||||||
'start_rss': _start_rss,
|
|
||||||
'has_objgraph': _has_objgraph,
|
|
||||||
'has_pympler': _has_pympler,
|
|
||||||
}
|
|
||||||
|
|
||||||
# Objgraph most common types (if available)
|
|
||||||
if _has_objgraph:
|
|
||||||
try:
|
|
||||||
result['objgraph_common'] = objgraph.most_common_types(limit=15)
|
|
||||||
except Exception as e:
|
|
||||||
result['objgraph_error'] = str(e)
|
|
||||||
|
|
||||||
# Pympler summary (if available)
|
|
||||||
if _has_pympler:
|
|
||||||
try:
|
|
||||||
all_objects = muppy.get_objects()
|
|
||||||
sum_table = summary.summarize(all_objects)
|
|
||||||
result['pympler_summary'] = [
|
|
||||||
{'type': row[0], 'count': row[1], 'size': row[2]}
|
|
||||||
for row in sum_table[:20]
|
|
||||||
]
|
|
||||||
except Exception as e:
|
|
||||||
result['pympler_error'] = str(e)
|
|
||||||
|
|
||||||
return json.dumps(result, indent=2), 'application/json', 200
|
|
||||||
except Exception as e:
|
|
||||||
return json.dumps({'error': str(e)}), 'application/json', 500
|
|
||||||
elif path == '/health':
|
elif path == '/health':
|
||||||
return json.dumps({'status': 'ok', 'version': __version__, 'timestamp': int(time.time())}), 'application/json', 200
|
return json.dumps({'status': 'ok', 'version': __version__, 'timestamp': int(time.time())}), 'application/json', 200
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user