branch: master
scorecard.py
41495 bytesRaw
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#***************************************************************************
# _ _ ____ _
# Project ___| | | | _ \| |
# / __| | | | |_) | |
# | (__| |_| | _ <| |___
# \___|\___/|_| \_\_____|
#
# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at https://curl.se/docs/copyright.html.
#
# You may opt to use, copy, modify, merge, publish, distribute and/or sell
# copies of the Software, and permit persons to whom the Software is
# furnished to do so, under the terms of the COPYING file.
#
# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
# KIND, either express or implied.
#
# SPDX-License-Identifier: curl
#
###########################################################################
#
import argparse
import datetime
import json
import logging
import os
import re
import sys
from statistics import mean
from typing import Dict, Any, Optional, List
from testenv import Env, Httpd, CurlClient, Caddy, ExecResult, NghttpxQuic, RunProfile, Dante
log = logging.getLogger(__name__)
class ScoreCardError(Exception):
pass
class Card:
@classmethod
def fmt_ms(cls, tval):
return f'{int(tval*1000)} ms' if tval >= 0 else '--'
@classmethod
def fmt_size(cls, val):
if val >= (1024*1024*1024):
return f'{val / (1024*1024*1024):0.000f}GB'
elif val >= (1024 * 1024):
return f'{val / (1024*1024):0.000f}MB'
elif val >= 1024:
return f'{val / 1024:0.000f}KB'
else:
return f'{val:0.000f}B'
@classmethod
def fmt_mbs(cls, val):
if val is None or val < 0:
return '--'
if val >= (1024*1024):
return f'{val/(1024*1024):.3g} MB/s'
elif val >= 1024:
return f'{val / 1024:.3g} KB/s'
else:
return f'{val:.3g} B/s'
@classmethod
def fmt_speed(cls, val):
if val is None or val < 0:
return '--'
if val >= (10*1024*1024):
return f'{(val/(1024*1024)):.3f} MB/s'
elif val >= (10*1024):
return f'{val/1024:.3f} KB/s'
else:
return f'{val:.3f} B/s'
@classmethod
def fmt_speed_result(cls, val, limit):
if val is None or val < 0:
return '--'
pct = ((val / limit) * 100) - 100
if val >= (10*1024*1024):
return f'{(val/(1024*1024)):.3f} MB/s, {pct:+.1f}%'
elif val >= (10*1024):
return f'{val/1024:.3f} KB/s, {pct:+.1f}%'
else:
return f'{val:.3f} B/s, {pct:+.1f}%'
@classmethod
def fmt_reqs(cls, val):
return f'{val:0.000f} r/s' if val >= 0 else '--'
@classmethod
def mk_mbs_cell(cls, samples, profiles, errors):
val = mean(samples) if len(samples) else -1
cell = {
'val': val,
'sval': Card.fmt_mbs(val) if val >= 0 else '--',
}
if len(profiles):
cell['stats'] = RunProfile.AverageStats(profiles)
if len(errors):
cell['errors'] = errors
return cell
@classmethod
def mk_speed_cell(cls, samples, profiles, errors, limit):
val = mean(samples) if len(samples) else -1
cell = {
'val': val,
'sval': Card.fmt_speed_result(val, limit) if val >= 0 else '--',
}
if len(profiles):
cell['stats'] = RunProfile.AverageStats(profiles)
if len(errors):
cell['errors'] = errors
return cell
@classmethod
def mk_reqs_cell(cls, samples, profiles, errors):
val = mean(samples) if len(samples) else -1
cell = {
'val': val,
'sval': Card.fmt_reqs(val) if val >= 0 else '--',
}
if len(profiles):
cell['stats'] = RunProfile.AverageStats(profiles)
if len(errors):
cell['errors'] = errors
return cell
@classmethod
def parse_size(cls, s):
m = re.match(r'(\d+)(mb|kb|gb)?', s, re.IGNORECASE)
if m is None:
raise Exception(f'unrecognized size: {s}')
size = int(m.group(1))
if not m.group(2):
pass
elif m.group(2).lower() == 'kb':
size *= 1024
elif m.group(2).lower() == 'mb':
size *= 1024 * 1024
elif m.group(2).lower() == 'gb':
size *= 1024 * 1024 * 1024
return size
@classmethod
def print_score(cls, score):
print(f'Scorecard curl, protocol {score["meta"]["protocol"]} '
f'via {score["meta"]["implementation"]}/'
f'{score["meta"]["implementation_version"]}')
print(f'Date: {score["meta"]["date"]}')
if 'curl_V' in score["meta"]:
print(f'Version: {score["meta"]["curl_V"]}')
if 'curl_features' in score["meta"]:
print(f'Features: {score["meta"]["curl_features"]}')
if 'limit-rate' in score['meta']:
print(f'--limit-rate: {score["meta"]["limit-rate"]}')
print(f'Samples Size: {score["meta"]["samples"]}')
if 'handshakes' in score:
print(f'{"Handshakes":<24} {"ipv4":25} {"ipv6":28}')
print(f' {"Host":<17} {"Connect":>12} {"Handshake":>12} '
f'{"Connect":>12} {"Handshake":>12} {"Errors":<20}')
for key, val in score["handshakes"].items():
print(f' {key:<17} {Card.fmt_ms(val["ipv4-connect"]):>12} '
f'{Card.fmt_ms(val["ipv4-handshake"]):>12} '
f'{Card.fmt_ms(val["ipv6-connect"]):>12} '
f'{Card.fmt_ms(val["ipv6-handshake"]):>12} '
f'{"/".join(val["ipv4-errors"] + val["ipv6-errors"]):<20}'
)
for name in ['downloads', 'uploads', 'requests']:
if name in score:
Card.print_score_table(score[name])
@classmethod
def print_score_table(cls, score):
cols = score['cols']
rows = score['rows']
colw = []
statw = 13
errors = []
col_has_stats = []
for idx, col in enumerate(cols):
cellw = max([len(r[idx]["sval"]) for r in rows])
colw.append(max(cellw, len(col)))
col_has_stats.append(False)
for row in rows:
if 'stats' in row[idx]:
col_has_stats[idx] = True
break
if 'title' in score['meta']:
print(score['meta']['title'])
for idx, col in enumerate(cols):
if col_has_stats[idx]:
print(f' {col:>{colw[idx]}} {"[cpu/rss]":<{statw}}', end='')
else:
print(f' {col:>{colw[idx]}}', end='')
print('')
for row in rows:
for idx, cell in enumerate(row):
print(f' {cell["sval"]:>{colw[idx]}}', end='')
if col_has_stats[idx]:
if 'stats' in cell:
s = f'[{cell["stats"]["cpu"]:>.1f}%' \
f'/{Card.fmt_size(cell["stats"]["rss"])}]'
else:
s = ''
print(f' {s:<{statw}}', end='')
if 'errors' in cell:
errors.extend(cell['errors'])
print('')
if len(errors):
print(f'Errors: {errors}')
class ScoreRunner:
def __init__(self, env: Env,
protocol: str,
server_descr: str,
server_port: int,
verbose: int,
curl_verbose: int,
download_parallel: int = 0,
upload_parallel: int = 0,
server_addr: Optional[str] = None,
with_flame: bool = False,
socks_args: Optional[List[str]] = None,
limit_rate: Optional[str] = None,
suppress_cl: bool = False):
self.verbose = verbose
self.env = env
self.protocol = protocol
self.server_descr = server_descr
self.server_addr = server_addr
self.server_port = server_port
self._silent_curl = not curl_verbose
self._download_parallel = download_parallel
self._upload_parallel = upload_parallel
self._with_flame = with_flame
self._socks_args = socks_args
self._limit_rate_num = 0
self._limit_rate = limit_rate
if self._limit_rate:
m = re.match(r'(\d+(\.\d+)?)([gmkb])?', self._limit_rate.lower())
if not m:
raise Exception(f'unrecognised limit-rate: {self._limit_rate}')
self._limit_rate_num = float(m.group(1))
if m.group(3) == 'g':
self._limit_rate_num *= (1024*1024*1024)
elif m.group(3) == 'm':
self._limit_rate_num *= (1024*1024)
elif m.group(3) == 'k':
self._limit_rate_num *= (1024)
elif m.group(3) == 'b':
pass
else:
raise Exception(f'unrecognised limit-rate: {self._limit_rate}')
self.suppress_cl = suppress_cl
def info(self, msg):
if self.verbose > 0:
sys.stderr.write(msg)
sys.stderr.flush()
def mk_curl_client(self):
return CurlClient(env=self.env, silent=self._silent_curl,
server_addr=self.server_addr,
with_flame=self._with_flame,
socks_args=self._socks_args)
def handshakes(self) -> Dict[str, Any]:
props = {}
sample_size = 5
self.info('TLS Handshake\n')
for authority in [
'curl.se', 'google.com', 'cloudflare.com', 'nghttp2.org'
]:
self.info(f' {authority}...')
props[authority] = {}
for ipv in ['ipv4', 'ipv6']:
self.info(f'{ipv}...')
c_samples = []
hs_samples = []
errors = []
for _ in range(sample_size):
curl = self.mk_curl_client()
args = [
'--http3-only' if self.protocol == 'h3' else '--http2',
f'--{ipv}', f'https://{authority}/'
]
r = curl.run_direct(args=args, with_stats=True)
if r.exit_code == 0 and len(r.stats) == 1:
c_samples.append(r.stats[0]['time_connect'])
hs_samples.append(r.stats[0]['time_appconnect'])
else:
errors.append(f'exit={r.exit_code}')
props[authority][f'{ipv}-connect'] = mean(c_samples) \
if len(c_samples) else -1
props[authority][f'{ipv}-handshake'] = mean(hs_samples) \
if len(hs_samples) else -1
props[authority][f'{ipv}-errors'] = errors
self.info('ok.\n')
return props
def _make_docs_file(self, docs_dir: str, fname: str, fsize: int):
fpath = os.path.join(docs_dir, fname)
data1k = 1024*'x'
flen = 0
with open(fpath, 'w') as fd:
while flen < fsize:
fd.write(data1k)
flen += len(data1k)
return fpath
def setup_resources(self, server_docs: str,
downloads: Optional[List[int]] = None):
if downloads is not None:
for fsize in downloads:
label = Card.fmt_size(fsize)
fname = f'score{label}.data'
self._make_docs_file(docs_dir=server_docs,
fname=fname, fsize=fsize)
self._make_docs_file(docs_dir=server_docs,
fname='reqs10.data', fsize=10*1024)
def _check_downloads(self, r: ExecResult, count: int):
error = ''
if r.exit_code != 0:
error += f'exit={r.exit_code} '
if r.exit_code != 0 or len(r.stats) != count:
error += f'stats={len(r.stats)}/{count} '
fails = [s for s in r.stats if s['response_code'] != 200]
if len(fails) > 0:
error += f'{len(fails)} failed'
return error if len(error) > 0 else None
def dl_single(self, url: str, nsamples: int = 1):
count = 1
samples = []
errors = []
profiles = []
self.info('single...')
for _ in range(nsamples):
curl = self.mk_curl_client()
r = curl.http_download(urls=[url], alpn_proto=self.protocol,
no_save=True, with_headers=False,
with_profile=True,
limit_rate=self._limit_rate)
err = self._check_downloads(r, count)
if err:
errors.append(err)
elif self._limit_rate:
total_speed = sum([s['speed_download'] for s in r.stats])
samples.append(total_speed / len(r.stats))
profiles.append(r.profile)
else:
total_size = sum([s['size_download'] for s in r.stats])
samples.append(total_size / r.duration.total_seconds())
profiles.append(r.profile)
if self._limit_rate:
return Card.mk_speed_cell(samples, profiles, errors, self._limit_rate_num)
else:
return Card.mk_mbs_cell(samples, profiles, errors)
def dl_serial(self, url: str, count: int, nsamples: int = 1):
samples = []
errors = []
profiles = []
url = f'{url}?[0-{count - 1}]'
self.info('serial...')
for _ in range(nsamples):
curl = self.mk_curl_client()
r = curl.http_download(urls=[url], alpn_proto=self.protocol,
no_save=True,
with_headers=False,
with_profile=True,
limit_rate=self._limit_rate)
err = self._check_downloads(r, count)
if err:
errors.append(err)
elif self._limit_rate:
total_speed = sum([s['speed_download'] for s in r.stats])
samples.append(total_speed / len(r.stats))
profiles.append(r.profile)
else:
total_size = sum([s['size_download'] for s in r.stats])
samples.append(total_size / r.duration.total_seconds())
profiles.append(r.profile)
if self._limit_rate:
return Card.mk_speed_cell(samples, profiles, errors, self._limit_rate_num)
else:
return Card.mk_mbs_cell(samples, profiles, errors)
def dl_parallel(self, url: str, count: int, nsamples: int = 1):
samples = []
errors = []
profiles = []
max_parallel = self._download_parallel if self._download_parallel > 0 else count
url = f'{url}?[0-{count - 1}]'
self.info('parallel...')
for _ in range(nsamples):
curl = self.mk_curl_client()
r = curl.http_download(urls=[url], alpn_proto=self.protocol,
no_save=True,
with_headers=False,
with_profile=True,
limit_rate=self._limit_rate,
extra_args=[
'--parallel',
'--parallel-max', str(max_parallel)
])
err = self._check_downloads(r, count)
if err:
errors.append(err)
elif self._limit_rate:
total_speed = sum([s['speed_download'] for s in r.stats])
samples.append(total_speed / len(r.stats))
profiles.append(r.profile)
else:
total_size = sum([s['size_download'] for s in r.stats])
samples.append(total_size / r.duration.total_seconds())
profiles.append(r.profile)
if self._limit_rate:
return Card.mk_speed_cell(samples, profiles, errors, self._limit_rate_num)
else:
return Card.mk_mbs_cell(samples, profiles, errors)
def downloads(self, count: int, fsizes: List[int], meta: Dict[str, Any]) -> Dict[str, Any]:
nsamples = meta['samples']
max_parallel = self._download_parallel if self._download_parallel > 0 else count
cols = ['size']
if not self._download_parallel:
cols.append('single')
if count > 1:
cols.append(f'serial({count})')
if count > 1:
if max_parallel == 1:
cols.append(f'serial({count})')
else:
cols.append(f'parallel({count}x{max_parallel})')
rows = []
for fsize in fsizes:
row = [{
'val': fsize,
'sval': Card.fmt_size(fsize)
}]
self.info(f'{row[0]["sval"]} downloads...')
url = f'https://{self.env.domain1}:{self.server_port}/score{row[0]["sval"]}.data'
if 'single' in cols:
row.append(self.dl_single(url=url, nsamples=nsamples))
if count > 1:
if 'single' in cols:
row.append(self.dl_serial(url=url, count=count, nsamples=nsamples))
row.append(self.dl_parallel(url=url, count=count, nsamples=nsamples))
rows.append(row)
self.info('done.\n')
if self._limit_rate:
title = f'Download Speed ({self.protocol}), limit={Card.fmt_speed(self._limit_rate_num)}, from {meta["server"]}'
else:
title = f'Downloads ({self.protocol})from {meta["server"]}'
if self._socks_args:
title += f' via {self._socks_args}'
return {
'meta': {
'title': title,
'count': count,
'max-parallel': max_parallel,
},
'cols': cols,
'rows': rows,
}
def _check_uploads(self, r: ExecResult, count: int):
error = ''
if r.exit_code != 0:
error += f'exit={r.exit_code} '
if r.exit_code != 0 or len(r.stats) != count:
error += f'stats={len(r.stats)}/{count} '
fails = [s for s in r.stats if s['response_code'] != 200]
if len(fails) > 0:
error += f'{len(fails)} failed'
for f in fails:
error += f'[{f["response_code"]}]'
return error if len(error) > 0 else None
def ul_single(self, url: str, fpath: str, nsamples: int = 1):
samples = []
errors = []
profiles = []
self.info('single...')
for _ in range(nsamples):
curl = self.mk_curl_client()
r = curl.http_put(urls=[url], fdata=fpath, alpn_proto=self.protocol,
with_headers=False, with_profile=True,
suppress_cl=self.suppress_cl)
err = self._check_uploads(r, 1)
if err:
errors.append(err)
else:
total_size = sum([s['size_upload'] for s in r.stats])
samples.append(total_size / r.duration.total_seconds())
profiles.append(r.profile)
return Card.mk_mbs_cell(samples, profiles, errors)
def ul_serial(self, url: str, fpath: str, count: int, nsamples: int = 1):
samples = []
errors = []
profiles = []
url = f'{url}?id=[0-{count - 1}]'
self.info('serial...')
for _ in range(nsamples):
curl = self.mk_curl_client()
r = curl.http_put(urls=[url], fdata=fpath, alpn_proto=self.protocol,
with_headers=False, with_profile=True,
suppress_cl=self.suppress_cl)
err = self._check_uploads(r, count)
if err:
errors.append(err)
else:
total_size = sum([s['size_upload'] for s in r.stats])
samples.append(total_size / r.duration.total_seconds())
profiles.append(r.profile)
return Card.mk_mbs_cell(samples, profiles, errors)
def ul_parallel(self, url: str, fpath: str, count: int, nsamples: int = 1):
samples = []
errors = []
profiles = []
max_parallel = self._download_parallel if self._download_parallel > 0 else count
url = f'{url}?id=[0-{count - 1}]'
self.info('parallel...')
for _ in range(nsamples):
curl = self.mk_curl_client()
r = curl.http_put(urls=[url], fdata=fpath, alpn_proto=self.protocol,
with_headers=False, with_profile=True,
suppress_cl=self.suppress_cl,
extra_args=[
'--parallel',
'--parallel-max', str(max_parallel),
])
err = self._check_uploads(r, count)
if err:
errors.append(err)
else:
total_size = sum([s['size_upload'] for s in r.stats])
samples.append(total_size / r.duration.total_seconds())
profiles.append(r.profile)
return Card.mk_mbs_cell(samples, profiles, errors)
def uploads(self, count: int, fsizes: List[int], meta: Dict[str, Any]) -> Dict[str, Any]:
nsamples = meta['samples']
max_parallel = self._upload_parallel if self._upload_parallel > 0 else count
cols = ['size']
run_single = not self._upload_parallel
run_serial = not self._upload_parallel and count > 1
run_parallel = self._upload_parallel or count > 1
if run_single:
cols.append('single')
if run_serial:
cols.append(f'serial({count})')
if run_parallel:
cols.append(f'parallel({count}x{max_parallel})')
rows = []
for fsize in fsizes:
row = [{
'val': fsize,
'sval': Card.fmt_size(fsize)
}]
self.info(f'{row[0]["sval"]} uploads...')
url = f'https://{self.env.domain1}:{self.server_port}/curltest/put'
fname = f'upload{row[0]["sval"]}.data'
fpath = self._make_docs_file(docs_dir=self.env.gen_dir,
fname=fname, fsize=fsize)
if run_single:
row.append(self.ul_single(url=url, fpath=fpath, nsamples=nsamples))
if run_serial:
row.append(self.ul_serial(url=url, fpath=fpath, count=count, nsamples=nsamples))
if run_parallel:
row.append(self.ul_parallel(url=url, fpath=fpath, count=count, nsamples=nsamples))
rows.append(row)
self.info('done.\n')
title = f'Uploads to {meta["server"]}'
if self._socks_args:
title += f' via {self._socks_args}'
return {
'meta': {
'title': title,
'count': count,
'max-parallel': max_parallel,
},
'cols': cols,
'rows': rows,
}
def do_requests(self, url: str, count: int, max_parallel: int = 1, nsamples: int = 1):
samples = []
errors = []
profiles = []
url = f'{url}?[0-{count - 1}]'
extra_args = [
'-w', '%{response_code},\\n',
]
if max_parallel > 1:
extra_args.extend([
'--parallel', '--parallel-max', str(max_parallel)
])
self.info(f'{max_parallel}...')
for _ in range(nsamples):
curl = self.mk_curl_client()
r = curl.http_download(urls=[url], alpn_proto=self.protocol, no_save=True,
with_headers=False, with_profile=True,
with_stats=False, extra_args=extra_args)
if r.exit_code != 0:
errors.append(f'exit={r.exit_code}')
else:
samples.append(count / r.duration.total_seconds())
non_200s = 0
for line in r.stdout.splitlines():
if not line.startswith('200,'):
non_200s += 1
if non_200s > 0:
errors.append(f'responses != 200: {non_200s}')
profiles.append(r.profile)
return Card.mk_reqs_cell(samples, profiles, errors)
def requests(self, count: int, meta: Dict[str, Any]) -> Dict[str, Any]:
url = f'https://{self.env.domain1}:{self.server_port}/reqs10.data'
fsize = 10*1024
cols = ['size', 'total']
rows = []
mparallel = meta['request_parallels']
cols.extend([f'{mp} max' for mp in mparallel])
row = [{
'val': fsize,
'sval': Card.fmt_size(fsize)
},{
'val': count,
'sval': f'{count}',
}]
self.info('requests, max parallel...')
row.extend([self.do_requests(url=url, count=count,
max_parallel=mp, nsamples=meta["samples"])
for mp in mparallel])
rows.append(row)
self.info('done.\n')
title = f'Requests in parallel to {meta["server"]}'
if self._socks_args:
title += f' via {self._socks_args}'
return {
'meta': {
'title': title,
'count': count,
},
'cols': cols,
'rows': rows,
}
def score(self,
handshakes: bool = True,
downloads: Optional[List[int]] = None,
download_count: int = 50,
uploads: Optional[List[int]] = None,
upload_count: int = 50,
req_count=5000,
request_parallels=None,
nsamples: int = 1,
requests: bool = True):
self.info(f"scoring {self.protocol} against {self.server_descr}\n")
score = {
'meta': {
'curl_version': self.env.curl_version(),
'curl_V': self.env.curl_fullname(),
'curl_features': self.env.curl_features_string(),
'os': self.env.curl_os(),
'server': self.server_descr,
'samples': nsamples,
'date': f'{datetime.datetime.now(tz=datetime.timezone.utc).isoformat()}',
}
}
if self._limit_rate:
score['meta']['limit-rate'] = self._limit_rate
if self.protocol == 'h3':
score['meta']['protocol'] = 'h3'
if not self.env.have_h3_curl():
raise ScoreCardError('curl does not support HTTP/3')
for lib in ['ngtcp2', 'quiche', 'nghttp3']:
if self.env.curl_uses_lib(lib):
score['meta']['implementation'] = lib
break
elif self.protocol == 'h2':
score['meta']['protocol'] = 'h2'
if not self.env.have_h2_curl():
raise ScoreCardError('curl does not support HTTP/2')
for lib in ['nghttp2']:
if self.env.curl_uses_lib(lib):
score['meta']['implementation'] = lib
break
elif self.protocol == 'h1' or self.protocol == 'http/1.1':
score['meta']['protocol'] = 'http/1.1'
score['meta']['implementation'] = 'native'
else:
raise ScoreCardError(f"unknown protocol: {self.protocol}")
if 'implementation' not in score['meta']:
raise ScoreCardError('did not recognized protocol lib')
score['meta']['implementation_version'] = Env.curl_lib_version(score['meta']['implementation'])
if handshakes:
score['handshakes'] = self.handshakes()
if downloads and len(downloads) > 0:
score['downloads'] = self.downloads(count=download_count,
fsizes=downloads,
meta=score['meta'])
if uploads and len(uploads) > 0:
score['uploads'] = self.uploads(count=upload_count,
fsizes=uploads,
meta=score['meta'])
if requests:
if request_parallels is None:
request_parallels = [1, 6, 25, 50, 100, 300]
score['meta']['request_parallels'] = request_parallels
score['requests'] = self.requests(count=req_count, meta=score['meta'])
return score
def run_score(args, protocol):
if protocol not in ['http/1.1', 'h1', 'h2', 'h3']:
sys.stderr.write(f'ERROR: protocol "{protocol}" not known to scorecard\n')
sys.exit(1)
if protocol == 'h1':
protocol = 'http/1.1'
handshakes = True
downloads = [1024 * 1024, 10 * 1024 * 1024, 100 * 1024 * 1024]
if args.download_sizes is not None:
downloads = []
for x in args.download_sizes:
downloads.extend([Card.parse_size(s) for s in x.split(',')])
uploads = [1024 * 1024, 10 * 1024 * 1024, 100 * 1024 * 1024]
if args.upload_sizes is not None:
uploads = []
for x in args.upload_sizes:
uploads.extend([Card.parse_size(s) for s in x.split(',')])
requests = True
request_parallels = None
if args.request_parallels:
request_parallels = []
for x in args.request_parallels:
request_parallels.extend([int(s) for s in x.split(',')])
if args.downloads or args.uploads or args.requests or args.handshakes:
handshakes = args.handshakes
if not args.downloads:
downloads = None
if not args.uploads:
uploads = None
requests = args.requests
test_httpd = protocol != 'h3'
test_caddy = protocol == 'h3'
if args.caddy or args.httpd:
test_caddy = args.caddy
test_httpd = args.httpd
rv = 0
env = Env()
env.setup()
env.test_timeout = None
sockd = None
socks_args = None
if args.socks4 and args.socks5:
raise ScoreCardError('unable to run --socks4 and --socks5 together')
elif args.socks4 or args.socks5:
sockd = Dante(env=env)
if sockd:
assert sockd.initial_start()
socks_args = [
'--socks4' if args.socks4 else '--socks5',
f'127.0.0.1:{sockd.port}',
]
httpd = None
nghttpx = None
caddy = None
try:
cards = []
if args.remote:
m = re.match(r'^(.+):(\d+)$', args.remote)
if m is None:
raise ScoreCardError(f'unable to parse ip:port from --remote {args.remote}')
test_httpd = False
test_caddy = False
remote_addr = m.group(1)
remote_port = int(m.group(2))
card = ScoreRunner(env=env,
protocol=protocol,
server_descr=f'Server at {args.remote}',
server_addr=remote_addr,
server_port=remote_port,
verbose=args.verbose,
curl_verbose=args.curl_verbose,
download_parallel=args.download_parallel,
upload_parallel=args.upload_parallel,
with_flame=args.flame,
socks_args=socks_args,
limit_rate=args.limit_rate,
suppress_cl=args.upload_no_cl)
cards.append(card)
if test_httpd:
httpd = Httpd(env=env)
assert httpd.exists(), \
f'httpd not found: {env.httpd}'
httpd.clear_logs()
server_docs = httpd.docs_dir
assert httpd.initial_start()
if protocol == 'h3':
nghttpx = NghttpxQuic(env=env)
nghttpx.clear_logs()
assert nghttpx.initial_start()
server_descr = f'nghttpx: https:{env.h3_port} [backend httpd/{env.httpd_version()}]'
server_port = env.h3_port
else:
server_descr = f'httpd/{env.httpd_version()}'
server_port = env.https_port
card = ScoreRunner(env=env,
protocol=protocol,
server_descr=server_descr,
server_port=server_port,
verbose=args.verbose, curl_verbose=args.curl_verbose,
download_parallel=args.download_parallel,
upload_parallel=args.upload_parallel,
with_flame=args.flame,
socks_args=socks_args,
limit_rate=args.limit_rate)
card.setup_resources(server_docs, downloads)
cards.append(card)
if test_caddy and env.caddy:
backend = ''
if uploads and httpd is None:
backend = f' [backend httpd: {env.httpd_version()}]'
httpd = Httpd(env=env)
assert httpd.exists(), \
f'httpd not found: {env.httpd}'
httpd.clear_logs()
assert httpd.initial_start()
caddy = Caddy(env=env)
caddy.clear_logs()
assert caddy.initial_start()
server_descr = f'Caddy/{env.caddy_version()} {backend}'
server_port = caddy.port
server_docs = caddy.docs_dir
card = ScoreRunner(env=env,
protocol=protocol,
server_descr=server_descr,
server_port=server_port,
verbose=args.verbose, curl_verbose=args.curl_verbose,
download_parallel=args.download_parallel,
upload_parallel=args.upload_parallel,
with_flame=args.flame,
socks_args=socks_args,
limit_rate=args.limit_rate)
card.setup_resources(server_docs, downloads)
cards.append(card)
if args.start_only:
print('started servers:')
for card in cards:
print(f'{card.server_descr}')
sys.stderr.write('press [RETURN] to finish')
sys.stderr.flush()
sys.stdin.readline()
else:
for card in cards:
score = card.score(handshakes=handshakes,
downloads=downloads,
download_count=args.download_count,
uploads=uploads,
upload_count=args.upload_count,
req_count=args.request_count,
requests=requests,
request_parallels=request_parallels,
nsamples=args.samples)
if args.json:
print(json.JSONEncoder(indent=2).encode(score))
else:
Card.print_score(score)
except ScoreCardError as ex:
sys.stderr.write(f"ERROR: {ex}\n")
rv = 1
except KeyboardInterrupt:
log.warning("aborted")
rv = 1
finally:
if caddy:
caddy.stop()
if nghttpx:
nghttpx.stop(wait_dead=False)
if httpd:
httpd.stop()
if sockd:
sockd.stop()
return rv
def print_file(filename):
if not os.path.exists(filename):
sys.stderr.write(f"ERROR: file does not exist {filename}\n")
return 1
with open(filename) as file:
data = json.load(file)
Card.print_score(data)
return 0
def main():
parser = argparse.ArgumentParser(prog='scorecard', description="""
Run a range of tests to give a scorecard for an HTTP protocol
'h3' or 'h2' implementation in curl.
""")
parser.add_argument("-v", "--verbose", action='count', default=1,
help="log more output on stderr")
parser.add_argument("-j", "--json", action='store_true',
default=False, help="print json instead of text")
parser.add_argument("--samples", action='store', type=int, metavar='number',
default=1, help="how many sample runs to make")
parser.add_argument("--httpd", action='store_true', default=False,
help="evaluate httpd server only")
parser.add_argument("--caddy", action='store_true', default=False,
help="evaluate caddy server only")
parser.add_argument("--curl-verbose", action='store_true',
default=False, help="run curl with `-v`")
parser.add_argument("--print", type=str, default=None, metavar='filename',
help="print the results from a JSON file")
parser.add_argument("protocol", default=None, nargs='?',
help="Name of protocol to score")
parser.add_argument("--start-only", action='store_true', default=False,
help="only start the servers")
parser.add_argument("--remote", action='store', type=str,
default=None, help="score against the remote server at <ip>:<port>")
parser.add_argument("--flame", action='store_true',
default = False, help="produce a flame graph on curl")
parser.add_argument("--limit-rate", action='store', type=str,
default=None, help="use curl's --limit-rate")
parser.add_argument("-H", "--handshakes", action='store_true',
default=False, help="evaluate handshakes only")
parser.add_argument("-d", "--downloads", action='store_true',
default=False, help="evaluate downloads")
parser.add_argument("--download-sizes", action='append', type=str,
metavar='numberlist',
default=None, help="evaluate download size")
parser.add_argument("--download-count", action='store', type=int,
metavar='number',
default=50, help="perform that many downloads")
parser.add_argument("--download-parallel", action='store', type=int,
metavar='number', default=0,
help="perform that many downloads in parallel (default all)")
parser.add_argument("-u", "--uploads", action='store_true',
default=False, help="evaluate uploads")
parser.add_argument("--upload-sizes", action='append', type=str,
metavar='numberlist',
default=None, help="evaluate upload size")
parser.add_argument("--upload-count", action='store', type=int,
metavar='number', default=50,
help="perform that many uploads")
parser.add_argument("--upload-parallel", action='store', type=int,
metavar='number', default=0,
help="perform that many uploads in parallel (default all)")
parser.add_argument("--upload-no-cl", action='store_true',
default=False, help="suppress content-length on upload")
parser.add_argument("-r", "--requests", action='store_true',
default=False, help="evaluate requests")
parser.add_argument("--request-count", action='store', type=int,
metavar='number',
default=5000, help="perform that many requests")
parser.add_argument("--request-parallels", action='append', type=str,
metavar='numberlist',
default=None, help="evaluate request with these max-parallel numbers")
parser.add_argument("--socks4", action='store_true',
default=False, help="test with SOCKS4 proxy")
parser.add_argument("--socks5", action='store_true',
default=False, help="test with SOCKS5 proxy")
args = parser.parse_args()
if args.verbose > 0:
console = logging.StreamHandler()
console.setLevel(logging.INFO)
console.setFormatter(logging.Formatter(logging.BASIC_FORMAT))
logging.getLogger('').addHandler(console)
if args.print:
rv = print_file(args.print)
elif not args.protocol:
parser.print_usage()
rv = 1
else:
rv = run_score(args, args.protocol)
sys.exit(rv)
if __name__ == "__main__":
main()