PATCH: remove nclients/nthreads constraint from pgbench
Remove pgbench constraint that the number of clients must be a multiple of
the number of threads, by sharing clients among threads as evenly as
possible.
Rational: allows to test the database load with any number of client
without unrelated constraint. The current setting means for instance that
testing with a prime number of clients always uses one thread, whereas
other number of clients can use a different number of threads. This
constraint does not make much sense.
--
Fabien.
Attachments:
pgbench-nclients-1.patchtext/x-diff; name=pgbench-nclients-1.patchDownload
diff --git a/doc/src/sgml/ref/pgbench.sgml b/doc/src/sgml/ref/pgbench.sgml
index a808546..2517a3a 100644
--- a/doc/src/sgml/ref/pgbench.sgml
+++ b/doc/src/sgml/ref/pgbench.sgml
@@ -326,8 +326,7 @@ pgbench <optional> <replaceable>options</> </optional> <replaceable>dbname</>
<para>
Number of worker threads within <application>pgbench</application>.
Using more than one thread can be helpful on multi-CPU machines.
- The number of clients must be a multiple of the number of threads,
- since each thread is given the same number of client sessions to manage.
+ Clients are distributed as evenly as possible among available threads.
Default is 1.
</para>
</listitem>
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index 06a4dfd..36ff427 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -2762,6 +2762,7 @@ main(int argc, char **argv)
int c;
int nclients = 1; /* default number of simulated clients */
int nthreads = 1; /* default number of threads */
+ int nclients_shared;/* clients shared between threads */
int is_init_mode = 0; /* initialize mode? */
int is_no_vacuum = 0; /* no vacuum at all before testing? */
int do_vacuum_accounts = 0; /* do vacuum accounts before testing? */
@@ -3122,12 +3123,6 @@ main(int argc, char **argv)
if (nxacts <= 0 && duration <= 0)
nxacts = DEFAULT_NXACTS;
- if (nclients % nthreads != 0)
- {
- fprintf(stderr, "number of clients (%d) must be a multiple of number of threads (%d)\n", nclients, nthreads);
- exit(1);
- }
-
/* --sampling-rate may be used only with -l */
if (sample_rate > 0.0 && !use_log)
{
@@ -3328,19 +3323,24 @@ main(int argc, char **argv)
/* set up thread data structures */
threads = (TState *) pg_malloc(sizeof(TState) * nthreads);
+ nclients_shared = 0;
+
for (i = 0; i < nthreads; i++)
{
TState *thread = &threads[i];
thread->tid = i;
- thread->state = &state[nclients / nthreads * i];
- thread->nstate = nclients / nthreads;
+ thread->state = &state[nclients_shared];
+ thread->nstate =
+ (nclients - nclients_shared + nthreads - i - 1) / (nthreads - i);
thread->random_state[0] = random();
thread->random_state[1] = random();
thread->random_state[2] = random();
thread->throttle_latency_skipped = 0;
thread->latency_late = 0;
+ nclients_shared += thread->nstate;
+
if (is_latencies)
{
/* Reserve memory for the thread to store per-command latencies */
@@ -3364,6 +3364,9 @@ main(int argc, char **argv)
}
}
+ /* all clients must be shared between threads */
+ Assert(nclients_shared == nclients);
+
/* get start up time */
INSTR_TIME_SET_CURRENT(start_time);
Minor v2 update to change a not badly chosen variable name.
--
Fabien.
Attachments:
pgbench-nclients-2.patchtext/x-diff; name=pgbench-nclients-2.patchDownload
diff --git a/doc/src/sgml/ref/pgbench.sgml b/doc/src/sgml/ref/pgbench.sgml
index a808546..2517a3a 100644
--- a/doc/src/sgml/ref/pgbench.sgml
+++ b/doc/src/sgml/ref/pgbench.sgml
@@ -326,8 +326,7 @@ pgbench <optional> <replaceable>options</> </optional> <replaceable>dbname</>
<para>
Number of worker threads within <application>pgbench</application>.
Using more than one thread can be helpful on multi-CPU machines.
- The number of clients must be a multiple of the number of threads,
- since each thread is given the same number of client sessions to manage.
+ Clients are distributed as evenly as possible among available threads.
Default is 1.
</para>
</listitem>
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index 06a4dfd..5343837 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -2762,6 +2762,7 @@ main(int argc, char **argv)
int c;
int nclients = 1; /* default number of simulated clients */
int nthreads = 1; /* default number of threads */
+ int nclients_dealt; /* clients distributed between threads */
int is_init_mode = 0; /* initialize mode? */
int is_no_vacuum = 0; /* no vacuum at all before testing? */
int do_vacuum_accounts = 0; /* do vacuum accounts before testing? */
@@ -3122,12 +3123,6 @@ main(int argc, char **argv)
if (nxacts <= 0 && duration <= 0)
nxacts = DEFAULT_NXACTS;
- if (nclients % nthreads != 0)
- {
- fprintf(stderr, "number of clients (%d) must be a multiple of number of threads (%d)\n", nclients, nthreads);
- exit(1);
- }
-
/* --sampling-rate may be used only with -l */
if (sample_rate > 0.0 && !use_log)
{
@@ -3328,19 +3323,24 @@ main(int argc, char **argv)
/* set up thread data structures */
threads = (TState *) pg_malloc(sizeof(TState) * nthreads);
+ nclients_dealt = 0;
+
for (i = 0; i < nthreads; i++)
{
TState *thread = &threads[i];
thread->tid = i;
- thread->state = &state[nclients / nthreads * i];
- thread->nstate = nclients / nthreads;
+ thread->state = &state[nclients_dealt];
+ thread->nstate =
+ (nclients - nclients_dealt + nthreads - i - 1) / (nthreads - i);
thread->random_state[0] = random();
thread->random_state[1] = random();
thread->random_state[2] = random();
thread->throttle_latency_skipped = 0;
thread->latency_late = 0;
+ nclients_dealt += thread->nstate;
+
if (is_latencies)
{
/* Reserve memory for the thread to store per-command latencies */
@@ -3364,6 +3364,9 @@ main(int argc, char **argv)
}
}
+ /* all clients must be assigned to a thread */
+ Assert(nclients_dealt == nclients);
+
/* get start up time */
INSTR_TIME_SET_CURRENT(start_time);
On 05/08/2015 03:02 PM, Fabien COELHO wrote:
Remove pgbench constraint that the number of clients must be a multiple of
the number of threads, by sharing clients among threads as evenly as
possible.Rational: allows to test the database load with any number of client
without unrelated constraint. The current setting means for instance that
testing with a prime number of clients always uses one thread, whereas
other number of clients can use a different number of threads. This
constraint does not make much sense.
..
Minor v2 update to change a not badly chosen variable name.
+1, it's really annoying that you can't just do -j<high number> and
always run with that.
This doesn't behave correctly if you set -j to greater than -c, and also
use the rate limit option:
---- This works ----
$ ./pgbench -R 100 -j3 -c 3 -T 10 postgres
starting vacuum...end.
transaction type: TPC-B (sort of)
scaling factor: 5
query mode: simple
number of clients: 3
number of threads: 3
duration: 10 s
number of transactions actually processed: 1031
latency average: 1.397 ms
latency stddev: 0.276 ms
rate limit schedule lag: avg 0.119 (max 1.949) ms
tps = 102.947257 (including connections establishing)
tps = 102.967251 (excluding connections establishing)
---- This does not; too small TPS ----
$ ./pgbench -R 100 -j100 -c 3 -T 10 postgres
starting vacuum...end.
transaction type: TPC-B (sort of)
scaling factor: 5
query mode: simple
number of clients: 3
number of threads: 100
duration: 10 s
number of transactions actually processed: 40
latency average: 3.573 ms
latency stddev: 1.724 ms
rate limit schedule lag: avg 0.813 (max 4.722) ms
tps = 3.246618 (including connections establishing)
tps = 3.246639 (excluding connections establishing)
I think that can be fixed by just setting nthreads internally to
nclients, if nthreads > nclients. But please double-check that the logic
used to calculate throttle_delay makes sense in general, when nthreads
is not a multiple of nclients.
- Heikki
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
This doesn't behave correctly if you set -j to greater than -c, and also use
the rate limit option:
[...]I think that can be fixed by just setting nthreads internally to nclients, if
nthreads > nclients. But please double-check that the logic used to calculate
throttle_delay makes sense in general, when nthreads is not a multiple of
nclients.
Indeed, I probably did not consider nthreads > nclients. The fix you
suggest seems ok. I'll check it before sending a new v3.
--
Fabien.
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
This doesn't behave correctly if you set -j to greater than -c, and also use
the rate limit option:
Indeed. v3 attached fixed the case when nthreads > nclients.
--
Fabien.
Attachments:
pgbench-nclients-3.patchtext/x-diff; charset=us-ascii; name=pgbench-nclients-3.patchDownload
diff --git a/doc/src/sgml/ref/pgbench.sgml b/doc/src/sgml/ref/pgbench.sgml
index a808546..2517a3a 100644
--- a/doc/src/sgml/ref/pgbench.sgml
+++ b/doc/src/sgml/ref/pgbench.sgml
@@ -326,8 +326,7 @@ pgbench <optional> <replaceable>options</> </optional> <replaceable>dbname</>
<para>
Number of worker threads within <application>pgbench</application>.
Using more than one thread can be helpful on multi-CPU machines.
- The number of clients must be a multiple of the number of threads,
- since each thread is given the same number of client sessions to manage.
+ Clients are distributed as evenly as possible among available threads.
Default is 1.
</para>
</listitem>
diff --git a/src/bin/pgbench/pgbench.c b/src/bin/pgbench/pgbench.c
index 2c3e365..8dfc2fb 100644
--- a/src/bin/pgbench/pgbench.c
+++ b/src/bin/pgbench/pgbench.c
@@ -2792,6 +2792,7 @@ main(int argc, char **argv)
int c;
int nclients = 1; /* default number of simulated clients */
int nthreads = 1; /* default number of threads */
+ int nclients_dealt; /* clients distributed between threads */
int is_init_mode = 0; /* initialize mode? */
int is_no_vacuum = 0; /* no vacuum at all before testing? */
int do_vacuum_accounts = 0; /* do vacuum accounts before testing? */
@@ -3114,6 +3115,10 @@ main(int argc, char **argv)
}
}
+ /* Adjust threads to clients */
+ if (nthreads > nclients)
+ nthreads = nclients;
+
/* compute a per thread delay */
throttle_delay *= nthreads;
@@ -3153,12 +3158,6 @@ main(int argc, char **argv)
if (nxacts <= 0 && duration <= 0)
nxacts = DEFAULT_NXACTS;
- if (nclients % nthreads != 0)
- {
- fprintf(stderr, "number of clients (%d) must be a multiple of number of threads (%d)\n", nclients, nthreads);
- exit(1);
- }
-
/* --sampling-rate may be used only with -l */
if (sample_rate > 0.0 && !use_log)
{
@@ -3359,19 +3358,24 @@ main(int argc, char **argv)
/* set up thread data structures */
threads = (TState *) pg_malloc(sizeof(TState) * nthreads);
+ nclients_dealt = 0;
+
for (i = 0; i < nthreads; i++)
{
TState *thread = &threads[i];
thread->tid = i;
- thread->state = &state[nclients / nthreads * i];
- thread->nstate = nclients / nthreads;
+ thread->state = &state[nclients_dealt];
+ thread->nstate =
+ (nclients - nclients_dealt + nthreads - i - 1) / (nthreads - i);
thread->random_state[0] = random();
thread->random_state[1] = random();
thread->random_state[2] = random();
thread->throttle_latency_skipped = 0;
thread->latency_late = 0;
+ nclients_dealt += thread->nstate;
+
if (is_latencies)
{
/* Reserve memory for the thread to store per-command latencies */
@@ -3395,6 +3399,9 @@ main(int argc, char **argv)
}
}
+ /* all clients must be assigned to a thread */
+ Assert(nclients_dealt == nclients);
+
/* get start up time */
INSTR_TIME_SET_CURRENT(start_time);
On 07/03/2015 07:50 AM, Fabien COELHO wrote:
This doesn't behave correctly if you set -j to greater than -c, and also use
the rate limit option:Indeed. v3 attached fixed the case when nthreads > nclients.
Ok, committed. Thanks!
- Heikki
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
Indeed. v3 attached fixed the case when nthreads > nclients.
Ok, committed. Thanks!
Thanks!
--
Fabien.
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers